1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Intel Corporation 3 */ 4 5 #include <unistd.h> 6 7 #include <rte_common.h> 8 #include <rte_log.h> 9 #include <rte_dev.h> 10 #include <rte_malloc.h> 11 #include <rte_mempool.h> 12 #include <rte_errno.h> 13 #include <rte_pci.h> 14 #include <rte_bus_pci.h> 15 #include <rte_byteorder.h> 16 #ifdef RTE_BBDEV_OFFLOAD_COST 17 #include <rte_cycles.h> 18 #endif 19 20 #include <rte_bbdev.h> 21 #include <rte_bbdev_pmd.h> 22 23 #include "fpga_5gnr_fec.h" 24 #include "rte_pmd_fpga_5gnr_fec.h" 25 26 #ifdef RTE_LIBRTE_BBDEV_DEBUG 27 RTE_LOG_REGISTER_DEFAULT(fpga_5gnr_fec_logtype, DEBUG); 28 #else 29 RTE_LOG_REGISTER_DEFAULT(fpga_5gnr_fec_logtype, NOTICE); 30 #endif 31 32 #ifdef RTE_LIBRTE_BBDEV_DEBUG 33 34 /* Read Ring Control Register of FPGA 5GNR FEC device */ 35 static inline void 36 print_ring_reg_debug_info(void *mmio_base, uint32_t offset) 37 { 38 rte_bbdev_log_debug( 39 "FPGA MMIO base address @ %p | Ring Control Register @ offset = 0x%08" 40 PRIx32, mmio_base, offset); 41 rte_bbdev_log_debug( 42 "RING_BASE_ADDR = 0x%016"PRIx64, 43 fpga_reg_read_64(mmio_base, offset)); 44 rte_bbdev_log_debug( 45 "RING_HEAD_ADDR = 0x%016"PRIx64, 46 fpga_reg_read_64(mmio_base, offset + 47 FPGA_5GNR_FEC_RING_HEAD_ADDR)); 48 rte_bbdev_log_debug( 49 "RING_SIZE = 0x%04"PRIx16, 50 fpga_reg_read_16(mmio_base, offset + 51 FPGA_5GNR_FEC_RING_SIZE)); 52 rte_bbdev_log_debug( 53 "RING_MISC = 0x%02"PRIx8, 54 fpga_reg_read_8(mmio_base, offset + 55 FPGA_5GNR_FEC_RING_MISC)); 56 rte_bbdev_log_debug( 57 "RING_ENABLE = 0x%02"PRIx8, 58 fpga_reg_read_8(mmio_base, offset + 59 FPGA_5GNR_FEC_RING_ENABLE)); 60 rte_bbdev_log_debug( 61 "RING_FLUSH_QUEUE_EN = 0x%02"PRIx8, 62 fpga_reg_read_8(mmio_base, offset + 63 FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN)); 64 rte_bbdev_log_debug( 65 "RING_SHADOW_TAIL = 0x%04"PRIx16, 66 fpga_reg_read_16(mmio_base, offset + 67 FPGA_5GNR_FEC_RING_SHADOW_TAIL)); 68 rte_bbdev_log_debug( 69 "RING_HEAD_POINT = 0x%04"PRIx16, 70 fpga_reg_read_16(mmio_base, offset + 71 FPGA_5GNR_FEC_RING_HEAD_POINT)); 72 } 73 74 /* Read Static Register of FPGA 5GNR FEC device */ 75 static inline void 76 print_static_reg_debug_info(void *mmio_base) 77 { 78 uint16_t config = fpga_reg_read_16(mmio_base, 79 FPGA_5GNR_FEC_CONFIGURATION); 80 uint8_t qmap_done = fpga_reg_read_8(mmio_base, 81 FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE); 82 uint16_t lb_factor = fpga_reg_read_16(mmio_base, 83 FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR); 84 uint16_t ring_desc_len = fpga_reg_read_16(mmio_base, 85 FPGA_5GNR_FEC_RING_DESC_LEN); 86 87 rte_bbdev_log_debug("UL.DL Weights = %u.%u", 88 ((uint8_t)config), ((uint8_t)(config >> 8))); 89 rte_bbdev_log_debug("UL.DL Load Balance = %u.%u", 90 ((uint8_t)lb_factor), ((uint8_t)(lb_factor >> 8))); 91 rte_bbdev_log_debug("Queue-PF/VF Mapping Table = %s", 92 (qmap_done > 0) ? "READY" : "NOT-READY"); 93 rte_bbdev_log_debug("Ring Descriptor Size = %u bytes", 94 ring_desc_len*FPGA_RING_DESC_LEN_UNIT_BYTES); 95 } 96 97 /* Print decode DMA Descriptor of FPGA 5GNR Decoder device */ 98 static void 99 print_dma_dec_desc_debug_info(union fpga_dma_desc *desc) 100 { 101 rte_bbdev_log_debug("DMA response desc %p\n" 102 "\t-- done(%"PRIu32") | iter(%"PRIu32") | et_pass(%"PRIu32")" 103 " | crcb_pass (%"PRIu32") | error(%"PRIu32")\n" 104 "\t-- qm_idx(%"PRIu32") | max_iter(%"PRIu32") | " 105 "bg_idx (%"PRIu32") | harqin_en(%"PRIu32") | zc(%"PRIu32")\n" 106 "\t-- hbstroe_offset(%"PRIu32") | num_null (%"PRIu32") " 107 "| irq_en(%"PRIu32")\n" 108 "\t-- ncb(%"PRIu32") | desc_idx (%"PRIu32") | " 109 "drop_crc24b(%"PRIu32") | RV (%"PRIu32")\n" 110 "\t-- crc24b_ind(%"PRIu32") | et_dis (%"PRIu32")\n" 111 "\t-- harq_input_length(%"PRIu32") | rm_e(%"PRIu32")\n" 112 "\t-- cbs_in_op(%"PRIu32") | in_add (0x%08"PRIx32"%08"PRIx32")" 113 "| out_add (0x%08"PRIx32"%08"PRIx32")", 114 desc, 115 (uint32_t)desc->dec_req.done, 116 (uint32_t)desc->dec_req.iter, 117 (uint32_t)desc->dec_req.et_pass, 118 (uint32_t)desc->dec_req.crcb_pass, 119 (uint32_t)desc->dec_req.error, 120 (uint32_t)desc->dec_req.qm_idx, 121 (uint32_t)desc->dec_req.max_iter, 122 (uint32_t)desc->dec_req.bg_idx, 123 (uint32_t)desc->dec_req.harqin_en, 124 (uint32_t)desc->dec_req.zc, 125 (uint32_t)desc->dec_req.hbstroe_offset, 126 (uint32_t)desc->dec_req.num_null, 127 (uint32_t)desc->dec_req.irq_en, 128 (uint32_t)desc->dec_req.ncb, 129 (uint32_t)desc->dec_req.desc_idx, 130 (uint32_t)desc->dec_req.drop_crc24b, 131 (uint32_t)desc->dec_req.rv, 132 (uint32_t)desc->dec_req.crc24b_ind, 133 (uint32_t)desc->dec_req.et_dis, 134 (uint32_t)desc->dec_req.harq_input_length, 135 (uint32_t)desc->dec_req.rm_e, 136 (uint32_t)desc->dec_req.cbs_in_op, 137 (uint32_t)desc->dec_req.in_addr_hi, 138 (uint32_t)desc->dec_req.in_addr_lw, 139 (uint32_t)desc->dec_req.out_addr_hi, 140 (uint32_t)desc->dec_req.out_addr_lw); 141 uint32_t *word = (uint32_t *) desc; 142 rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n" 143 "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n", 144 word[0], word[1], word[2], word[3], 145 word[4], word[5], word[6], word[7]); 146 } 147 148 /* Print decode DMA Descriptor of FPGA 5GNR encoder device */ 149 static void 150 print_dma_enc_desc_debug_info(union fpga_dma_desc *desc) 151 { 152 rte_bbdev_log_debug("DMA response desc %p\n" 153 "%"PRIu32" %"PRIu32"\n" 154 "K' %"PRIu32" E %"PRIu32" desc %"PRIu32" Z %"PRIu32"\n" 155 "BG %"PRIu32" Qm %"PRIu32" CRC %"PRIu32" IRQ %"PRIu32"\n" 156 "k0 %"PRIu32" Ncb %"PRIu32" F %"PRIu32"\n", 157 desc, 158 (uint32_t)desc->enc_req.done, 159 (uint32_t)desc->enc_req.error, 160 161 (uint32_t)desc->enc_req.k_, 162 (uint32_t)desc->enc_req.rm_e, 163 (uint32_t)desc->enc_req.desc_idx, 164 (uint32_t)desc->enc_req.zc, 165 166 (uint32_t)desc->enc_req.bg_idx, 167 (uint32_t)desc->enc_req.qm_idx, 168 (uint32_t)desc->enc_req.crc_en, 169 (uint32_t)desc->enc_req.irq_en, 170 171 (uint32_t)desc->enc_req.k0, 172 (uint32_t)desc->enc_req.ncb, 173 (uint32_t)desc->enc_req.num_null); 174 uint32_t *word = (uint32_t *) desc; 175 rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n" 176 "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n", 177 word[0], word[1], word[2], word[3], 178 word[4], word[5], word[6], word[7]); 179 } 180 181 #endif 182 183 static int 184 fpga_setup_queues(struct rte_bbdev *dev, uint16_t num_queues, int socket_id) 185 { 186 /* Number of queues bound to a PF/VF */ 187 uint32_t hw_q_num = 0; 188 uint32_t ring_size, payload, address, q_id, offset; 189 rte_iova_t phys_addr; 190 struct fpga_ring_ctrl_reg ring_reg; 191 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private; 192 193 address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE; 194 if (!(fpga_reg_read_32(fpga_dev->mmio_base, address) & 0x1)) { 195 rte_bbdev_log(ERR, 196 "Queue-PF/VF mapping is not set! Was PF configured for device (%s) ?", 197 dev->data->name); 198 return -EPERM; 199 } 200 201 /* Clear queue registers structure */ 202 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg)); 203 204 /* Scan queue map. 205 * If a queue is valid and mapped to a calling PF/VF the read value is 206 * replaced with a queue ID and if it's not then 207 * FPGA_INVALID_HW_QUEUE_ID is returned. 208 */ 209 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 210 uint32_t hw_q_id = fpga_reg_read_32(fpga_dev->mmio_base, 211 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2)); 212 213 rte_bbdev_log_debug("%s: queue ID: %u, registry queue ID: %u", 214 dev->device->name, q_id, hw_q_id); 215 216 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) { 217 fpga_dev->q_bound_bit_map |= (1ULL << q_id); 218 /* Clear queue register of found queue */ 219 offset = FPGA_5GNR_FEC_RING_CTRL_REGS + 220 (sizeof(struct fpga_ring_ctrl_reg) * q_id); 221 fpga_ring_reg_write(fpga_dev->mmio_base, 222 offset, ring_reg); 223 ++hw_q_num; 224 } 225 } 226 if (hw_q_num == 0) { 227 rte_bbdev_log(ERR, 228 "No HW queues assigned to this device. Probably this is a VF configured for PF mode. Check device configuration!"); 229 return -ENODEV; 230 } 231 232 if (num_queues > hw_q_num) { 233 rte_bbdev_log(ERR, 234 "Not enough queues for device %s! Requested: %u, available: %u", 235 dev->device->name, num_queues, hw_q_num); 236 return -EINVAL; 237 } 238 239 ring_size = FPGA_RING_MAX_SIZE * sizeof(struct fpga_dma_dec_desc); 240 241 /* Enforce 32 byte alignment */ 242 RTE_BUILD_BUG_ON((RTE_CACHE_LINE_SIZE % 32) != 0); 243 244 /* Allocate memory for SW descriptor rings */ 245 fpga_dev->sw_rings = rte_zmalloc_socket(dev->device->driver->name, 246 num_queues * ring_size, RTE_CACHE_LINE_SIZE, 247 socket_id); 248 if (fpga_dev->sw_rings == NULL) { 249 rte_bbdev_log(ERR, 250 "Failed to allocate memory for %s:%u sw_rings", 251 dev->device->driver->name, dev->data->dev_id); 252 return -ENOMEM; 253 } 254 255 fpga_dev->sw_rings_phys = rte_malloc_virt2iova(fpga_dev->sw_rings); 256 fpga_dev->sw_ring_size = ring_size; 257 fpga_dev->sw_ring_max_depth = FPGA_RING_MAX_SIZE; 258 259 /* Allocate memory for ring flush status */ 260 fpga_dev->flush_queue_status = rte_zmalloc_socket(NULL, 261 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, socket_id); 262 if (fpga_dev->flush_queue_status == NULL) { 263 rte_bbdev_log(ERR, 264 "Failed to allocate memory for %s:%u flush_queue_status", 265 dev->device->driver->name, dev->data->dev_id); 266 return -ENOMEM; 267 } 268 269 /* Set the flush status address registers */ 270 phys_addr = rte_malloc_virt2iova(fpga_dev->flush_queue_status); 271 272 address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_LW; 273 payload = (uint32_t)(phys_addr); 274 fpga_reg_write_32(fpga_dev->mmio_base, address, payload); 275 276 address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_HI; 277 payload = (uint32_t)(phys_addr >> 32); 278 fpga_reg_write_32(fpga_dev->mmio_base, address, payload); 279 280 return 0; 281 } 282 283 static int 284 fpga_dev_close(struct rte_bbdev *dev) 285 { 286 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private; 287 288 rte_free(fpga_dev->sw_rings); 289 rte_free(fpga_dev->flush_queue_status); 290 291 return 0; 292 } 293 294 static void 295 fpga_dev_info_get(struct rte_bbdev *dev, 296 struct rte_bbdev_driver_info *dev_info) 297 { 298 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 299 uint32_t q_id = 0; 300 301 static const struct rte_bbdev_op_cap bbdev_capabilities[] = { 302 { 303 .type = RTE_BBDEV_OP_LDPC_ENC, 304 .cap.ldpc_enc = { 305 .capability_flags = 306 RTE_BBDEV_LDPC_RATE_MATCH | 307 RTE_BBDEV_LDPC_ENC_INTERRUPTS | 308 RTE_BBDEV_LDPC_CRC_24B_ATTACH, 309 .num_buffers_src = 310 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS, 311 .num_buffers_dst = 312 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS, 313 } 314 }, 315 { 316 .type = RTE_BBDEV_OP_LDPC_DEC, 317 .cap.ldpc_dec = { 318 .capability_flags = 319 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK | 320 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP | 321 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE | 322 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE | 323 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE | 324 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE | 325 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE | 326 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK | 327 RTE_BBDEV_LDPC_DEC_INTERRUPTS | 328 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS, 329 .llr_size = 6, 330 .llr_decimals = 2, 331 .num_buffers_src = 332 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS, 333 .num_buffers_hard_out = 334 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS, 335 .num_buffers_soft_out = 0, 336 } 337 }, 338 RTE_BBDEV_END_OF_CAPABILITIES_LIST() 339 }; 340 341 /* Check the HARQ DDR size available */ 342 uint8_t timeout_counter = 0; 343 uint32_t harq_buf_ready = fpga_reg_read_32(d->mmio_base, 344 FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS); 345 while (harq_buf_ready != 1) { 346 usleep(FPGA_TIMEOUT_CHECK_INTERVAL); 347 timeout_counter++; 348 harq_buf_ready = fpga_reg_read_32(d->mmio_base, 349 FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS); 350 if (timeout_counter > FPGA_HARQ_RDY_TIMEOUT) { 351 rte_bbdev_log(ERR, "HARQ Buffer not ready %d", 352 harq_buf_ready); 353 harq_buf_ready = 1; 354 } 355 } 356 uint32_t harq_buf_size = fpga_reg_read_32(d->mmio_base, 357 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS); 358 359 static struct rte_bbdev_queue_conf default_queue_conf; 360 default_queue_conf.socket = dev->data->socket_id; 361 default_queue_conf.queue_size = FPGA_RING_MAX_SIZE; 362 363 dev_info->driver_name = dev->device->driver->name; 364 dev_info->queue_size_lim = FPGA_RING_MAX_SIZE; 365 dev_info->hardware_accelerated = true; 366 dev_info->min_alignment = 64; 367 dev_info->harq_buffer_size = (harq_buf_size >> 10) + 1; 368 dev_info->default_queue_conf = default_queue_conf; 369 dev_info->capabilities = bbdev_capabilities; 370 dev_info->cpu_flag_reqs = NULL; 371 dev_info->data_endianness = RTE_LITTLE_ENDIAN; 372 373 /* Calculates number of queues assigned to device */ 374 dev_info->max_num_queues = 0; 375 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 376 uint32_t hw_q_id = fpga_reg_read_32(d->mmio_base, 377 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2)); 378 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) 379 dev_info->max_num_queues++; 380 } 381 } 382 383 /** 384 * Find index of queue bound to current PF/VF which is unassigned. Return -1 385 * when there is no available queue 386 */ 387 static inline int 388 fpga_find_free_queue_idx(struct rte_bbdev *dev, 389 const struct rte_bbdev_queue_conf *conf) 390 { 391 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 392 uint64_t q_idx; 393 uint8_t i = 0; 394 uint8_t range = FPGA_TOTAL_NUM_QUEUES >> 1; 395 396 if (conf->op_type == RTE_BBDEV_OP_LDPC_ENC) { 397 i = FPGA_NUM_DL_QUEUES; 398 range = FPGA_TOTAL_NUM_QUEUES; 399 } 400 401 for (; i < range; ++i) { 402 q_idx = 1ULL << i; 403 /* Check if index of queue is bound to current PF/VF */ 404 if (d->q_bound_bit_map & q_idx) 405 /* Check if found queue was not already assigned */ 406 if (!(d->q_assigned_bit_map & q_idx)) { 407 d->q_assigned_bit_map |= q_idx; 408 return i; 409 } 410 } 411 412 rte_bbdev_log(INFO, "Failed to find free queue on %s", dev->data->name); 413 414 return -1; 415 } 416 417 static int 418 fpga_queue_setup(struct rte_bbdev *dev, uint16_t queue_id, 419 const struct rte_bbdev_queue_conf *conf) 420 { 421 uint32_t address, ring_offset; 422 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 423 struct fpga_queue *q; 424 int8_t q_idx; 425 426 /* Check if there is a free queue to assign */ 427 q_idx = fpga_find_free_queue_idx(dev, conf); 428 if (q_idx == -1) 429 return -1; 430 431 /* Allocate the queue data structure. */ 432 q = rte_zmalloc_socket(dev->device->driver->name, sizeof(*q), 433 RTE_CACHE_LINE_SIZE, conf->socket); 434 if (q == NULL) { 435 /* Mark queue as un-assigned */ 436 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 437 rte_bbdev_log(ERR, "Failed to allocate queue memory"); 438 return -ENOMEM; 439 } 440 441 q->d = d; 442 q->q_idx = q_idx; 443 444 /* Set ring_base_addr */ 445 q->ring_addr = RTE_PTR_ADD(d->sw_rings, (d->sw_ring_size * queue_id)); 446 q->ring_ctrl_reg.ring_base_addr = d->sw_rings_phys + 447 (d->sw_ring_size * queue_id); 448 449 /* Allocate memory for Completion Head variable*/ 450 q->ring_head_addr = rte_zmalloc_socket(dev->device->driver->name, 451 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, conf->socket); 452 if (q->ring_head_addr == NULL) { 453 /* Mark queue as un-assigned */ 454 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 455 rte_free(q); 456 rte_bbdev_log(ERR, 457 "Failed to allocate memory for %s:%u completion_head", 458 dev->device->driver->name, dev->data->dev_id); 459 return -ENOMEM; 460 } 461 /* Set ring_head_addr */ 462 q->ring_ctrl_reg.ring_head_addr = 463 rte_malloc_virt2iova(q->ring_head_addr); 464 465 /* Clear shadow_completion_head */ 466 q->shadow_completion_head = 0; 467 468 /* Set ring_size */ 469 if (conf->queue_size > FPGA_RING_MAX_SIZE) { 470 /* Mark queue as un-assigned */ 471 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 472 rte_free(q->ring_head_addr); 473 rte_free(q); 474 rte_bbdev_log(ERR, 475 "Size of queue is too big %d (MAX: %d ) for %s:%u", 476 conf->queue_size, FPGA_RING_MAX_SIZE, 477 dev->device->driver->name, dev->data->dev_id); 478 return -EINVAL; 479 } 480 q->ring_ctrl_reg.ring_size = conf->queue_size; 481 482 /* Set Miscellaneous FPGA register*/ 483 /* Max iteration number for TTI mitigation - todo */ 484 q->ring_ctrl_reg.max_ul_dec = 0; 485 /* Enable max iteration number for TTI - todo */ 486 q->ring_ctrl_reg.max_ul_dec_en = 0; 487 488 /* Enable the ring */ 489 q->ring_ctrl_reg.enable = 1; 490 491 /* Set FPGA head_point and tail registers */ 492 q->ring_ctrl_reg.head_point = q->tail = 0; 493 494 /* Set FPGA shadow_tail register */ 495 q->ring_ctrl_reg.shadow_tail = q->tail; 496 497 /* Calculates the ring offset for found queue */ 498 ring_offset = FPGA_5GNR_FEC_RING_CTRL_REGS + 499 (sizeof(struct fpga_ring_ctrl_reg) * q_idx); 500 501 /* Set FPGA Ring Control Registers */ 502 fpga_ring_reg_write(d->mmio_base, ring_offset, q->ring_ctrl_reg); 503 504 /* Store MMIO register of shadow_tail */ 505 address = ring_offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL; 506 q->shadow_tail_addr = RTE_PTR_ADD(d->mmio_base, address); 507 508 q->head_free_desc = q->tail; 509 510 /* Set wrap mask */ 511 q->sw_ring_wrap_mask = conf->queue_size - 1; 512 513 rte_bbdev_log_debug("Setup dev%u q%u: queue_idx=%u", 514 dev->data->dev_id, queue_id, q->q_idx); 515 516 dev->data->queues[queue_id].queue_private = q; 517 518 rte_bbdev_log_debug("BBDEV queue[%d] set up for FPGA queue[%d]", 519 queue_id, q_idx); 520 521 #ifdef RTE_LIBRTE_BBDEV_DEBUG 522 /* Read FPGA Ring Control Registers after configuration*/ 523 print_ring_reg_debug_info(d->mmio_base, ring_offset); 524 #endif 525 return 0; 526 } 527 528 static int 529 fpga_queue_release(struct rte_bbdev *dev, uint16_t queue_id) 530 { 531 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 532 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 533 struct fpga_ring_ctrl_reg ring_reg; 534 uint32_t offset; 535 536 rte_bbdev_log_debug("FPGA Queue[%d] released", queue_id); 537 538 if (q != NULL) { 539 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg)); 540 offset = FPGA_5GNR_FEC_RING_CTRL_REGS + 541 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 542 /* Disable queue */ 543 fpga_reg_write_8(d->mmio_base, 544 offset + FPGA_5GNR_FEC_RING_ENABLE, 0x00); 545 /* Clear queue registers */ 546 fpga_ring_reg_write(d->mmio_base, offset, ring_reg); 547 548 /* Mark the Queue as un-assigned */ 549 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q->q_idx)); 550 rte_free(q->ring_head_addr); 551 rte_free(q); 552 dev->data->queues[queue_id].queue_private = NULL; 553 } 554 555 return 0; 556 } 557 558 /* Function starts a device queue. */ 559 static int 560 fpga_queue_start(struct rte_bbdev *dev, uint16_t queue_id) 561 { 562 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 563 #ifdef RTE_LIBRTE_BBDEV_DEBUG 564 if (d == NULL) { 565 rte_bbdev_log(ERR, "Invalid device pointer"); 566 return -1; 567 } 568 #endif 569 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 570 uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS + 571 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 572 uint8_t enable = 0x01; 573 uint16_t zero = 0x0000; 574 575 /* Clear queue head and tail variables */ 576 q->tail = q->head_free_desc = 0; 577 578 /* Clear FPGA head_point and tail registers */ 579 fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_HEAD_POINT, 580 zero); 581 fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL, 582 zero); 583 584 /* Enable queue */ 585 fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE, 586 enable); 587 588 rte_bbdev_log_debug("FPGA Queue[%d] started", queue_id); 589 return 0; 590 } 591 592 /* Function stops a device queue. */ 593 static int 594 fpga_queue_stop(struct rte_bbdev *dev, uint16_t queue_id) 595 { 596 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 597 #ifdef RTE_LIBRTE_BBDEV_DEBUG 598 if (d == NULL) { 599 rte_bbdev_log(ERR, "Invalid device pointer"); 600 return -1; 601 } 602 #endif 603 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 604 uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS + 605 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 606 uint8_t payload = 0x01; 607 uint8_t counter = 0; 608 uint8_t timeout = FPGA_QUEUE_FLUSH_TIMEOUT_US / 609 FPGA_TIMEOUT_CHECK_INTERVAL; 610 611 /* Set flush_queue_en bit to trigger queue flushing */ 612 fpga_reg_write_8(d->mmio_base, 613 offset + FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN, payload); 614 615 /** Check if queue flush is completed. 616 * FPGA will update the completion flag after queue flushing is 617 * completed. If completion flag is not updated within 1ms it is 618 * considered as a failure. 619 */ 620 while (!(*((volatile uint8_t *)d->flush_queue_status + q->q_idx) 621 & payload)) { 622 if (counter > timeout) { 623 rte_bbdev_log(ERR, "FPGA Queue Flush failed for queue %d", 624 queue_id); 625 return -1; 626 } 627 usleep(FPGA_TIMEOUT_CHECK_INTERVAL); 628 counter++; 629 } 630 631 /* Disable queue */ 632 payload = 0x00; 633 fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE, 634 payload); 635 636 rte_bbdev_log_debug("FPGA Queue[%d] stopped", queue_id); 637 return 0; 638 } 639 640 static inline uint16_t 641 get_queue_id(struct rte_bbdev_data *data, uint8_t q_idx) 642 { 643 uint16_t queue_id; 644 645 for (queue_id = 0; queue_id < data->num_queues; ++queue_id) { 646 struct fpga_queue *q = data->queues[queue_id].queue_private; 647 if (q != NULL && q->q_idx == q_idx) 648 return queue_id; 649 } 650 651 return -1; 652 } 653 654 /* Interrupt handler triggered by FPGA dev for handling specific interrupt */ 655 static void 656 fpga_dev_interrupt_handler(void *cb_arg) 657 { 658 struct rte_bbdev *dev = cb_arg; 659 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private; 660 struct fpga_queue *q; 661 uint64_t ring_head; 662 uint64_t q_idx; 663 uint16_t queue_id; 664 uint8_t i; 665 666 /* Scan queue assigned to this device */ 667 for (i = 0; i < FPGA_TOTAL_NUM_QUEUES; ++i) { 668 q_idx = 1ULL << i; 669 if (fpga_dev->q_bound_bit_map & q_idx) { 670 queue_id = get_queue_id(dev->data, i); 671 if (queue_id == (uint16_t) -1) 672 continue; 673 674 /* Check if completion head was changed */ 675 q = dev->data->queues[queue_id].queue_private; 676 ring_head = *q->ring_head_addr; 677 if (q->shadow_completion_head != ring_head && 678 q->irq_enable == 1) { 679 q->shadow_completion_head = ring_head; 680 rte_bbdev_pmd_callback_process( 681 dev, 682 RTE_BBDEV_EVENT_DEQUEUE, 683 &queue_id); 684 } 685 } 686 } 687 } 688 689 static int 690 fpga_queue_intr_enable(struct rte_bbdev *dev, uint16_t queue_id) 691 { 692 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 693 694 if (!rte_intr_cap_multiple(dev->intr_handle)) 695 return -ENOTSUP; 696 697 q->irq_enable = 1; 698 699 return 0; 700 } 701 702 static int 703 fpga_queue_intr_disable(struct rte_bbdev *dev, uint16_t queue_id) 704 { 705 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 706 q->irq_enable = 0; 707 708 return 0; 709 } 710 711 static int 712 fpga_intr_enable(struct rte_bbdev *dev) 713 { 714 int ret; 715 uint8_t i; 716 717 if (!rte_intr_cap_multiple(dev->intr_handle)) { 718 rte_bbdev_log(ERR, "Multiple intr vector is not supported by FPGA (%s)", 719 dev->data->name); 720 return -ENOTSUP; 721 } 722 723 /* Create event file descriptors for each of 64 queue. Event fds will be 724 * mapped to FPGA IRQs in rte_intr_enable(). This is a 1:1 mapping where 725 * the IRQ number is a direct translation to the queue number. 726 * 727 * 63 (FPGA_NUM_INTR_VEC) event fds are created as rte_intr_enable() 728 * mapped the first IRQ to already created interrupt event file 729 * descriptor (intr_handle->fd). 730 */ 731 if (rte_intr_efd_enable(dev->intr_handle, FPGA_NUM_INTR_VEC)) { 732 rte_bbdev_log(ERR, "Failed to create fds for %u queues", 733 dev->data->num_queues); 734 return -1; 735 } 736 737 /* TODO Each event file descriptor is overwritten by interrupt event 738 * file descriptor. That descriptor is added to epoll observed list. 739 * It ensures that callback function assigned to that descriptor will 740 * invoked when any FPGA queue issues interrupt. 741 */ 742 for (i = 0; i < FPGA_NUM_INTR_VEC; ++i) { 743 if (rte_intr_efds_index_set(dev->intr_handle, i, 744 rte_intr_fd_get(dev->intr_handle))) 745 return -rte_errno; 746 } 747 748 if (rte_intr_vec_list_alloc(dev->intr_handle, "intr_vec", 749 dev->data->num_queues)) { 750 rte_bbdev_log(ERR, "Failed to allocate %u vectors", 751 dev->data->num_queues); 752 return -ENOMEM; 753 } 754 755 ret = rte_intr_enable(dev->intr_handle); 756 if (ret < 0) { 757 rte_bbdev_log(ERR, 758 "Couldn't enable interrupts for device: %s", 759 dev->data->name); 760 return ret; 761 } 762 763 ret = rte_intr_callback_register(dev->intr_handle, 764 fpga_dev_interrupt_handler, dev); 765 if (ret < 0) { 766 rte_bbdev_log(ERR, 767 "Couldn't register interrupt callback for device: %s", 768 dev->data->name); 769 return ret; 770 } 771 772 return 0; 773 } 774 775 static const struct rte_bbdev_ops fpga_ops = { 776 .setup_queues = fpga_setup_queues, 777 .intr_enable = fpga_intr_enable, 778 .close = fpga_dev_close, 779 .info_get = fpga_dev_info_get, 780 .queue_setup = fpga_queue_setup, 781 .queue_stop = fpga_queue_stop, 782 .queue_start = fpga_queue_start, 783 .queue_release = fpga_queue_release, 784 .queue_intr_enable = fpga_queue_intr_enable, 785 .queue_intr_disable = fpga_queue_intr_disable 786 }; 787 788 static inline void 789 fpga_dma_enqueue(struct fpga_queue *q, uint16_t num_desc, 790 struct rte_bbdev_stats *queue_stats) 791 { 792 #ifdef RTE_BBDEV_OFFLOAD_COST 793 uint64_t start_time = 0; 794 queue_stats->acc_offload_cycles = 0; 795 #else 796 RTE_SET_USED(queue_stats); 797 #endif 798 799 /* Update tail and shadow_tail register */ 800 q->tail = (q->tail + num_desc) & q->sw_ring_wrap_mask; 801 802 rte_wmb(); 803 804 #ifdef RTE_BBDEV_OFFLOAD_COST 805 /* Start time measurement for enqueue function offload. */ 806 start_time = rte_rdtsc_precise(); 807 #endif 808 mmio_write_16(q->shadow_tail_addr, q->tail); 809 810 #ifdef RTE_BBDEV_OFFLOAD_COST 811 rte_wmb(); 812 queue_stats->acc_offload_cycles += rte_rdtsc_precise() - start_time; 813 #endif 814 } 815 816 /* Read flag value 0/1/ from bitmap */ 817 static inline bool 818 check_bit(uint32_t bitmap, uint32_t bitmask) 819 { 820 return bitmap & bitmask; 821 } 822 823 /* Print an error if a descriptor error has occurred. 824 * Return 0 on success, 1 on failure 825 */ 826 static inline int 827 check_desc_error(uint32_t error_code) { 828 switch (error_code) { 829 case DESC_ERR_NO_ERR: 830 return 0; 831 case DESC_ERR_K_P_OUT_OF_RANGE: 832 rte_bbdev_log(ERR, "Encode block size K' is out of range"); 833 break; 834 case DESC_ERR_Z_C_NOT_LEGAL: 835 rte_bbdev_log(ERR, "Zc is illegal"); 836 break; 837 case DESC_ERR_DESC_OFFSET_ERR: 838 rte_bbdev_log(ERR, 839 "Queue offset does not meet the expectation in the FPGA" 840 ); 841 break; 842 case DESC_ERR_DESC_READ_FAIL: 843 rte_bbdev_log(ERR, "Unsuccessful completion for descriptor read"); 844 break; 845 case DESC_ERR_DESC_READ_TIMEOUT: 846 rte_bbdev_log(ERR, "Descriptor read time-out"); 847 break; 848 case DESC_ERR_DESC_READ_TLP_POISONED: 849 rte_bbdev_log(ERR, "Descriptor read TLP poisoned"); 850 break; 851 case DESC_ERR_HARQ_INPUT_LEN: 852 rte_bbdev_log(ERR, "HARQ input length is invalid"); 853 break; 854 case DESC_ERR_CB_READ_FAIL: 855 rte_bbdev_log(ERR, "Unsuccessful completion for code block"); 856 break; 857 case DESC_ERR_CB_READ_TIMEOUT: 858 rte_bbdev_log(ERR, "Code block read time-out"); 859 break; 860 case DESC_ERR_CB_READ_TLP_POISONED: 861 rte_bbdev_log(ERR, "Code block read TLP poisoned"); 862 break; 863 case DESC_ERR_HBSTORE_ERR: 864 rte_bbdev_log(ERR, "Hbstroe exceeds HARQ buffer size."); 865 break; 866 default: 867 rte_bbdev_log(ERR, "Descriptor error unknown error code %u", 868 error_code); 869 break; 870 } 871 return 1; 872 } 873 874 /* Compute value of k0. 875 * Based on 3GPP 38.212 Table 5.4.2.1-2 876 * Starting position of different redundancy versions, k0 877 */ 878 static inline uint16_t 879 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index) 880 { 881 if (rv_index == 0) 882 return 0; 883 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c; 884 if (n_cb == n) { 885 if (rv_index == 1) 886 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c; 887 else if (rv_index == 2) 888 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c; 889 else 890 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c; 891 } 892 /* LBRM case - includes a division by N */ 893 if (rv_index == 1) 894 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb) 895 / n) * z_c; 896 else if (rv_index == 2) 897 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb) 898 / n) * z_c; 899 else 900 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb) 901 / n) * z_c; 902 } 903 904 /** 905 * Set DMA descriptor for encode operation (1 Code Block) 906 * 907 * @param op 908 * Pointer to a single encode operation. 909 * @param desc 910 * Pointer to DMA descriptor. 911 * @param input 912 * Pointer to pointer to input data which will be decoded. 913 * @param e 914 * E value (length of output in bits). 915 * @param ncb 916 * Ncb value (size of the soft buffer). 917 * @param out_length 918 * Length of output buffer 919 * @param in_offset 920 * Input offset in rte_mbuf structure. It is used for calculating the point 921 * where data is starting. 922 * @param out_offset 923 * Output offset in rte_mbuf structure. It is used for calculating the point 924 * where hard output data will be stored. 925 * @param cbs_in_op 926 * Number of CBs contained in one operation. 927 */ 928 static inline int 929 fpga_dma_desc_te_fill(struct rte_bbdev_enc_op *op, 930 struct fpga_dma_enc_desc *desc, struct rte_mbuf *input, 931 struct rte_mbuf *output, uint16_t k_, uint16_t e, 932 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset, 933 uint8_t cbs_in_op) 934 { 935 /* reset */ 936 desc->done = 0; 937 desc->error = 0; 938 desc->k_ = k_; 939 desc->rm_e = e; 940 desc->desc_idx = desc_offset; 941 desc->zc = op->ldpc_enc.z_c; 942 desc->bg_idx = op->ldpc_enc.basegraph - 1; 943 desc->qm_idx = op->ldpc_enc.q_m / 2; 944 desc->crc_en = check_bit(op->ldpc_enc.op_flags, 945 RTE_BBDEV_LDPC_CRC_24B_ATTACH); 946 desc->irq_en = 0; 947 desc->k0 = get_k0(op->ldpc_enc.n_cb, op->ldpc_enc.z_c, 948 op->ldpc_enc.basegraph, op->ldpc_enc.rv_index); 949 desc->ncb = op->ldpc_enc.n_cb; 950 desc->num_null = op->ldpc_enc.n_filler; 951 /* Set inbound data buffer address */ 952 desc->in_addr_hi = (uint32_t)( 953 rte_pktmbuf_iova_offset(input, in_offset) >> 32); 954 desc->in_addr_lw = (uint32_t)( 955 rte_pktmbuf_iova_offset(input, in_offset)); 956 957 desc->out_addr_hi = (uint32_t)( 958 rte_pktmbuf_iova_offset(output, out_offset) >> 32); 959 desc->out_addr_lw = (uint32_t)( 960 rte_pktmbuf_iova_offset(output, out_offset)); 961 /* Save software context needed for dequeue */ 962 desc->op_addr = op; 963 /* Set total number of CBs in an op */ 964 desc->cbs_in_op = cbs_in_op; 965 return 0; 966 } 967 968 /** 969 * Set DMA descriptor for decode operation (1 Code Block) 970 * 971 * @param op 972 * Pointer to a single encode operation. 973 * @param desc 974 * Pointer to DMA descriptor. 975 * @param input 976 * Pointer to pointer to input data which will be decoded. 977 * @param in_offset 978 * Input offset in rte_mbuf structure. It is used for calculating the point 979 * where data is starting. 980 * @param out_offset 981 * Output offset in rte_mbuf structure. It is used for calculating the point 982 * where hard output data will be stored. 983 * @param cbs_in_op 984 * Number of CBs contained in one operation. 985 */ 986 static inline int 987 fpga_dma_desc_ld_fill(struct rte_bbdev_dec_op *op, 988 struct fpga_dma_dec_desc *desc, 989 struct rte_mbuf *input, struct rte_mbuf *output, 990 uint16_t harq_in_length, 991 uint32_t in_offset, uint32_t out_offset, 992 uint32_t harq_offset, 993 uint16_t desc_offset, 994 uint8_t cbs_in_op) 995 { 996 /* reset */ 997 desc->done = 0; 998 desc->error = 0; 999 /* Set inbound data buffer address */ 1000 desc->in_addr_hi = (uint32_t)( 1001 rte_pktmbuf_iova_offset(input, in_offset) >> 32); 1002 desc->in_addr_lw = (uint32_t)( 1003 rte_pktmbuf_iova_offset(input, in_offset)); 1004 desc->rm_e = op->ldpc_dec.cb_params.e; 1005 desc->harq_input_length = harq_in_length; 1006 desc->et_dis = !check_bit(op->ldpc_dec.op_flags, 1007 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE); 1008 desc->rv = op->ldpc_dec.rv_index; 1009 desc->crc24b_ind = check_bit(op->ldpc_dec.op_flags, 1010 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK); 1011 desc->drop_crc24b = check_bit(op->ldpc_dec.op_flags, 1012 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP); 1013 desc->desc_idx = desc_offset; 1014 desc->ncb = op->ldpc_dec.n_cb; 1015 desc->num_null = op->ldpc_dec.n_filler; 1016 desc->hbstroe_offset = harq_offset >> 10; 1017 desc->zc = op->ldpc_dec.z_c; 1018 desc->harqin_en = check_bit(op->ldpc_dec.op_flags, 1019 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE); 1020 desc->bg_idx = op->ldpc_dec.basegraph - 1; 1021 desc->max_iter = op->ldpc_dec.iter_max; 1022 desc->qm_idx = op->ldpc_dec.q_m / 2; 1023 desc->out_addr_hi = (uint32_t)( 1024 rte_pktmbuf_iova_offset(output, out_offset) >> 32); 1025 desc->out_addr_lw = (uint32_t)( 1026 rte_pktmbuf_iova_offset(output, out_offset)); 1027 /* Save software context needed for dequeue */ 1028 desc->op_addr = op; 1029 /* Set total number of CBs in an op */ 1030 desc->cbs_in_op = cbs_in_op; 1031 1032 return 0; 1033 } 1034 1035 /* Validates LDPC encoder parameters */ 1036 static inline int 1037 validate_ldpc_enc_op(struct rte_bbdev_enc_op *op) 1038 { 1039 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &op->ldpc_enc; 1040 1041 if (op->mempool == NULL) { 1042 rte_bbdev_log(ERR, "Invalid mempool pointer"); 1043 return -1; 1044 } 1045 if (ldpc_enc->input.data == NULL) { 1046 rte_bbdev_log(ERR, "Invalid input pointer"); 1047 return -1; 1048 } 1049 if (ldpc_enc->output.data == NULL) { 1050 rte_bbdev_log(ERR, "Invalid output pointer"); 1051 return -1; 1052 } 1053 if (ldpc_enc->input.length == 0) { 1054 rte_bbdev_log(ERR, "CB size (%u) is null", 1055 ldpc_enc->input.length); 1056 return -1; 1057 } 1058 if ((ldpc_enc->basegraph > 2) || (ldpc_enc->basegraph == 0)) { 1059 rte_bbdev_log(ERR, 1060 "BG (%u) is out of range 1 <= value <= 2", 1061 ldpc_enc->basegraph); 1062 return -1; 1063 } 1064 if (ldpc_enc->rv_index > 3) { 1065 rte_bbdev_log(ERR, 1066 "rv_index (%u) is out of range 0 <= value <= 3", 1067 ldpc_enc->rv_index); 1068 return -1; 1069 } 1070 if (ldpc_enc->code_block_mode > RTE_BBDEV_CODE_BLOCK) { 1071 rte_bbdev_log(ERR, 1072 "code_block_mode (%u) is out of range 0 <= value <= 1", 1073 ldpc_enc->code_block_mode); 1074 return -1; 1075 } 1076 1077 if (ldpc_enc->input.length > 1078 RTE_BBDEV_LDPC_MAX_CB_SIZE >> 3) { 1079 rte_bbdev_log(ERR, "CB size (%u) is too big, max: %d", 1080 ldpc_enc->input.length, 1081 RTE_BBDEV_LDPC_MAX_CB_SIZE); 1082 return -1; 1083 } 1084 int z_c = ldpc_enc->z_c; 1085 /* Check Zc is valid value */ 1086 if ((z_c > 384) || (z_c < 4)) { 1087 rte_bbdev_log(ERR, "Zc (%u) is out of range", z_c); 1088 return -1; 1089 } 1090 if (z_c > 256) { 1091 if ((z_c % 32) != 0) { 1092 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1093 return -1; 1094 } 1095 } else if (z_c > 128) { 1096 if ((z_c % 16) != 0) { 1097 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1098 return -1; 1099 } 1100 } else if (z_c > 64) { 1101 if ((z_c % 8) != 0) { 1102 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1103 return -1; 1104 } 1105 } else if (z_c > 32) { 1106 if ((z_c % 4) != 0) { 1107 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1108 return -1; 1109 } 1110 } else if (z_c > 16) { 1111 if ((z_c % 2) != 0) { 1112 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1113 return -1; 1114 } 1115 } 1116 1117 int n_filler = ldpc_enc->n_filler; 1118 int K = (ldpc_enc->basegraph == 1 ? 22 : 10) * ldpc_enc->z_c; 1119 int Kp = K - n_filler; 1120 int q_m = ldpc_enc->q_m; 1121 int n_cb = ldpc_enc->n_cb; 1122 int N = (ldpc_enc->basegraph == 1 ? N_ZC_1 : N_ZC_2) * z_c; 1123 int k0 = get_k0(n_cb, z_c, ldpc_enc->basegraph, 1124 ldpc_enc->rv_index); 1125 int crc24 = 0; 1126 int32_t L, Lcb, cw, cw_rm; 1127 int32_t e = ldpc_enc->cb_params.e; 1128 if (check_bit(op->ldpc_enc.op_flags, 1129 RTE_BBDEV_LDPC_CRC_24B_ATTACH)) 1130 crc24 = 24; 1131 1132 if (K < (int) (ldpc_enc->input.length * 8 + n_filler) + crc24) { 1133 rte_bbdev_log(ERR, "K and F not matching input size %u %u %u", 1134 K, n_filler, ldpc_enc->input.length); 1135 return -1; 1136 } 1137 if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1138 rte_bbdev_log(ERR, "TB mode not supported"); 1139 return -1; 1140 1141 } 1142 1143 /* K' range check */ 1144 if (Kp % 8 > 0) { 1145 rte_bbdev_log(ERR, "K' not byte aligned %u", Kp); 1146 return -1; 1147 } 1148 if ((crc24 > 0) && (Kp < 292)) { 1149 rte_bbdev_log(ERR, "Invalid CRC24 for small block %u", Kp); 1150 return -1; 1151 } 1152 if (Kp < 24) { 1153 rte_bbdev_log(ERR, "K' too small %u", Kp); 1154 return -1; 1155 } 1156 if (n_filler >= (K - 2 * z_c)) { 1157 rte_bbdev_log(ERR, "K - F invalid %u %u", K, n_filler); 1158 return -1; 1159 } 1160 /* Ncb range check */ 1161 if ((n_cb > N) || (n_cb < 32) || (n_cb <= (Kp - crc24))) { 1162 rte_bbdev_log(ERR, "Ncb (%u) is out of range K %d N %d", n_cb, K, N); 1163 return -1; 1164 } 1165 /* Qm range check */ 1166 if (!check_bit(op->ldpc_enc.op_flags, RTE_BBDEV_LDPC_INTERLEAVER_BYPASS) && 1167 ((q_m == 0) || ((q_m > 2) && ((q_m % 2) == 1)) || (q_m > 8))) { 1168 rte_bbdev_log(ERR, "Qm (%u) is out of range", q_m); 1169 return -1; 1170 } 1171 /* K0 range check */ 1172 if (((k0 % z_c) > 0) || (k0 >= n_cb) || ((k0 >= (Kp - 2 * z_c)) 1173 && (k0 < (K - 2 * z_c)))) { 1174 rte_bbdev_log(ERR, "K0 (%u) is out of range", k0); 1175 return -1; 1176 } 1177 /* E range check */ 1178 if (e <= RTE_MAX(32, z_c)) { 1179 rte_bbdev_log(ERR, "E is too small %"PRIu32"", e); 1180 return -1; 1181 } 1182 if ((e > 0xFFFF)) { 1183 rte_bbdev_log(ERR, "E is too large for N3000 %"PRIu32" > 64k", e); 1184 return -1; 1185 } 1186 if (q_m > 0) { 1187 if (e % q_m > 0) { 1188 rte_bbdev_log(ERR, "E %"PRIu32" not multiple of qm %d", e, q_m); 1189 return -1; 1190 } 1191 } 1192 /* Code word in RM range check */ 1193 if (k0 > (Kp - 2 * z_c)) 1194 L = k0 + e; 1195 else 1196 L = k0 + e + n_filler; 1197 Lcb = RTE_MIN(L, n_cb); 1198 if (ldpc_enc->basegraph == 1) { 1199 if (Lcb <= 25 * z_c) 1200 cw = 25 * z_c; 1201 else if (Lcb <= 27 * z_c) 1202 cw = 27 * z_c; 1203 else if (Lcb <= 30 * z_c) 1204 cw = 30 * z_c; 1205 else if (Lcb <= 33 * z_c) 1206 cw = 33 * z_c; 1207 else if (Lcb <= 44 * z_c) 1208 cw = 44 * z_c; 1209 else if (Lcb <= 55 * z_c) 1210 cw = 55 * z_c; 1211 else 1212 cw = 66 * z_c; 1213 } else { 1214 if (Lcb <= 15 * z_c) 1215 cw = 15 * z_c; 1216 else if (Lcb <= 20 * z_c) 1217 cw = 20 * z_c; 1218 else if (Lcb <= 25 * z_c) 1219 cw = 25 * z_c; 1220 else if (Lcb <= 30 * z_c) 1221 cw = 30 * z_c; 1222 else 1223 cw = 50 * z_c; 1224 } 1225 if (n_cb < Kp - 2 * z_c) 1226 cw_rm = n_cb; 1227 else if ((Kp - 2 * z_c <= n_cb) && (n_cb < K - 2 * z_c)) 1228 cw_rm = Kp - 2 * z_c; 1229 else if ((K - 2 * z_c <= n_cb) && (n_cb < cw)) 1230 cw_rm = n_cb - n_filler; 1231 else 1232 cw_rm = cw - n_filler; 1233 if (cw_rm <= 32) { 1234 rte_bbdev_log(ERR, 1235 "Invalid Ratematching"); 1236 return -1; 1237 } 1238 return 0; 1239 } 1240 1241 /* Validates LDPC decoder parameters */ 1242 static inline int 1243 validate_ldpc_dec_op(struct rte_bbdev_dec_op *op) 1244 { 1245 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &op->ldpc_dec; 1246 if (check_bit(ldpc_dec->op_flags, 1247 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) 1248 return 0; 1249 if (ldpc_dec->input.data == NULL) { 1250 rte_bbdev_log(ERR, "Invalid input pointer"); 1251 return -1; 1252 } 1253 if (ldpc_dec->hard_output.data == NULL) { 1254 rte_bbdev_log(ERR, "Invalid output pointer"); 1255 return -1; 1256 } 1257 if (ldpc_dec->input.length == 0) { 1258 rte_bbdev_log(ERR, "input is null"); 1259 return -1; 1260 } 1261 if ((ldpc_dec->basegraph > 2) || (ldpc_dec->basegraph == 0)) { 1262 rte_bbdev_log(ERR, 1263 "BG (%u) is out of range 1 <= value <= 2", 1264 ldpc_dec->basegraph); 1265 return -1; 1266 } 1267 if (ldpc_dec->iter_max == 0) { 1268 rte_bbdev_log(ERR, 1269 "iter_max (%u) is equal to 0", 1270 ldpc_dec->iter_max); 1271 return -1; 1272 } 1273 if (ldpc_dec->rv_index > 3) { 1274 rte_bbdev_log(ERR, 1275 "rv_index (%u) is out of range 0 <= value <= 3", 1276 ldpc_dec->rv_index); 1277 return -1; 1278 } 1279 if (ldpc_dec->code_block_mode > RTE_BBDEV_CODE_BLOCK) { 1280 rte_bbdev_log(ERR, 1281 "code_block_mode (%u) is out of range 0 <= value <= 1", 1282 ldpc_dec->code_block_mode); 1283 return -1; 1284 } 1285 if (check_bit(op->ldpc_dec.op_flags, 1286 RTE_BBDEV_LDPC_DECODE_BYPASS)) { 1287 rte_bbdev_log(ERR, "Avoid LDPC Decode bypass"); 1288 return -1; 1289 } 1290 int z_c = ldpc_dec->z_c; 1291 /* Check Zc is valid value */ 1292 if ((z_c > 384) || (z_c < 4)) { 1293 rte_bbdev_log(ERR, 1294 "Zc (%u) is out of range", 1295 z_c); 1296 return -1; 1297 } 1298 if (z_c > 256) { 1299 if ((z_c % 32) != 0) { 1300 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1301 return -1; 1302 } 1303 } else if (z_c > 128) { 1304 if ((z_c % 16) != 0) { 1305 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1306 return -1; 1307 } 1308 } else if (z_c > 64) { 1309 if ((z_c % 8) != 0) { 1310 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1311 return -1; 1312 } 1313 } else if (z_c > 32) { 1314 if ((z_c % 4) != 0) { 1315 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1316 return -1; 1317 } 1318 } else if (z_c > 16) { 1319 if ((z_c % 2) != 0) { 1320 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1321 return -1; 1322 } 1323 } 1324 1325 int n_filler = ldpc_dec->n_filler; 1326 int K = (ldpc_dec->basegraph == 1 ? 22 : 10) * ldpc_dec->z_c; 1327 int Kp = K - n_filler; 1328 int q_m = ldpc_dec->q_m; 1329 int n_cb = ldpc_dec->n_cb; 1330 int N = (ldpc_dec->basegraph == 1 ? N_ZC_1 : N_ZC_2) * z_c; 1331 int k0 = get_k0(n_cb, z_c, ldpc_dec->basegraph, 1332 ldpc_dec->rv_index); 1333 int crc24 = 0; 1334 int32_t L, Lcb, cw, cw_rm; 1335 int32_t e = ldpc_dec->cb_params.e; 1336 if (check_bit(op->ldpc_dec.op_flags, 1337 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK)) 1338 crc24 = 24; 1339 1340 if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1341 rte_bbdev_log(ERR, 1342 "TB mode not supported"); 1343 return -1; 1344 } 1345 /* Enforce HARQ input length */ 1346 ldpc_dec->harq_combined_input.length = RTE_MIN((uint32_t) n_cb, 1347 ldpc_dec->harq_combined_input.length); 1348 if ((ldpc_dec->harq_combined_input.length == 0) && 1349 check_bit(ldpc_dec->op_flags, 1350 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) { 1351 rte_bbdev_log(ERR, 1352 "HARQ input length (%u) should not be null", 1353 ldpc_dec->harq_combined_input.length); 1354 return -1; 1355 } 1356 if ((ldpc_dec->harq_combined_input.length > 0) && 1357 !check_bit(ldpc_dec->op_flags, 1358 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) { 1359 ldpc_dec->harq_combined_input.length = 0; 1360 } 1361 1362 /* K' range check */ 1363 if (Kp % 8 > 0) { 1364 rte_bbdev_log(ERR, 1365 "K' not byte aligned %u", 1366 Kp); 1367 return -1; 1368 } 1369 if ((crc24 > 0) && (Kp < 292)) { 1370 rte_bbdev_log(ERR, 1371 "Invalid CRC24 for small block %u", 1372 Kp); 1373 return -1; 1374 } 1375 if (Kp < 24) { 1376 rte_bbdev_log(ERR, 1377 "K' too small %u", 1378 Kp); 1379 return -1; 1380 } 1381 if (n_filler >= (K - 2 * z_c)) { 1382 rte_bbdev_log(ERR, 1383 "K - F invalid %u %u", 1384 K, n_filler); 1385 return -1; 1386 } 1387 /* Ncb range check */ 1388 if (n_cb != N) { 1389 rte_bbdev_log(ERR, 1390 "Ncb (%u) is out of range K %d N %d", 1391 n_cb, K, N); 1392 return -1; 1393 } 1394 /* Qm range check */ 1395 if (!check_bit(op->ldpc_dec.op_flags, 1396 RTE_BBDEV_LDPC_INTERLEAVER_BYPASS) && 1397 ((q_m == 0) || ((q_m > 2) && ((q_m % 2) == 1)) 1398 || (q_m > 8))) { 1399 rte_bbdev_log(ERR, 1400 "Qm (%u) is out of range", 1401 q_m); 1402 return -1; 1403 } 1404 /* K0 range check */ 1405 if (((k0 % z_c) > 0) || (k0 >= n_cb) || ((k0 >= (Kp - 2 * z_c)) 1406 && (k0 < (K - 2 * z_c)))) { 1407 rte_bbdev_log(ERR, 1408 "K0 (%u) is out of range", 1409 k0); 1410 return -1; 1411 } 1412 /* E range check */ 1413 if (e <= RTE_MAX(32, z_c)) { 1414 rte_bbdev_log(ERR, 1415 "E is too small"); 1416 return -1; 1417 } 1418 if ((e > 0xFFFF)) { 1419 rte_bbdev_log(ERR, 1420 "E is too large"); 1421 return -1; 1422 } 1423 if (q_m > 0) { 1424 if (e % q_m > 0) { 1425 rte_bbdev_log(ERR, 1426 "E not multiple of qm %d", q_m); 1427 return -1; 1428 } 1429 } 1430 /* Code word in RM range check */ 1431 if (k0 > (Kp - 2 * z_c)) 1432 L = k0 + e; 1433 else 1434 L = k0 + e + n_filler; 1435 Lcb = RTE_MIN(n_cb, RTE_MAX(L, 1436 (int32_t) ldpc_dec->harq_combined_input.length)); 1437 if (ldpc_dec->basegraph == 1) { 1438 if (Lcb <= 25 * z_c) 1439 cw = 25 * z_c; 1440 else if (Lcb <= 27 * z_c) 1441 cw = 27 * z_c; 1442 else if (Lcb <= 30 * z_c) 1443 cw = 30 * z_c; 1444 else if (Lcb <= 33 * z_c) 1445 cw = 33 * z_c; 1446 else if (Lcb <= 44 * z_c) 1447 cw = 44 * z_c; 1448 else if (Lcb <= 55 * z_c) 1449 cw = 55 * z_c; 1450 else 1451 cw = 66 * z_c; 1452 } else { 1453 if (Lcb <= 15 * z_c) 1454 cw = 15 * z_c; 1455 else if (Lcb <= 20 * z_c) 1456 cw = 20 * z_c; 1457 else if (Lcb <= 25 * z_c) 1458 cw = 25 * z_c; 1459 else if (Lcb <= 30 * z_c) 1460 cw = 30 * z_c; 1461 else 1462 cw = 50 * z_c; 1463 } 1464 cw_rm = cw - n_filler; 1465 if (cw_rm <= 32) { 1466 rte_bbdev_log(ERR, 1467 "Invalid Ratematching"); 1468 return -1; 1469 } 1470 return 0; 1471 } 1472 1473 static inline char * 1474 mbuf_append(struct rte_mbuf *m_head, struct rte_mbuf *m, uint16_t len) 1475 { 1476 if (unlikely(len > rte_pktmbuf_tailroom(m))) 1477 return NULL; 1478 1479 char *tail = (char *)m->buf_addr + m->data_off + m->data_len; 1480 m->data_len = (uint16_t)(m->data_len + len); 1481 m_head->pkt_len = (m_head->pkt_len + len); 1482 return tail; 1483 } 1484 1485 static inline void 1486 fpga_mutex_acquisition(struct fpga_queue *q) 1487 { 1488 uint32_t mutex_ctrl, mutex_read, cnt = 0; 1489 /* Assign a unique id for the duration of the DDR access */ 1490 q->ddr_mutex_uuid = rand(); 1491 /* Request and wait for acquisition of the mutex */ 1492 mutex_ctrl = (q->ddr_mutex_uuid << 16) + 1; 1493 do { 1494 if (cnt > 0) 1495 usleep(FPGA_TIMEOUT_CHECK_INTERVAL); 1496 rte_bbdev_log_debug("Acquiring Mutex for %x\n", 1497 q->ddr_mutex_uuid); 1498 fpga_reg_write_32(q->d->mmio_base, 1499 FPGA_5GNR_FEC_MUTEX, 1500 mutex_ctrl); 1501 mutex_read = fpga_reg_read_32(q->d->mmio_base, 1502 FPGA_5GNR_FEC_MUTEX); 1503 rte_bbdev_log_debug("Mutex %x cnt %d owner %x\n", 1504 mutex_read, cnt, q->ddr_mutex_uuid); 1505 cnt++; 1506 } while ((mutex_read >> 16) != q->ddr_mutex_uuid); 1507 } 1508 1509 static inline void 1510 fpga_mutex_free(struct fpga_queue *q) 1511 { 1512 uint32_t mutex_ctrl = q->ddr_mutex_uuid << 16; 1513 fpga_reg_write_32(q->d->mmio_base, 1514 FPGA_5GNR_FEC_MUTEX, 1515 mutex_ctrl); 1516 } 1517 1518 static inline int 1519 fpga_harq_write_loopback(struct fpga_queue *q, 1520 struct rte_mbuf *harq_input, uint16_t harq_in_length, 1521 uint32_t harq_in_offset, uint32_t harq_out_offset) 1522 { 1523 fpga_mutex_acquisition(q); 1524 uint32_t out_offset = harq_out_offset; 1525 uint32_t in_offset = harq_in_offset; 1526 uint32_t left_length = harq_in_length; 1527 uint32_t reg_32, increment = 0; 1528 uint64_t *input = NULL; 1529 uint32_t last_transaction = left_length 1530 % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; 1531 uint64_t last_word; 1532 1533 if (last_transaction > 0) 1534 left_length -= last_transaction; 1535 1536 /* 1537 * Get HARQ buffer size for each VF/PF: When 0x00, there is no 1538 * available DDR space for the corresponding VF/PF. 1539 */ 1540 reg_32 = fpga_reg_read_32(q->d->mmio_base, 1541 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS); 1542 if (reg_32 < harq_in_length) { 1543 left_length = reg_32; 1544 rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n"); 1545 } 1546 1547 input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_input, 1548 uint8_t *, in_offset); 1549 1550 while (left_length > 0) { 1551 if (fpga_reg_read_8(q->d->mmio_base, 1552 FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) == 1) { 1553 fpga_reg_write_32(q->d->mmio_base, 1554 FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS, 1555 out_offset); 1556 fpga_reg_write_64(q->d->mmio_base, 1557 FPGA_5GNR_FEC_DDR4_WR_DATA_REGS, 1558 input[increment]); 1559 left_length -= FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; 1560 out_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; 1561 increment++; 1562 fpga_reg_write_8(q->d->mmio_base, 1563 FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1); 1564 } 1565 } 1566 while (last_transaction > 0) { 1567 if (fpga_reg_read_8(q->d->mmio_base, 1568 FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) == 1) { 1569 fpga_reg_write_32(q->d->mmio_base, 1570 FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS, 1571 out_offset); 1572 last_word = input[increment]; 1573 last_word &= (uint64_t)(1 << (last_transaction * 4)) 1574 - 1; 1575 fpga_reg_write_64(q->d->mmio_base, 1576 FPGA_5GNR_FEC_DDR4_WR_DATA_REGS, 1577 last_word); 1578 fpga_reg_write_8(q->d->mmio_base, 1579 FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1); 1580 last_transaction = 0; 1581 } 1582 } 1583 fpga_mutex_free(q); 1584 return 1; 1585 } 1586 1587 static inline int 1588 fpga_harq_read_loopback(struct fpga_queue *q, 1589 struct rte_mbuf *harq_output, uint16_t harq_in_length, 1590 uint32_t harq_in_offset, uint32_t harq_out_offset) 1591 { 1592 fpga_mutex_acquisition(q); 1593 uint32_t left_length, in_offset = harq_in_offset; 1594 uint64_t reg; 1595 uint32_t increment = 0; 1596 uint64_t *input = NULL; 1597 uint32_t last_transaction = harq_in_length 1598 % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; 1599 1600 if (last_transaction > 0) 1601 harq_in_length += (8 - last_transaction); 1602 1603 reg = fpga_reg_read_32(q->d->mmio_base, 1604 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS); 1605 if (reg < harq_in_length) { 1606 harq_in_length = reg; 1607 rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n"); 1608 } 1609 1610 if (!mbuf_append(harq_output, harq_output, harq_in_length)) { 1611 rte_bbdev_log(ERR, "HARQ output buffer warning %d %d\n", 1612 harq_output->buf_len - 1613 rte_pktmbuf_headroom(harq_output), 1614 harq_in_length); 1615 harq_in_length = harq_output->buf_len - 1616 rte_pktmbuf_headroom(harq_output); 1617 if (!mbuf_append(harq_output, harq_output, harq_in_length)) { 1618 rte_bbdev_log(ERR, "HARQ output buffer issue %d %d\n", 1619 harq_output->buf_len, harq_in_length); 1620 return -1; 1621 } 1622 } 1623 left_length = harq_in_length; 1624 1625 input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_output, 1626 uint8_t *, harq_out_offset); 1627 1628 while (left_length > 0) { 1629 fpga_reg_write_32(q->d->mmio_base, 1630 FPGA_5GNR_FEC_DDR4_RD_ADDR_REGS, in_offset); 1631 fpga_reg_write_8(q->d->mmio_base, 1632 FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 1); 1633 reg = fpga_reg_read_8(q->d->mmio_base, 1634 FPGA_5GNR_FEC_DDR4_RD_RDY_REGS); 1635 while (reg != 1) { 1636 reg = fpga_reg_read_8(q->d->mmio_base, 1637 FPGA_5GNR_FEC_DDR4_RD_RDY_REGS); 1638 if (reg == FPGA_DDR_OVERFLOW) { 1639 rte_bbdev_log(ERR, 1640 "Read address is overflow!\n"); 1641 return -1; 1642 } 1643 } 1644 input[increment] = fpga_reg_read_64(q->d->mmio_base, 1645 FPGA_5GNR_FEC_DDR4_RD_DATA_REGS); 1646 left_length -= FPGA_5GNR_FEC_DDR_RD_DATA_LEN_IN_BYTES; 1647 in_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; 1648 increment++; 1649 fpga_reg_write_8(q->d->mmio_base, 1650 FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 0); 1651 } 1652 fpga_mutex_free(q); 1653 return 1; 1654 } 1655 1656 static inline int 1657 enqueue_ldpc_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op *op, 1658 uint16_t desc_offset) 1659 { 1660 union fpga_dma_desc *desc; 1661 int ret; 1662 uint8_t c, crc24_bits = 0; 1663 struct rte_bbdev_op_ldpc_enc *enc = &op->ldpc_enc; 1664 uint16_t in_offset = enc->input.offset; 1665 uint16_t out_offset = enc->output.offset; 1666 struct rte_mbuf *m_in = enc->input.data; 1667 struct rte_mbuf *m_out = enc->output.data; 1668 struct rte_mbuf *m_out_head = enc->output.data; 1669 uint32_t in_length, out_length, e; 1670 uint16_t total_left = enc->input.length; 1671 uint16_t ring_offset; 1672 uint16_t K, k_; 1673 1674 1675 if (validate_ldpc_enc_op(op) == -1) { 1676 rte_bbdev_log(ERR, "LDPC encoder validation rejected"); 1677 return -EINVAL; 1678 } 1679 1680 /* Clear op status */ 1681 op->status = 0; 1682 1683 if (m_in == NULL || m_out == NULL) { 1684 rte_bbdev_log(ERR, "Invalid mbuf pointer"); 1685 op->status = 1 << RTE_BBDEV_DATA_ERROR; 1686 return -EINVAL; 1687 } 1688 1689 if (enc->op_flags & RTE_BBDEV_LDPC_CRC_24B_ATTACH) 1690 crc24_bits = 24; 1691 1692 if (enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1693 /* For Transport Block mode */ 1694 /* FIXME */ 1695 c = enc->tb_params.c; 1696 e = enc->tb_params.ea; 1697 } else { /* For Code Block mode */ 1698 c = 1; 1699 e = enc->cb_params.e; 1700 } 1701 1702 /* Update total_left */ 1703 K = (enc->basegraph == 1 ? 22 : 10) * enc->z_c; 1704 k_ = K - enc->n_filler; 1705 in_length = (k_ - crc24_bits) >> 3; 1706 out_length = (e + 7) >> 3; 1707 1708 total_left = rte_pktmbuf_data_len(m_in) - in_offset; 1709 1710 /* Update offsets */ 1711 if (total_left != in_length) { 1712 op->status |= 1 << RTE_BBDEV_DATA_ERROR; 1713 rte_bbdev_log(ERR, 1714 "Mismatch between mbuf length and included CBs sizes %d", 1715 total_left); 1716 } 1717 1718 mbuf_append(m_out_head, m_out, out_length); 1719 1720 /* Offset into the ring */ 1721 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1722 /* Setup DMA Descriptor */ 1723 desc = q->ring_addr + ring_offset; 1724 1725 ret = fpga_dma_desc_te_fill(op, &desc->enc_req, m_in, m_out, 1726 k_, e, in_offset, out_offset, ring_offset, c); 1727 if (unlikely(ret < 0)) 1728 return ret; 1729 1730 /* Update lengths */ 1731 total_left -= in_length; 1732 op->ldpc_enc.output.length += out_length; 1733 1734 if (total_left > 0) { 1735 rte_bbdev_log(ERR, 1736 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u", 1737 total_left, in_length); 1738 return -1; 1739 } 1740 1741 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1742 print_dma_enc_desc_debug_info(desc); 1743 #endif 1744 return 1; 1745 } 1746 1747 static inline int 1748 enqueue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op *op, 1749 uint16_t desc_offset) 1750 { 1751 union fpga_dma_desc *desc; 1752 int ret; 1753 uint16_t ring_offset; 1754 uint8_t c; 1755 uint16_t e, in_length, out_length, k0, l, seg_total_left, sys_cols; 1756 uint16_t K, parity_offset, harq_in_length = 0, harq_out_length = 0; 1757 uint16_t crc24_overlap = 0; 1758 struct rte_bbdev_op_ldpc_dec *dec = &op->ldpc_dec; 1759 struct rte_mbuf *m_in = dec->input.data; 1760 struct rte_mbuf *m_out = dec->hard_output.data; 1761 struct rte_mbuf *m_out_head = dec->hard_output.data; 1762 uint16_t in_offset = dec->input.offset; 1763 uint16_t out_offset = dec->hard_output.offset; 1764 uint32_t harq_offset = 0; 1765 1766 if (validate_ldpc_dec_op(op) == -1) { 1767 rte_bbdev_log(ERR, "LDPC decoder validation rejected"); 1768 return -EINVAL; 1769 } 1770 1771 /* Clear op status */ 1772 op->status = 0; 1773 1774 /* Setup DMA Descriptor */ 1775 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1776 desc = q->ring_addr + ring_offset; 1777 1778 if (check_bit(dec->op_flags, 1779 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 1780 struct rte_mbuf *harq_in = dec->harq_combined_input.data; 1781 struct rte_mbuf *harq_out = dec->harq_combined_output.data; 1782 harq_in_length = dec->harq_combined_input.length; 1783 uint32_t harq_in_offset = dec->harq_combined_input.offset; 1784 uint32_t harq_out_offset = dec->harq_combined_output.offset; 1785 1786 if (check_bit(dec->op_flags, 1787 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE 1788 )) { 1789 ret = fpga_harq_write_loopback(q, harq_in, 1790 harq_in_length, harq_in_offset, 1791 harq_out_offset); 1792 } else if (check_bit(dec->op_flags, 1793 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE 1794 )) { 1795 ret = fpga_harq_read_loopback(q, harq_out, 1796 harq_in_length, harq_in_offset, 1797 harq_out_offset); 1798 dec->harq_combined_output.length = harq_in_length; 1799 } else { 1800 rte_bbdev_log(ERR, "OP flag Err!"); 1801 ret = -1; 1802 } 1803 /* Set descriptor for dequeue */ 1804 desc->dec_req.done = 1; 1805 desc->dec_req.error = 0; 1806 desc->dec_req.op_addr = op; 1807 desc->dec_req.cbs_in_op = 1; 1808 /* Mark this dummy descriptor to be dropped by HW */ 1809 desc->dec_req.desc_idx = (ring_offset + 1) 1810 & q->sw_ring_wrap_mask; 1811 return ret; /* Error or number of CB */ 1812 } 1813 1814 if (m_in == NULL || m_out == NULL) { 1815 rte_bbdev_log(ERR, "Invalid mbuf pointer"); 1816 op->status = 1 << RTE_BBDEV_DATA_ERROR; 1817 return -1; 1818 } 1819 1820 c = 1; 1821 e = dec->cb_params.e; 1822 1823 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP)) 1824 crc24_overlap = 24; 1825 1826 sys_cols = (dec->basegraph == 1) ? 22 : 10; 1827 K = sys_cols * dec->z_c; 1828 parity_offset = K - 2 * dec->z_c; 1829 1830 out_length = ((K - crc24_overlap - dec->n_filler) >> 3); 1831 in_length = e; 1832 seg_total_left = dec->input.length; 1833 1834 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) { 1835 harq_in_length = RTE_MIN(dec->harq_combined_input.length, 1836 (uint32_t)dec->n_cb); 1837 } 1838 1839 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) { 1840 k0 = get_k0(dec->n_cb, dec->z_c, 1841 dec->basegraph, dec->rv_index); 1842 if (k0 > parity_offset) 1843 l = k0 + e; 1844 else 1845 l = k0 + e + dec->n_filler; 1846 harq_out_length = RTE_MIN(RTE_MAX(harq_in_length, l), 1847 dec->n_cb); 1848 dec->harq_combined_output.length = harq_out_length; 1849 } 1850 1851 mbuf_append(m_out_head, m_out, out_length); 1852 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) 1853 harq_offset = dec->harq_combined_input.offset; 1854 else if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) 1855 harq_offset = dec->harq_combined_output.offset; 1856 1857 if ((harq_offset & 0x3FF) > 0) { 1858 rte_bbdev_log(ERR, "Invalid HARQ offset %d", harq_offset); 1859 op->status = 1 << RTE_BBDEV_DATA_ERROR; 1860 return -1; 1861 } 1862 1863 ret = fpga_dma_desc_ld_fill(op, &desc->dec_req, m_in, m_out, 1864 harq_in_length, in_offset, out_offset, harq_offset, 1865 ring_offset, c); 1866 if (unlikely(ret < 0)) 1867 return ret; 1868 /* Update lengths */ 1869 seg_total_left -= in_length; 1870 op->ldpc_dec.hard_output.length += out_length; 1871 if (seg_total_left > 0) { 1872 rte_bbdev_log(ERR, 1873 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u", 1874 seg_total_left, in_length); 1875 return -1; 1876 } 1877 1878 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1879 print_dma_dec_desc_debug_info(desc); 1880 #endif 1881 1882 return 1; 1883 } 1884 1885 static uint16_t 1886 fpga_enqueue_ldpc_enc(struct rte_bbdev_queue_data *q_data, 1887 struct rte_bbdev_enc_op **ops, uint16_t num) 1888 { 1889 uint16_t i, total_enqueued_cbs = 0; 1890 int32_t avail; 1891 int enqueued_cbs; 1892 struct fpga_queue *q = q_data->queue_private; 1893 union fpga_dma_desc *desc; 1894 1895 /* Check if queue is not full */ 1896 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) == 1897 q->head_free_desc)) 1898 return 0; 1899 1900 /* Calculates available space */ 1901 avail = (q->head_free_desc > q->tail) ? 1902 q->head_free_desc - q->tail - 1 : 1903 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1; 1904 1905 for (i = 0; i < num; ++i) { 1906 1907 /* Check if there is available space for further 1908 * processing 1909 */ 1910 if (unlikely(avail - 1 < 0)) 1911 break; 1912 avail -= 1; 1913 enqueued_cbs = enqueue_ldpc_enc_one_op_cb(q, ops[i], 1914 total_enqueued_cbs); 1915 1916 if (enqueued_cbs < 0) 1917 break; 1918 1919 total_enqueued_cbs += enqueued_cbs; 1920 1921 rte_bbdev_log_debug("enqueuing enc ops [%d/%d] | head %d | tail %d", 1922 total_enqueued_cbs, num, 1923 q->head_free_desc, q->tail); 1924 } 1925 1926 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt 1927 * only when all previous CBs were already processed. 1928 */ 1929 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1) 1930 & q->sw_ring_wrap_mask); 1931 desc->enc_req.irq_en = q->irq_enable; 1932 1933 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats); 1934 1935 /* Update stats */ 1936 q_data->queue_stats.enqueued_count += i; 1937 q_data->queue_stats.enqueue_err_count += num - i; 1938 1939 return i; 1940 } 1941 1942 static uint16_t 1943 fpga_enqueue_ldpc_dec(struct rte_bbdev_queue_data *q_data, 1944 struct rte_bbdev_dec_op **ops, uint16_t num) 1945 { 1946 uint16_t i, total_enqueued_cbs = 0; 1947 int32_t avail; 1948 int enqueued_cbs; 1949 struct fpga_queue *q = q_data->queue_private; 1950 union fpga_dma_desc *desc; 1951 1952 /* Check if queue is not full */ 1953 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) == 1954 q->head_free_desc)) 1955 return 0; 1956 1957 /* Calculates available space */ 1958 avail = (q->head_free_desc > q->tail) ? 1959 q->head_free_desc - q->tail - 1 : 1960 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1; 1961 1962 for (i = 0; i < num; ++i) { 1963 1964 /* Check if there is available space for further 1965 * processing 1966 */ 1967 if (unlikely(avail - 1 < 0)) 1968 break; 1969 avail -= 1; 1970 enqueued_cbs = enqueue_ldpc_dec_one_op_cb(q, ops[i], 1971 total_enqueued_cbs); 1972 1973 if (enqueued_cbs < 0) 1974 break; 1975 1976 total_enqueued_cbs += enqueued_cbs; 1977 1978 rte_bbdev_log_debug("enqueuing dec ops [%d/%d] | head %d | tail %d", 1979 total_enqueued_cbs, num, 1980 q->head_free_desc, q->tail); 1981 } 1982 1983 /* Update stats */ 1984 q_data->queue_stats.enqueued_count += i; 1985 q_data->queue_stats.enqueue_err_count += num - i; 1986 1987 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt 1988 * only when all previous CBs were already processed. 1989 */ 1990 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1) 1991 & q->sw_ring_wrap_mask); 1992 desc->enc_req.irq_en = q->irq_enable; 1993 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats); 1994 return i; 1995 } 1996 1997 1998 static inline int 1999 dequeue_ldpc_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op **op, 2000 uint16_t desc_offset) 2001 { 2002 union fpga_dma_desc *desc; 2003 int desc_error; 2004 /* Set current desc */ 2005 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2006 & q->sw_ring_wrap_mask); 2007 2008 /*check if done */ 2009 if (desc->enc_req.done == 0) 2010 return -1; 2011 2012 /* make sure the response is read atomically */ 2013 rte_smp_rmb(); 2014 2015 rte_bbdev_log_debug("DMA response desc %p", desc); 2016 2017 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2018 print_dma_enc_desc_debug_info(desc); 2019 #endif 2020 2021 *op = desc->enc_req.op_addr; 2022 /* Check the descriptor error field, return 1 on error */ 2023 desc_error = check_desc_error(desc->enc_req.error); 2024 (*op)->status = desc_error << RTE_BBDEV_DATA_ERROR; 2025 2026 return 1; 2027 } 2028 2029 2030 static inline int 2031 dequeue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op **op, 2032 uint16_t desc_offset) 2033 { 2034 union fpga_dma_desc *desc; 2035 int desc_error; 2036 /* Set descriptor */ 2037 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2038 & q->sw_ring_wrap_mask); 2039 2040 /* Verify done bit is set */ 2041 if (desc->dec_req.done == 0) 2042 return -1; 2043 2044 /* make sure the response is read atomically */ 2045 rte_smp_rmb(); 2046 2047 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2048 print_dma_dec_desc_debug_info(desc); 2049 #endif 2050 2051 *op = desc->dec_req.op_addr; 2052 2053 if (check_bit((*op)->ldpc_dec.op_flags, 2054 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 2055 (*op)->status = 0; 2056 return 1; 2057 } 2058 2059 /* FPGA reports iterations based on round-up minus 1 */ 2060 (*op)->ldpc_dec.iter_count = desc->dec_req.iter + 1; 2061 /* CRC Check criteria */ 2062 if (desc->dec_req.crc24b_ind && !(desc->dec_req.crcb_pass)) 2063 (*op)->status = 1 << RTE_BBDEV_CRC_ERROR; 2064 /* et_pass = 0 when decoder fails */ 2065 (*op)->status |= !(desc->dec_req.et_pass) << RTE_BBDEV_SYNDROME_ERROR; 2066 /* Check the descriptor error field, return 1 on error */ 2067 desc_error = check_desc_error(desc->dec_req.error); 2068 (*op)->status |= desc_error << RTE_BBDEV_DATA_ERROR; 2069 return 1; 2070 } 2071 2072 static uint16_t 2073 fpga_dequeue_ldpc_enc(struct rte_bbdev_queue_data *q_data, 2074 struct rte_bbdev_enc_op **ops, uint16_t num) 2075 { 2076 struct fpga_queue *q = q_data->queue_private; 2077 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask; 2078 uint16_t i; 2079 uint16_t dequeued_cbs = 0; 2080 int ret; 2081 2082 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) { 2083 ret = dequeue_ldpc_enc_one_op_cb(q, &ops[i], dequeued_cbs); 2084 2085 if (ret < 0) 2086 break; 2087 2088 dequeued_cbs += ret; 2089 2090 rte_bbdev_log_debug("dequeuing enc ops [%d/%d] | head %d | tail %d", 2091 dequeued_cbs, num, q->head_free_desc, q->tail); 2092 } 2093 2094 /* Update head */ 2095 q->head_free_desc = (q->head_free_desc + dequeued_cbs) & 2096 q->sw_ring_wrap_mask; 2097 2098 /* Update stats */ 2099 q_data->queue_stats.dequeued_count += i; 2100 2101 return i; 2102 } 2103 2104 static uint16_t 2105 fpga_dequeue_ldpc_dec(struct rte_bbdev_queue_data *q_data, 2106 struct rte_bbdev_dec_op **ops, uint16_t num) 2107 { 2108 struct fpga_queue *q = q_data->queue_private; 2109 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask; 2110 uint16_t i; 2111 uint16_t dequeued_cbs = 0; 2112 int ret; 2113 2114 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) { 2115 ret = dequeue_ldpc_dec_one_op_cb(q, &ops[i], dequeued_cbs); 2116 2117 if (ret < 0) 2118 break; 2119 2120 dequeued_cbs += ret; 2121 2122 rte_bbdev_log_debug("dequeuing dec ops [%d/%d] | head %d | tail %d", 2123 dequeued_cbs, num, q->head_free_desc, q->tail); 2124 } 2125 2126 /* Update head */ 2127 q->head_free_desc = (q->head_free_desc + dequeued_cbs) & 2128 q->sw_ring_wrap_mask; 2129 2130 /* Update stats */ 2131 q_data->queue_stats.dequeued_count += i; 2132 2133 return i; 2134 } 2135 2136 2137 /* Initialization Function */ 2138 static void 2139 fpga_5gnr_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv) 2140 { 2141 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); 2142 2143 dev->dev_ops = &fpga_ops; 2144 dev->enqueue_ldpc_enc_ops = fpga_enqueue_ldpc_enc; 2145 dev->enqueue_ldpc_dec_ops = fpga_enqueue_ldpc_dec; 2146 dev->dequeue_ldpc_enc_ops = fpga_dequeue_ldpc_enc; 2147 dev->dequeue_ldpc_dec_ops = fpga_dequeue_ldpc_dec; 2148 2149 ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->pf_device = 2150 !strcmp(drv->driver.name, 2151 RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME)); 2152 ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->mmio_base = 2153 pci_dev->mem_resource[0].addr; 2154 2155 rte_bbdev_log_debug( 2156 "Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64, 2157 drv->driver.name, dev->data->name, 2158 (void *)pci_dev->mem_resource[0].addr, 2159 pci_dev->mem_resource[0].phys_addr); 2160 } 2161 2162 static int 2163 fpga_5gnr_fec_probe(struct rte_pci_driver *pci_drv, 2164 struct rte_pci_device *pci_dev) 2165 { 2166 struct rte_bbdev *bbdev = NULL; 2167 char dev_name[RTE_BBDEV_NAME_MAX_LEN]; 2168 2169 if (pci_dev == NULL) { 2170 rte_bbdev_log(ERR, "NULL PCI device"); 2171 return -EINVAL; 2172 } 2173 2174 rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name)); 2175 2176 /* Allocate memory to be used privately by drivers */ 2177 bbdev = rte_bbdev_allocate(pci_dev->device.name); 2178 if (bbdev == NULL) 2179 return -ENODEV; 2180 2181 /* allocate device private memory */ 2182 bbdev->data->dev_private = rte_zmalloc_socket(dev_name, 2183 sizeof(struct fpga_5gnr_fec_device), 2184 RTE_CACHE_LINE_SIZE, 2185 pci_dev->device.numa_node); 2186 2187 if (bbdev->data->dev_private == NULL) { 2188 rte_bbdev_log(CRIT, 2189 "Allocate of %zu bytes for device \"%s\" failed", 2190 sizeof(struct fpga_5gnr_fec_device), dev_name); 2191 rte_bbdev_release(bbdev); 2192 return -ENOMEM; 2193 } 2194 2195 /* Fill HW specific part of device structure */ 2196 bbdev->device = &pci_dev->device; 2197 bbdev->intr_handle = pci_dev->intr_handle; 2198 bbdev->data->socket_id = pci_dev->device.numa_node; 2199 2200 /* Invoke FEC FPGA device initialization function */ 2201 fpga_5gnr_fec_init(bbdev, pci_drv); 2202 2203 rte_bbdev_log_debug("bbdev id = %u [%s]", 2204 bbdev->data->dev_id, dev_name); 2205 2206 struct fpga_5gnr_fec_device *d = bbdev->data->dev_private; 2207 uint32_t version_id = fpga_reg_read_32(d->mmio_base, 2208 FPGA_5GNR_FEC_VERSION_ID); 2209 rte_bbdev_log(INFO, "FEC FPGA RTL v%u.%u", 2210 ((uint16_t)(version_id >> 16)), ((uint16_t)version_id)); 2211 2212 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2213 if (!strcmp(pci_drv->driver.name, 2214 RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME))) 2215 print_static_reg_debug_info(d->mmio_base); 2216 #endif 2217 return 0; 2218 } 2219 2220 static int 2221 fpga_5gnr_fec_remove(struct rte_pci_device *pci_dev) 2222 { 2223 struct rte_bbdev *bbdev; 2224 int ret; 2225 uint8_t dev_id; 2226 2227 if (pci_dev == NULL) 2228 return -EINVAL; 2229 2230 /* Find device */ 2231 bbdev = rte_bbdev_get_named_dev(pci_dev->device.name); 2232 if (bbdev == NULL) { 2233 rte_bbdev_log(CRIT, 2234 "Couldn't find HW dev \"%s\" to uninitialise it", 2235 pci_dev->device.name); 2236 return -ENODEV; 2237 } 2238 dev_id = bbdev->data->dev_id; 2239 2240 /* free device private memory before close */ 2241 rte_free(bbdev->data->dev_private); 2242 2243 /* Close device */ 2244 ret = rte_bbdev_close(dev_id); 2245 if (ret < 0) 2246 rte_bbdev_log(ERR, 2247 "Device %i failed to close during uninit: %i", 2248 dev_id, ret); 2249 2250 /* release bbdev from library */ 2251 ret = rte_bbdev_release(bbdev); 2252 if (ret) 2253 rte_bbdev_log(ERR, "Device %i failed to uninit: %i", dev_id, 2254 ret); 2255 2256 rte_bbdev_log_debug("Destroyed bbdev = %u", dev_id); 2257 2258 return 0; 2259 } 2260 2261 static inline void 2262 set_default_fpga_conf(struct rte_fpga_5gnr_fec_conf *def_conf) 2263 { 2264 /* clear default configuration before initialization */ 2265 memset(def_conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf)); 2266 /* Set pf mode to true */ 2267 def_conf->pf_mode_en = true; 2268 2269 /* Set ratio between UL and DL to 1:1 (unit of weight is 3 CBs) */ 2270 def_conf->ul_bandwidth = 3; 2271 def_conf->dl_bandwidth = 3; 2272 2273 /* Set Load Balance Factor to 64 */ 2274 def_conf->dl_load_balance = 64; 2275 def_conf->ul_load_balance = 64; 2276 } 2277 2278 /* Initial configuration of FPGA 5GNR FEC device */ 2279 int 2280 rte_fpga_5gnr_fec_configure(const char *dev_name, 2281 const struct rte_fpga_5gnr_fec_conf *conf) 2282 { 2283 uint32_t payload_32, address; 2284 uint16_t payload_16; 2285 uint8_t payload_8; 2286 uint16_t q_id, vf_id, total_q_id, total_ul_q_id, total_dl_q_id; 2287 struct rte_bbdev *bbdev = rte_bbdev_get_named_dev(dev_name); 2288 struct rte_fpga_5gnr_fec_conf def_conf; 2289 2290 if (bbdev == NULL) { 2291 rte_bbdev_log(ERR, 2292 "Invalid dev_name (%s), or device is not yet initialised", 2293 dev_name); 2294 return -ENODEV; 2295 } 2296 2297 struct fpga_5gnr_fec_device *d = bbdev->data->dev_private; 2298 2299 if (conf == NULL) { 2300 rte_bbdev_log(ERR, 2301 "FPGA Configuration was not provided. Default configuration will be loaded."); 2302 set_default_fpga_conf(&def_conf); 2303 conf = &def_conf; 2304 } 2305 2306 /* 2307 * Configure UL:DL ratio. 2308 * [7:0]: UL weight 2309 * [15:8]: DL weight 2310 */ 2311 payload_16 = (conf->dl_bandwidth << 8) | conf->ul_bandwidth; 2312 address = FPGA_5GNR_FEC_CONFIGURATION; 2313 fpga_reg_write_16(d->mmio_base, address, payload_16); 2314 2315 /* Clear all queues registers */ 2316 payload_32 = FPGA_INVALID_HW_QUEUE_ID; 2317 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 2318 address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP; 2319 fpga_reg_write_32(d->mmio_base, address, payload_32); 2320 } 2321 2322 /* 2323 * If PF mode is enabled allocate all queues for PF only. 2324 * 2325 * For VF mode each VF can have different number of UL and DL queues. 2326 * Total number of queues to configure cannot exceed FPGA 2327 * capabilities - 64 queues - 32 queues for UL and 32 queues for DL. 2328 * Queues mapping is done according to configuration: 2329 * 2330 * UL queues: 2331 * | Q_ID | VF_ID | 2332 * | 0 | 0 | 2333 * | ... | 0 | 2334 * | conf->vf_dl_queues_number[0] - 1 | 0 | 2335 * | conf->vf_dl_queues_number[0] | 1 | 2336 * | ... | 1 | 2337 * | conf->vf_dl_queues_number[1] - 1 | 1 | 2338 * | ... | ... | 2339 * | conf->vf_dl_queues_number[7] - 1 | 7 | 2340 * 2341 * DL queues: 2342 * | Q_ID | VF_ID | 2343 * | 32 | 0 | 2344 * | ... | 0 | 2345 * | conf->vf_ul_queues_number[0] - 1 | 0 | 2346 * | conf->vf_ul_queues_number[0] | 1 | 2347 * | ... | 1 | 2348 * | conf->vf_ul_queues_number[1] - 1 | 1 | 2349 * | ... | ... | 2350 * | conf->vf_ul_queues_number[7] - 1 | 7 | 2351 * 2352 * Example of configuration: 2353 * conf->vf_ul_queues_number[0] = 4; -> 4 UL queues for VF0 2354 * conf->vf_dl_queues_number[0] = 4; -> 4 DL queues for VF0 2355 * conf->vf_ul_queues_number[1] = 2; -> 2 UL queues for VF1 2356 * conf->vf_dl_queues_number[1] = 2; -> 2 DL queues for VF1 2357 * 2358 * UL: 2359 * | Q_ID | VF_ID | 2360 * | 0 | 0 | 2361 * | 1 | 0 | 2362 * | 2 | 0 | 2363 * | 3 | 0 | 2364 * | 4 | 1 | 2365 * | 5 | 1 | 2366 * 2367 * DL: 2368 * | Q_ID | VF_ID | 2369 * | 32 | 0 | 2370 * | 33 | 0 | 2371 * | 34 | 0 | 2372 * | 35 | 0 | 2373 * | 36 | 1 | 2374 * | 37 | 1 | 2375 */ 2376 if (conf->pf_mode_en) { 2377 payload_32 = 0x1; 2378 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 2379 address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP; 2380 fpga_reg_write_32(d->mmio_base, address, payload_32); 2381 } 2382 } else { 2383 /* Calculate total number of UL and DL queues to configure */ 2384 total_ul_q_id = total_dl_q_id = 0; 2385 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) { 2386 total_ul_q_id += conf->vf_ul_queues_number[vf_id]; 2387 total_dl_q_id += conf->vf_dl_queues_number[vf_id]; 2388 } 2389 total_q_id = total_dl_q_id + total_ul_q_id; 2390 /* 2391 * Check if total number of queues to configure does not exceed 2392 * FPGA capabilities (64 queues - 32 UL and 32 DL queues) 2393 */ 2394 if ((total_ul_q_id > FPGA_NUM_UL_QUEUES) || 2395 (total_dl_q_id > FPGA_NUM_DL_QUEUES) || 2396 (total_q_id > FPGA_TOTAL_NUM_QUEUES)) { 2397 rte_bbdev_log(ERR, 2398 "FPGA Configuration failed. Too many queues to configure: UL_Q %u, DL_Q %u, FPGA_Q %u", 2399 total_ul_q_id, total_dl_q_id, 2400 FPGA_TOTAL_NUM_QUEUES); 2401 return -EINVAL; 2402 } 2403 total_ul_q_id = 0; 2404 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) { 2405 for (q_id = 0; q_id < conf->vf_ul_queues_number[vf_id]; 2406 ++q_id, ++total_ul_q_id) { 2407 address = (total_ul_q_id << 2) + 2408 FPGA_5GNR_FEC_QUEUE_MAP; 2409 payload_32 = ((0x80 + vf_id) << 16) | 0x1; 2410 fpga_reg_write_32(d->mmio_base, address, 2411 payload_32); 2412 } 2413 } 2414 total_dl_q_id = 0; 2415 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) { 2416 for (q_id = 0; q_id < conf->vf_dl_queues_number[vf_id]; 2417 ++q_id, ++total_dl_q_id) { 2418 address = ((total_dl_q_id + FPGA_NUM_UL_QUEUES) 2419 << 2) + FPGA_5GNR_FEC_QUEUE_MAP; 2420 payload_32 = ((0x80 + vf_id) << 16) | 0x1; 2421 fpga_reg_write_32(d->mmio_base, address, 2422 payload_32); 2423 } 2424 } 2425 } 2426 2427 /* Setting Load Balance Factor */ 2428 payload_16 = (conf->dl_load_balance << 8) | (conf->ul_load_balance); 2429 address = FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR; 2430 fpga_reg_write_16(d->mmio_base, address, payload_16); 2431 2432 /* Setting length of ring descriptor entry */ 2433 payload_16 = FPGA_RING_DESC_ENTRY_LENGTH; 2434 address = FPGA_5GNR_FEC_RING_DESC_LEN; 2435 fpga_reg_write_16(d->mmio_base, address, payload_16); 2436 2437 /* Queue PF/VF mapping table is ready */ 2438 payload_8 = 0x1; 2439 address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE; 2440 fpga_reg_write_8(d->mmio_base, address, payload_8); 2441 2442 rte_bbdev_log_debug("PF FPGA 5GNR FEC configuration complete for %s", 2443 dev_name); 2444 2445 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2446 print_static_reg_debug_info(d->mmio_base); 2447 #endif 2448 return 0; 2449 } 2450 2451 /* FPGA 5GNR FEC PCI PF address map */ 2452 static struct rte_pci_id pci_id_fpga_5gnr_fec_pf_map[] = { 2453 { 2454 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID, 2455 FPGA_5GNR_FEC_PF_DEVICE_ID) 2456 }, 2457 {.device_id = 0}, 2458 }; 2459 2460 static struct rte_pci_driver fpga_5gnr_fec_pci_pf_driver = { 2461 .probe = fpga_5gnr_fec_probe, 2462 .remove = fpga_5gnr_fec_remove, 2463 .id_table = pci_id_fpga_5gnr_fec_pf_map, 2464 .drv_flags = RTE_PCI_DRV_NEED_MAPPING 2465 }; 2466 2467 /* FPGA 5GNR FEC PCI VF address map */ 2468 static struct rte_pci_id pci_id_fpga_5gnr_fec_vf_map[] = { 2469 { 2470 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID, 2471 FPGA_5GNR_FEC_VF_DEVICE_ID) 2472 }, 2473 {.device_id = 0}, 2474 }; 2475 2476 static struct rte_pci_driver fpga_5gnr_fec_pci_vf_driver = { 2477 .probe = fpga_5gnr_fec_probe, 2478 .remove = fpga_5gnr_fec_remove, 2479 .id_table = pci_id_fpga_5gnr_fec_vf_map, 2480 .drv_flags = RTE_PCI_DRV_NEED_MAPPING 2481 }; 2482 2483 2484 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_PF_DRIVER_NAME, fpga_5gnr_fec_pci_pf_driver); 2485 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_PF_DRIVER_NAME, 2486 pci_id_fpga_5gnr_fec_pf_map); 2487 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_VF_DRIVER_NAME, fpga_5gnr_fec_pci_vf_driver); 2488 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_VF_DRIVER_NAME, 2489 pci_id_fpga_5gnr_fec_vf_map); 2490