1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Intel Corporation 3 */ 4 5 #include <unistd.h> 6 7 #include <rte_common.h> 8 #include <rte_log.h> 9 #include <dev_driver.h> 10 #include <rte_malloc.h> 11 #include <rte_mempool.h> 12 #include <rte_errno.h> 13 #include <rte_pci.h> 14 #include <bus_pci_driver.h> 15 #include <rte_byteorder.h> 16 #include <rte_cycles.h> 17 18 #include <rte_bbdev.h> 19 #include <rte_bbdev_pmd.h> 20 21 #include "fpga_5gnr_fec.h" 22 #include "rte_pmd_fpga_5gnr_fec.h" 23 24 #ifdef RTE_LIBRTE_BBDEV_DEBUG 25 RTE_LOG_REGISTER_DEFAULT(fpga_5gnr_fec_logtype, DEBUG); 26 #else 27 RTE_LOG_REGISTER_DEFAULT(fpga_5gnr_fec_logtype, NOTICE); 28 #endif 29 30 #ifdef RTE_LIBRTE_BBDEV_DEBUG 31 32 /* Read Ring Control Register of FPGA 5GNR FEC device */ 33 static inline void 34 print_ring_reg_debug_info(void *mmio_base, uint32_t offset) 35 { 36 rte_bbdev_log_debug( 37 "FPGA MMIO base address @ %p | Ring Control Register @ offset = 0x%08" 38 PRIx32, mmio_base, offset); 39 rte_bbdev_log_debug( 40 "RING_BASE_ADDR = 0x%016"PRIx64, 41 fpga_reg_read_64(mmio_base, offset)); 42 rte_bbdev_log_debug( 43 "RING_HEAD_ADDR = 0x%016"PRIx64, 44 fpga_reg_read_64(mmio_base, offset + 45 FPGA_5GNR_FEC_RING_HEAD_ADDR)); 46 rte_bbdev_log_debug( 47 "RING_SIZE = 0x%04"PRIx16, 48 fpga_reg_read_16(mmio_base, offset + 49 FPGA_5GNR_FEC_RING_SIZE)); 50 rte_bbdev_log_debug( 51 "RING_MISC = 0x%02"PRIx8, 52 fpga_reg_read_8(mmio_base, offset + 53 FPGA_5GNR_FEC_RING_MISC)); 54 rte_bbdev_log_debug( 55 "RING_ENABLE = 0x%02"PRIx8, 56 fpga_reg_read_8(mmio_base, offset + 57 FPGA_5GNR_FEC_RING_ENABLE)); 58 rte_bbdev_log_debug( 59 "RING_FLUSH_QUEUE_EN = 0x%02"PRIx8, 60 fpga_reg_read_8(mmio_base, offset + 61 FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN)); 62 rte_bbdev_log_debug( 63 "RING_SHADOW_TAIL = 0x%04"PRIx16, 64 fpga_reg_read_16(mmio_base, offset + 65 FPGA_5GNR_FEC_RING_SHADOW_TAIL)); 66 rte_bbdev_log_debug( 67 "RING_HEAD_POINT = 0x%04"PRIx16, 68 fpga_reg_read_16(mmio_base, offset + 69 FPGA_5GNR_FEC_RING_HEAD_POINT)); 70 } 71 72 /* Read Static Register of FPGA 5GNR FEC device */ 73 static inline void 74 print_static_reg_debug_info(void *mmio_base) 75 { 76 uint16_t config = fpga_reg_read_16(mmio_base, 77 FPGA_5GNR_FEC_CONFIGURATION); 78 uint8_t qmap_done = fpga_reg_read_8(mmio_base, 79 FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE); 80 uint16_t lb_factor = fpga_reg_read_16(mmio_base, 81 FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR); 82 uint16_t ring_desc_len = fpga_reg_read_16(mmio_base, 83 FPGA_5GNR_FEC_RING_DESC_LEN); 84 85 rte_bbdev_log_debug("UL.DL Weights = %u.%u", 86 ((uint8_t)config), ((uint8_t)(config >> 8))); 87 rte_bbdev_log_debug("UL.DL Load Balance = %u.%u", 88 ((uint8_t)lb_factor), ((uint8_t)(lb_factor >> 8))); 89 rte_bbdev_log_debug("Queue-PF/VF Mapping Table = %s", 90 (qmap_done > 0) ? "READY" : "NOT-READY"); 91 rte_bbdev_log_debug("Ring Descriptor Size = %u bytes", 92 ring_desc_len*FPGA_RING_DESC_LEN_UNIT_BYTES); 93 } 94 95 /* Print decode DMA Descriptor of FPGA 5GNR Decoder device */ 96 static void 97 print_dma_dec_desc_debug_info(union fpga_dma_desc *desc) 98 { 99 rte_bbdev_log_debug("DMA response desc %p\n" 100 "\t-- done(%"PRIu32") | iter(%"PRIu32") | et_pass(%"PRIu32")" 101 " | crcb_pass (%"PRIu32") | error(%"PRIu32")\n" 102 "\t-- qm_idx(%"PRIu32") | max_iter(%"PRIu32") | " 103 "bg_idx (%"PRIu32") | harqin_en(%"PRIu32") | zc(%"PRIu32")\n" 104 "\t-- hbstroe_offset(%"PRIu32") | num_null (%"PRIu32") " 105 "| irq_en(%"PRIu32")\n" 106 "\t-- ncb(%"PRIu32") | desc_idx (%"PRIu32") | " 107 "drop_crc24b(%"PRIu32") | RV (%"PRIu32")\n" 108 "\t-- crc24b_ind(%"PRIu32") | et_dis (%"PRIu32")\n" 109 "\t-- harq_input_length(%"PRIu32") | rm_e(%"PRIu32")\n" 110 "\t-- cbs_in_op(%"PRIu32") | in_add (0x%08"PRIx32"%08"PRIx32")" 111 "| out_add (0x%08"PRIx32"%08"PRIx32")", 112 desc, 113 (uint32_t)desc->dec_req.done, 114 (uint32_t)desc->dec_req.iter, 115 (uint32_t)desc->dec_req.et_pass, 116 (uint32_t)desc->dec_req.crcb_pass, 117 (uint32_t)desc->dec_req.error, 118 (uint32_t)desc->dec_req.qm_idx, 119 (uint32_t)desc->dec_req.max_iter, 120 (uint32_t)desc->dec_req.bg_idx, 121 (uint32_t)desc->dec_req.harqin_en, 122 (uint32_t)desc->dec_req.zc, 123 (uint32_t)desc->dec_req.hbstroe_offset, 124 (uint32_t)desc->dec_req.num_null, 125 (uint32_t)desc->dec_req.irq_en, 126 (uint32_t)desc->dec_req.ncb, 127 (uint32_t)desc->dec_req.desc_idx, 128 (uint32_t)desc->dec_req.drop_crc24b, 129 (uint32_t)desc->dec_req.rv, 130 (uint32_t)desc->dec_req.crc24b_ind, 131 (uint32_t)desc->dec_req.et_dis, 132 (uint32_t)desc->dec_req.harq_input_length, 133 (uint32_t)desc->dec_req.rm_e, 134 (uint32_t)desc->dec_req.cbs_in_op, 135 (uint32_t)desc->dec_req.in_addr_hi, 136 (uint32_t)desc->dec_req.in_addr_lw, 137 (uint32_t)desc->dec_req.out_addr_hi, 138 (uint32_t)desc->dec_req.out_addr_lw); 139 uint32_t *word = (uint32_t *) desc; 140 rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n" 141 "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n", 142 word[0], word[1], word[2], word[3], 143 word[4], word[5], word[6], word[7]); 144 } 145 146 /* Print decode DMA Descriptor of FPGA 5GNR encoder device */ 147 static void 148 print_dma_enc_desc_debug_info(union fpga_dma_desc *desc) 149 { 150 rte_bbdev_log_debug("DMA response desc %p\n" 151 "%"PRIu32" %"PRIu32"\n" 152 "K' %"PRIu32" E %"PRIu32" desc %"PRIu32" Z %"PRIu32"\n" 153 "BG %"PRIu32" Qm %"PRIu32" CRC %"PRIu32" IRQ %"PRIu32"\n" 154 "k0 %"PRIu32" Ncb %"PRIu32" F %"PRIu32"\n", 155 desc, 156 (uint32_t)desc->enc_req.done, 157 (uint32_t)desc->enc_req.error, 158 159 (uint32_t)desc->enc_req.k_, 160 (uint32_t)desc->enc_req.rm_e, 161 (uint32_t)desc->enc_req.desc_idx, 162 (uint32_t)desc->enc_req.zc, 163 164 (uint32_t)desc->enc_req.bg_idx, 165 (uint32_t)desc->enc_req.qm_idx, 166 (uint32_t)desc->enc_req.crc_en, 167 (uint32_t)desc->enc_req.irq_en, 168 169 (uint32_t)desc->enc_req.k0, 170 (uint32_t)desc->enc_req.ncb, 171 (uint32_t)desc->enc_req.num_null); 172 uint32_t *word = (uint32_t *) desc; 173 rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n" 174 "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n", 175 word[0], word[1], word[2], word[3], 176 word[4], word[5], word[6], word[7]); 177 } 178 179 #endif 180 181 static int 182 fpga_setup_queues(struct rte_bbdev *dev, uint16_t num_queues, int socket_id) 183 { 184 /* Number of queues bound to a PF/VF */ 185 uint32_t hw_q_num = 0; 186 uint32_t ring_size, payload, address, q_id, offset; 187 rte_iova_t phys_addr; 188 struct fpga_ring_ctrl_reg ring_reg; 189 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private; 190 191 address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE; 192 if (!(fpga_reg_read_32(fpga_dev->mmio_base, address) & 0x1)) { 193 rte_bbdev_log(ERR, 194 "Queue-PF/VF mapping is not set! Was PF configured for device (%s) ?", 195 dev->data->name); 196 return -EPERM; 197 } 198 199 /* Clear queue registers structure */ 200 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg)); 201 202 /* Scan queue map. 203 * If a queue is valid and mapped to a calling PF/VF the read value is 204 * replaced with a queue ID and if it's not then 205 * FPGA_INVALID_HW_QUEUE_ID is returned. 206 */ 207 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 208 uint32_t hw_q_id = fpga_reg_read_32(fpga_dev->mmio_base, 209 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2)); 210 211 rte_bbdev_log_debug("%s: queue ID: %u, registry queue ID: %u", 212 dev->device->name, q_id, hw_q_id); 213 214 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) { 215 fpga_dev->q_bound_bit_map |= (1ULL << q_id); 216 /* Clear queue register of found queue */ 217 offset = FPGA_5GNR_FEC_RING_CTRL_REGS + 218 (sizeof(struct fpga_ring_ctrl_reg) * q_id); 219 fpga_ring_reg_write(fpga_dev->mmio_base, 220 offset, ring_reg); 221 ++hw_q_num; 222 } 223 } 224 if (hw_q_num == 0) { 225 rte_bbdev_log(ERR, 226 "No HW queues assigned to this device. Probably this is a VF configured for PF mode. Check device configuration!"); 227 return -ENODEV; 228 } 229 230 if (num_queues > hw_q_num) { 231 rte_bbdev_log(ERR, 232 "Not enough queues for device %s! Requested: %u, available: %u", 233 dev->device->name, num_queues, hw_q_num); 234 return -EINVAL; 235 } 236 237 ring_size = FPGA_RING_MAX_SIZE * sizeof(struct fpga_dma_dec_desc); 238 239 /* Enforce 32 byte alignment */ 240 RTE_BUILD_BUG_ON((RTE_CACHE_LINE_SIZE % 32) != 0); 241 242 /* Allocate memory for SW descriptor rings */ 243 fpga_dev->sw_rings = rte_zmalloc_socket(dev->device->driver->name, 244 num_queues * ring_size, RTE_CACHE_LINE_SIZE, 245 socket_id); 246 if (fpga_dev->sw_rings == NULL) { 247 rte_bbdev_log(ERR, 248 "Failed to allocate memory for %s:%u sw_rings", 249 dev->device->driver->name, dev->data->dev_id); 250 return -ENOMEM; 251 } 252 253 fpga_dev->sw_rings_phys = rte_malloc_virt2iova(fpga_dev->sw_rings); 254 fpga_dev->sw_ring_size = ring_size; 255 fpga_dev->sw_ring_max_depth = FPGA_RING_MAX_SIZE; 256 257 /* Allocate memory for ring flush status */ 258 fpga_dev->flush_queue_status = rte_zmalloc_socket(NULL, 259 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, socket_id); 260 if (fpga_dev->flush_queue_status == NULL) { 261 rte_bbdev_log(ERR, 262 "Failed to allocate memory for %s:%u flush_queue_status", 263 dev->device->driver->name, dev->data->dev_id); 264 return -ENOMEM; 265 } 266 267 /* Set the flush status address registers */ 268 phys_addr = rte_malloc_virt2iova(fpga_dev->flush_queue_status); 269 270 address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_LW; 271 payload = (uint32_t)(phys_addr); 272 fpga_reg_write_32(fpga_dev->mmio_base, address, payload); 273 274 address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_HI; 275 payload = (uint32_t)(phys_addr >> 32); 276 fpga_reg_write_32(fpga_dev->mmio_base, address, payload); 277 278 return 0; 279 } 280 281 static int 282 fpga_dev_close(struct rte_bbdev *dev) 283 { 284 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private; 285 286 rte_free(fpga_dev->sw_rings); 287 rte_free(fpga_dev->flush_queue_status); 288 289 return 0; 290 } 291 292 static void 293 fpga_dev_info_get(struct rte_bbdev *dev, 294 struct rte_bbdev_driver_info *dev_info) 295 { 296 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 297 uint32_t q_id = 0; 298 299 static const struct rte_bbdev_op_cap bbdev_capabilities[] = { 300 { 301 .type = RTE_BBDEV_OP_LDPC_ENC, 302 .cap.ldpc_enc = { 303 .capability_flags = 304 RTE_BBDEV_LDPC_RATE_MATCH | 305 RTE_BBDEV_LDPC_ENC_INTERRUPTS | 306 RTE_BBDEV_LDPC_CRC_24B_ATTACH, 307 .num_buffers_src = 308 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS, 309 .num_buffers_dst = 310 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS, 311 } 312 }, 313 { 314 .type = RTE_BBDEV_OP_LDPC_DEC, 315 .cap.ldpc_dec = { 316 .capability_flags = 317 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK | 318 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP | 319 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE | 320 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE | 321 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE | 322 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE | 323 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE | 324 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK | 325 RTE_BBDEV_LDPC_DEC_INTERRUPTS | 326 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS, 327 .llr_size = 6, 328 .llr_decimals = 2, 329 .num_buffers_src = 330 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS, 331 .num_buffers_hard_out = 332 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS, 333 .num_buffers_soft_out = 0, 334 } 335 }, 336 RTE_BBDEV_END_OF_CAPABILITIES_LIST() 337 }; 338 339 /* Check the HARQ DDR size available */ 340 uint8_t timeout_counter = 0; 341 uint32_t harq_buf_ready = fpga_reg_read_32(d->mmio_base, 342 FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS); 343 while (harq_buf_ready != 1) { 344 usleep(FPGA_TIMEOUT_CHECK_INTERVAL); 345 timeout_counter++; 346 harq_buf_ready = fpga_reg_read_32(d->mmio_base, 347 FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS); 348 if (timeout_counter > FPGA_HARQ_RDY_TIMEOUT) { 349 rte_bbdev_log(ERR, "HARQ Buffer not ready %d", 350 harq_buf_ready); 351 harq_buf_ready = 1; 352 } 353 } 354 uint32_t harq_buf_size = fpga_reg_read_32(d->mmio_base, 355 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS); 356 357 static struct rte_bbdev_queue_conf default_queue_conf; 358 default_queue_conf.socket = dev->data->socket_id; 359 default_queue_conf.queue_size = FPGA_RING_MAX_SIZE; 360 361 dev_info->driver_name = dev->device->driver->name; 362 dev_info->queue_size_lim = FPGA_RING_MAX_SIZE; 363 dev_info->hardware_accelerated = true; 364 dev_info->min_alignment = 64; 365 dev_info->harq_buffer_size = (harq_buf_size >> 10) + 1; 366 dev_info->default_queue_conf = default_queue_conf; 367 dev_info->capabilities = bbdev_capabilities; 368 dev_info->cpu_flag_reqs = NULL; 369 dev_info->data_endianness = RTE_LITTLE_ENDIAN; 370 dev_info->device_status = RTE_BBDEV_DEV_NOT_SUPPORTED; 371 372 /* Calculates number of queues assigned to device */ 373 dev_info->max_num_queues = 0; 374 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 375 uint32_t hw_q_id = fpga_reg_read_32(d->mmio_base, 376 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2)); 377 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) 378 dev_info->max_num_queues++; 379 } 380 /* Expose number of queue per operation type */ 381 dev_info->num_queues[RTE_BBDEV_OP_NONE] = 0; 382 dev_info->num_queues[RTE_BBDEV_OP_TURBO_DEC] = 0; 383 dev_info->num_queues[RTE_BBDEV_OP_TURBO_ENC] = 0; 384 dev_info->num_queues[RTE_BBDEV_OP_LDPC_DEC] = dev_info->max_num_queues / 2; 385 dev_info->num_queues[RTE_BBDEV_OP_LDPC_ENC] = dev_info->max_num_queues / 2; 386 dev_info->queue_priority[RTE_BBDEV_OP_LDPC_DEC] = 1; 387 dev_info->queue_priority[RTE_BBDEV_OP_LDPC_ENC] = 1; 388 } 389 390 /** 391 * Find index of queue bound to current PF/VF which is unassigned. Return -1 392 * when there is no available queue 393 */ 394 static inline int 395 fpga_find_free_queue_idx(struct rte_bbdev *dev, 396 const struct rte_bbdev_queue_conf *conf) 397 { 398 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 399 uint64_t q_idx; 400 uint8_t i = 0; 401 uint8_t range = FPGA_TOTAL_NUM_QUEUES >> 1; 402 403 if (conf->op_type == RTE_BBDEV_OP_LDPC_ENC) { 404 i = FPGA_NUM_DL_QUEUES; 405 range = FPGA_TOTAL_NUM_QUEUES; 406 } 407 408 for (; i < range; ++i) { 409 q_idx = 1ULL << i; 410 /* Check if index of queue is bound to current PF/VF */ 411 if (d->q_bound_bit_map & q_idx) 412 /* Check if found queue was not already assigned */ 413 if (!(d->q_assigned_bit_map & q_idx)) { 414 d->q_assigned_bit_map |= q_idx; 415 return i; 416 } 417 } 418 419 rte_bbdev_log(INFO, "Failed to find free queue on %s", dev->data->name); 420 421 return -1; 422 } 423 424 static int 425 fpga_queue_setup(struct rte_bbdev *dev, uint16_t queue_id, 426 const struct rte_bbdev_queue_conf *conf) 427 { 428 uint32_t address, ring_offset; 429 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 430 struct fpga_queue *q; 431 int8_t q_idx; 432 433 /* Check if there is a free queue to assign */ 434 q_idx = fpga_find_free_queue_idx(dev, conf); 435 if (q_idx == -1) 436 return -1; 437 438 /* Allocate the queue data structure. */ 439 q = rte_zmalloc_socket(dev->device->driver->name, sizeof(*q), 440 RTE_CACHE_LINE_SIZE, conf->socket); 441 if (q == NULL) { 442 /* Mark queue as un-assigned */ 443 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 444 rte_bbdev_log(ERR, "Failed to allocate queue memory"); 445 return -ENOMEM; 446 } 447 448 q->d = d; 449 q->q_idx = q_idx; 450 451 /* Set ring_base_addr */ 452 q->ring_addr = RTE_PTR_ADD(d->sw_rings, (d->sw_ring_size * queue_id)); 453 q->ring_ctrl_reg.ring_base_addr = d->sw_rings_phys + 454 (d->sw_ring_size * queue_id); 455 456 /* Allocate memory for Completion Head variable*/ 457 q->ring_head_addr = rte_zmalloc_socket(dev->device->driver->name, 458 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, conf->socket); 459 if (q->ring_head_addr == NULL) { 460 /* Mark queue as un-assigned */ 461 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 462 rte_free(q); 463 rte_bbdev_log(ERR, 464 "Failed to allocate memory for %s:%u completion_head", 465 dev->device->driver->name, dev->data->dev_id); 466 return -ENOMEM; 467 } 468 /* Set ring_head_addr */ 469 q->ring_ctrl_reg.ring_head_addr = 470 rte_malloc_virt2iova(q->ring_head_addr); 471 472 /* Clear shadow_completion_head */ 473 q->shadow_completion_head = 0; 474 475 /* Set ring_size */ 476 if (conf->queue_size > FPGA_RING_MAX_SIZE) { 477 /* Mark queue as un-assigned */ 478 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 479 rte_free(q->ring_head_addr); 480 rte_free(q); 481 rte_bbdev_log(ERR, 482 "Size of queue is too big %d (MAX: %d ) for %s:%u", 483 conf->queue_size, FPGA_RING_MAX_SIZE, 484 dev->device->driver->name, dev->data->dev_id); 485 return -EINVAL; 486 } 487 q->ring_ctrl_reg.ring_size = conf->queue_size; 488 489 /* Set Miscellaneous FPGA register*/ 490 /* Max iteration number for TTI mitigation - todo */ 491 q->ring_ctrl_reg.max_ul_dec = 0; 492 /* Enable max iteration number for TTI - todo */ 493 q->ring_ctrl_reg.max_ul_dec_en = 0; 494 495 /* Enable the ring */ 496 q->ring_ctrl_reg.enable = 1; 497 498 /* Set FPGA head_point and tail registers */ 499 q->ring_ctrl_reg.head_point = q->tail = 0; 500 501 /* Set FPGA shadow_tail register */ 502 q->ring_ctrl_reg.shadow_tail = q->tail; 503 504 /* Calculates the ring offset for found queue */ 505 ring_offset = FPGA_5GNR_FEC_RING_CTRL_REGS + 506 (sizeof(struct fpga_ring_ctrl_reg) * q_idx); 507 508 /* Set FPGA Ring Control Registers */ 509 fpga_ring_reg_write(d->mmio_base, ring_offset, q->ring_ctrl_reg); 510 511 /* Store MMIO register of shadow_tail */ 512 address = ring_offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL; 513 q->shadow_tail_addr = RTE_PTR_ADD(d->mmio_base, address); 514 515 q->head_free_desc = q->tail; 516 517 /* Set wrap mask */ 518 q->sw_ring_wrap_mask = conf->queue_size - 1; 519 520 rte_bbdev_log_debug("Setup dev%u q%u: queue_idx=%u", 521 dev->data->dev_id, queue_id, q->q_idx); 522 523 dev->data->queues[queue_id].queue_private = q; 524 525 rte_bbdev_log_debug("BBDEV queue[%d] set up for FPGA queue[%d]", 526 queue_id, q_idx); 527 528 #ifdef RTE_LIBRTE_BBDEV_DEBUG 529 /* Read FPGA Ring Control Registers after configuration*/ 530 print_ring_reg_debug_info(d->mmio_base, ring_offset); 531 #endif 532 return 0; 533 } 534 535 static int 536 fpga_queue_release(struct rte_bbdev *dev, uint16_t queue_id) 537 { 538 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 539 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 540 struct fpga_ring_ctrl_reg ring_reg; 541 uint32_t offset; 542 543 rte_bbdev_log_debug("FPGA Queue[%d] released", queue_id); 544 545 if (q != NULL) { 546 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg)); 547 offset = FPGA_5GNR_FEC_RING_CTRL_REGS + 548 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 549 /* Disable queue */ 550 fpga_reg_write_8(d->mmio_base, 551 offset + FPGA_5GNR_FEC_RING_ENABLE, 0x00); 552 /* Clear queue registers */ 553 fpga_ring_reg_write(d->mmio_base, offset, ring_reg); 554 555 /* Mark the Queue as un-assigned */ 556 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q->q_idx)); 557 rte_free(q->ring_head_addr); 558 rte_free(q); 559 dev->data->queues[queue_id].queue_private = NULL; 560 } 561 562 return 0; 563 } 564 565 /* Function starts a device queue. */ 566 static int 567 fpga_queue_start(struct rte_bbdev *dev, uint16_t queue_id) 568 { 569 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 570 #ifdef RTE_LIBRTE_BBDEV_DEBUG 571 if (d == NULL) { 572 rte_bbdev_log(ERR, "Invalid device pointer"); 573 return -1; 574 } 575 #endif 576 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 577 uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS + 578 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 579 uint8_t enable = 0x01; 580 uint16_t zero = 0x0000; 581 582 /* Clear queue head and tail variables */ 583 q->tail = q->head_free_desc = 0; 584 585 /* Clear FPGA head_point and tail registers */ 586 fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_HEAD_POINT, 587 zero); 588 fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL, 589 zero); 590 591 /* Enable queue */ 592 fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE, 593 enable); 594 595 rte_bbdev_log_debug("FPGA Queue[%d] started", queue_id); 596 return 0; 597 } 598 599 /* Function stops a device queue. */ 600 static int 601 fpga_queue_stop(struct rte_bbdev *dev, uint16_t queue_id) 602 { 603 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 604 #ifdef RTE_LIBRTE_BBDEV_DEBUG 605 if (d == NULL) { 606 rte_bbdev_log(ERR, "Invalid device pointer"); 607 return -1; 608 } 609 #endif 610 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 611 uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS + 612 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 613 uint8_t payload = 0x01; 614 uint8_t counter = 0; 615 uint8_t timeout = FPGA_QUEUE_FLUSH_TIMEOUT_US / 616 FPGA_TIMEOUT_CHECK_INTERVAL; 617 618 /* Set flush_queue_en bit to trigger queue flushing */ 619 fpga_reg_write_8(d->mmio_base, 620 offset + FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN, payload); 621 622 /** Check if queue flush is completed. 623 * FPGA will update the completion flag after queue flushing is 624 * completed. If completion flag is not updated within 1ms it is 625 * considered as a failure. 626 */ 627 while (!(*((volatile uint8_t *)d->flush_queue_status + q->q_idx) 628 & payload)) { 629 if (counter > timeout) { 630 rte_bbdev_log(ERR, "FPGA Queue Flush failed for queue %d", 631 queue_id); 632 return -1; 633 } 634 usleep(FPGA_TIMEOUT_CHECK_INTERVAL); 635 counter++; 636 } 637 638 /* Disable queue */ 639 payload = 0x00; 640 fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE, 641 payload); 642 643 rte_bbdev_log_debug("FPGA Queue[%d] stopped", queue_id); 644 return 0; 645 } 646 647 static inline uint16_t 648 get_queue_id(struct rte_bbdev_data *data, uint8_t q_idx) 649 { 650 uint16_t queue_id; 651 652 for (queue_id = 0; queue_id < data->num_queues; ++queue_id) { 653 struct fpga_queue *q = data->queues[queue_id].queue_private; 654 if (q != NULL && q->q_idx == q_idx) 655 return queue_id; 656 } 657 658 return -1; 659 } 660 661 /* Interrupt handler triggered by FPGA dev for handling specific interrupt */ 662 static void 663 fpga_dev_interrupt_handler(void *cb_arg) 664 { 665 struct rte_bbdev *dev = cb_arg; 666 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private; 667 struct fpga_queue *q; 668 uint64_t ring_head; 669 uint64_t q_idx; 670 uint16_t queue_id; 671 uint8_t i; 672 673 /* Scan queue assigned to this device */ 674 for (i = 0; i < FPGA_TOTAL_NUM_QUEUES; ++i) { 675 q_idx = 1ULL << i; 676 if (fpga_dev->q_bound_bit_map & q_idx) { 677 queue_id = get_queue_id(dev->data, i); 678 if (queue_id == (uint16_t) -1) 679 continue; 680 681 /* Check if completion head was changed */ 682 q = dev->data->queues[queue_id].queue_private; 683 ring_head = *q->ring_head_addr; 684 if (q->shadow_completion_head != ring_head && 685 q->irq_enable == 1) { 686 q->shadow_completion_head = ring_head; 687 rte_bbdev_pmd_callback_process( 688 dev, 689 RTE_BBDEV_EVENT_DEQUEUE, 690 &queue_id); 691 } 692 } 693 } 694 } 695 696 static int 697 fpga_queue_intr_enable(struct rte_bbdev *dev, uint16_t queue_id) 698 { 699 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 700 701 if (!rte_intr_cap_multiple(dev->intr_handle)) 702 return -ENOTSUP; 703 704 q->irq_enable = 1; 705 706 return 0; 707 } 708 709 static int 710 fpga_queue_intr_disable(struct rte_bbdev *dev, uint16_t queue_id) 711 { 712 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 713 q->irq_enable = 0; 714 715 return 0; 716 } 717 718 static int 719 fpga_intr_enable(struct rte_bbdev *dev) 720 { 721 int ret; 722 uint8_t i; 723 724 if (!rte_intr_cap_multiple(dev->intr_handle)) { 725 rte_bbdev_log(ERR, "Multiple intr vector is not supported by FPGA (%s)", 726 dev->data->name); 727 return -ENOTSUP; 728 } 729 730 /* Create event file descriptors for each of 64 queue. Event fds will be 731 * mapped to FPGA IRQs in rte_intr_enable(). This is a 1:1 mapping where 732 * the IRQ number is a direct translation to the queue number. 733 * 734 * 63 (FPGA_NUM_INTR_VEC) event fds are created as rte_intr_enable() 735 * mapped the first IRQ to already created interrupt event file 736 * descriptor (intr_handle->fd). 737 */ 738 if (rte_intr_efd_enable(dev->intr_handle, FPGA_NUM_INTR_VEC)) { 739 rte_bbdev_log(ERR, "Failed to create fds for %u queues", 740 dev->data->num_queues); 741 return -1; 742 } 743 744 /* TODO Each event file descriptor is overwritten by interrupt event 745 * file descriptor. That descriptor is added to epoll observed list. 746 * It ensures that callback function assigned to that descriptor will 747 * invoked when any FPGA queue issues interrupt. 748 */ 749 for (i = 0; i < FPGA_NUM_INTR_VEC; ++i) { 750 if (rte_intr_efds_index_set(dev->intr_handle, i, 751 rte_intr_fd_get(dev->intr_handle))) 752 return -rte_errno; 753 } 754 755 if (rte_intr_vec_list_alloc(dev->intr_handle, "intr_vec", 756 dev->data->num_queues)) { 757 rte_bbdev_log(ERR, "Failed to allocate %u vectors", 758 dev->data->num_queues); 759 return -ENOMEM; 760 } 761 762 ret = rte_intr_enable(dev->intr_handle); 763 if (ret < 0) { 764 rte_bbdev_log(ERR, 765 "Couldn't enable interrupts for device: %s", 766 dev->data->name); 767 return ret; 768 } 769 770 ret = rte_intr_callback_register(dev->intr_handle, 771 fpga_dev_interrupt_handler, dev); 772 if (ret < 0) { 773 rte_bbdev_log(ERR, 774 "Couldn't register interrupt callback for device: %s", 775 dev->data->name); 776 return ret; 777 } 778 779 return 0; 780 } 781 782 static const struct rte_bbdev_ops fpga_ops = { 783 .setup_queues = fpga_setup_queues, 784 .intr_enable = fpga_intr_enable, 785 .close = fpga_dev_close, 786 .info_get = fpga_dev_info_get, 787 .queue_setup = fpga_queue_setup, 788 .queue_stop = fpga_queue_stop, 789 .queue_start = fpga_queue_start, 790 .queue_release = fpga_queue_release, 791 .queue_intr_enable = fpga_queue_intr_enable, 792 .queue_intr_disable = fpga_queue_intr_disable 793 }; 794 795 static inline void 796 fpga_dma_enqueue(struct fpga_queue *q, uint16_t num_desc, 797 struct rte_bbdev_stats *queue_stats) 798 { 799 uint64_t start_time = 0; 800 queue_stats->acc_offload_cycles = 0; 801 802 /* Update tail and shadow_tail register */ 803 q->tail = (q->tail + num_desc) & q->sw_ring_wrap_mask; 804 805 rte_wmb(); 806 807 /* Start time measurement for enqueue function offload. */ 808 start_time = rte_rdtsc_precise(); 809 mmio_write_16(q->shadow_tail_addr, q->tail); 810 811 rte_wmb(); 812 queue_stats->acc_offload_cycles += rte_rdtsc_precise() - start_time; 813 } 814 815 /* Read flag value 0/1/ from bitmap */ 816 static inline bool 817 check_bit(uint32_t bitmap, uint32_t bitmask) 818 { 819 return bitmap & bitmask; 820 } 821 822 /* Print an error if a descriptor error has occurred. 823 * Return 0 on success, 1 on failure 824 */ 825 static inline int 826 check_desc_error(uint32_t error_code) { 827 switch (error_code) { 828 case DESC_ERR_NO_ERR: 829 return 0; 830 case DESC_ERR_K_P_OUT_OF_RANGE: 831 rte_bbdev_log(ERR, "Encode block size K' is out of range"); 832 break; 833 case DESC_ERR_Z_C_NOT_LEGAL: 834 rte_bbdev_log(ERR, "Zc is illegal"); 835 break; 836 case DESC_ERR_DESC_OFFSET_ERR: 837 rte_bbdev_log(ERR, 838 "Queue offset does not meet the expectation in the FPGA" 839 ); 840 break; 841 case DESC_ERR_DESC_READ_FAIL: 842 rte_bbdev_log(ERR, "Unsuccessful completion for descriptor read"); 843 break; 844 case DESC_ERR_DESC_READ_TIMEOUT: 845 rte_bbdev_log(ERR, "Descriptor read time-out"); 846 break; 847 case DESC_ERR_DESC_READ_TLP_POISONED: 848 rte_bbdev_log(ERR, "Descriptor read TLP poisoned"); 849 break; 850 case DESC_ERR_HARQ_INPUT_LEN: 851 rte_bbdev_log(ERR, "HARQ input length is invalid"); 852 break; 853 case DESC_ERR_CB_READ_FAIL: 854 rte_bbdev_log(ERR, "Unsuccessful completion for code block"); 855 break; 856 case DESC_ERR_CB_READ_TIMEOUT: 857 rte_bbdev_log(ERR, "Code block read time-out"); 858 break; 859 case DESC_ERR_CB_READ_TLP_POISONED: 860 rte_bbdev_log(ERR, "Code block read TLP poisoned"); 861 break; 862 case DESC_ERR_HBSTORE_ERR: 863 rte_bbdev_log(ERR, "Hbstroe exceeds HARQ buffer size."); 864 break; 865 default: 866 rte_bbdev_log(ERR, "Descriptor error unknown error code %u", 867 error_code); 868 break; 869 } 870 return 1; 871 } 872 873 /* Compute value of k0. 874 * Based on 3GPP 38.212 Table 5.4.2.1-2 875 * Starting position of different redundancy versions, k0 876 */ 877 static inline uint16_t 878 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index) 879 { 880 if (rv_index == 0) 881 return 0; 882 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c; 883 if (n_cb == n) { 884 if (rv_index == 1) 885 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c; 886 else if (rv_index == 2) 887 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c; 888 else 889 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c; 890 } 891 /* LBRM case - includes a division by N */ 892 if (rv_index == 1) 893 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb) 894 / n) * z_c; 895 else if (rv_index == 2) 896 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb) 897 / n) * z_c; 898 else 899 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb) 900 / n) * z_c; 901 } 902 903 /** 904 * Set DMA descriptor for encode operation (1 Code Block) 905 * 906 * @param op 907 * Pointer to a single encode operation. 908 * @param desc 909 * Pointer to DMA descriptor. 910 * @param input 911 * Pointer to pointer to input data which will be decoded. 912 * @param e 913 * E value (length of output in bits). 914 * @param ncb 915 * Ncb value (size of the soft buffer). 916 * @param out_length 917 * Length of output buffer 918 * @param in_offset 919 * Input offset in rte_mbuf structure. It is used for calculating the point 920 * where data is starting. 921 * @param out_offset 922 * Output offset in rte_mbuf structure. It is used for calculating the point 923 * where hard output data will be stored. 924 * @param cbs_in_op 925 * Number of CBs contained in one operation. 926 */ 927 static inline int 928 fpga_dma_desc_te_fill(struct rte_bbdev_enc_op *op, 929 struct fpga_dma_enc_desc *desc, struct rte_mbuf *input, 930 struct rte_mbuf *output, uint16_t k_, uint16_t e, 931 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset, 932 uint8_t cbs_in_op) 933 { 934 /* reset */ 935 desc->done = 0; 936 desc->error = 0; 937 desc->k_ = k_; 938 desc->rm_e = e; 939 desc->desc_idx = desc_offset; 940 desc->zc = op->ldpc_enc.z_c; 941 desc->bg_idx = op->ldpc_enc.basegraph - 1; 942 desc->qm_idx = op->ldpc_enc.q_m / 2; 943 desc->crc_en = check_bit(op->ldpc_enc.op_flags, 944 RTE_BBDEV_LDPC_CRC_24B_ATTACH); 945 desc->irq_en = 0; 946 desc->k0 = get_k0(op->ldpc_enc.n_cb, op->ldpc_enc.z_c, 947 op->ldpc_enc.basegraph, op->ldpc_enc.rv_index); 948 desc->ncb = op->ldpc_enc.n_cb; 949 desc->num_null = op->ldpc_enc.n_filler; 950 /* Set inbound data buffer address */ 951 desc->in_addr_hi = (uint32_t)( 952 rte_pktmbuf_iova_offset(input, in_offset) >> 32); 953 desc->in_addr_lw = (uint32_t)( 954 rte_pktmbuf_iova_offset(input, in_offset)); 955 956 desc->out_addr_hi = (uint32_t)( 957 rte_pktmbuf_iova_offset(output, out_offset) >> 32); 958 desc->out_addr_lw = (uint32_t)( 959 rte_pktmbuf_iova_offset(output, out_offset)); 960 /* Save software context needed for dequeue */ 961 desc->op_addr = op; 962 /* Set total number of CBs in an op */ 963 desc->cbs_in_op = cbs_in_op; 964 return 0; 965 } 966 967 /** 968 * Set DMA descriptor for decode operation (1 Code Block) 969 * 970 * @param op 971 * Pointer to a single encode operation. 972 * @param desc 973 * Pointer to DMA descriptor. 974 * @param input 975 * Pointer to pointer to input data which will be decoded. 976 * @param in_offset 977 * Input offset in rte_mbuf structure. It is used for calculating the point 978 * where data is starting. 979 * @param out_offset 980 * Output offset in rte_mbuf structure. It is used for calculating the point 981 * where hard output data will be stored. 982 * @param cbs_in_op 983 * Number of CBs contained in one operation. 984 */ 985 static inline int 986 fpga_dma_desc_ld_fill(struct rte_bbdev_dec_op *op, 987 struct fpga_dma_dec_desc *desc, 988 struct rte_mbuf *input, struct rte_mbuf *output, 989 uint16_t harq_in_length, 990 uint32_t in_offset, uint32_t out_offset, 991 uint32_t harq_offset, 992 uint16_t desc_offset, 993 uint8_t cbs_in_op) 994 { 995 /* reset */ 996 desc->done = 0; 997 desc->error = 0; 998 /* Set inbound data buffer address */ 999 desc->in_addr_hi = (uint32_t)( 1000 rte_pktmbuf_iova_offset(input, in_offset) >> 32); 1001 desc->in_addr_lw = (uint32_t)( 1002 rte_pktmbuf_iova_offset(input, in_offset)); 1003 desc->rm_e = op->ldpc_dec.cb_params.e; 1004 desc->harq_input_length = harq_in_length; 1005 desc->et_dis = !check_bit(op->ldpc_dec.op_flags, 1006 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE); 1007 desc->rv = op->ldpc_dec.rv_index; 1008 desc->crc24b_ind = check_bit(op->ldpc_dec.op_flags, 1009 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK); 1010 desc->drop_crc24b = check_bit(op->ldpc_dec.op_flags, 1011 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP); 1012 desc->desc_idx = desc_offset; 1013 desc->ncb = op->ldpc_dec.n_cb; 1014 desc->num_null = op->ldpc_dec.n_filler; 1015 desc->hbstroe_offset = harq_offset >> 10; 1016 desc->zc = op->ldpc_dec.z_c; 1017 desc->harqin_en = check_bit(op->ldpc_dec.op_flags, 1018 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE); 1019 desc->bg_idx = op->ldpc_dec.basegraph - 1; 1020 desc->max_iter = op->ldpc_dec.iter_max; 1021 desc->qm_idx = op->ldpc_dec.q_m / 2; 1022 desc->out_addr_hi = (uint32_t)( 1023 rte_pktmbuf_iova_offset(output, out_offset) >> 32); 1024 desc->out_addr_lw = (uint32_t)( 1025 rte_pktmbuf_iova_offset(output, out_offset)); 1026 /* Save software context needed for dequeue */ 1027 desc->op_addr = op; 1028 /* Set total number of CBs in an op */ 1029 desc->cbs_in_op = cbs_in_op; 1030 1031 return 0; 1032 } 1033 1034 /* Validates LDPC encoder parameters */ 1035 static inline int 1036 validate_ldpc_enc_op(struct rte_bbdev_enc_op *op) 1037 { 1038 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &op->ldpc_enc; 1039 1040 if (op->mempool == NULL) { 1041 rte_bbdev_log(ERR, "Invalid mempool pointer"); 1042 return -1; 1043 } 1044 if (ldpc_enc->input.data == NULL) { 1045 rte_bbdev_log(ERR, "Invalid input pointer"); 1046 return -1; 1047 } 1048 if (ldpc_enc->output.data == NULL) { 1049 rte_bbdev_log(ERR, "Invalid output pointer"); 1050 return -1; 1051 } 1052 if (ldpc_enc->input.length == 0) { 1053 rte_bbdev_log(ERR, "CB size (%u) is null", 1054 ldpc_enc->input.length); 1055 return -1; 1056 } 1057 if ((ldpc_enc->basegraph > 2) || (ldpc_enc->basegraph == 0)) { 1058 rte_bbdev_log(ERR, 1059 "BG (%u) is out of range 1 <= value <= 2", 1060 ldpc_enc->basegraph); 1061 return -1; 1062 } 1063 if (ldpc_enc->rv_index > 3) { 1064 rte_bbdev_log(ERR, 1065 "rv_index (%u) is out of range 0 <= value <= 3", 1066 ldpc_enc->rv_index); 1067 return -1; 1068 } 1069 if (ldpc_enc->code_block_mode > RTE_BBDEV_CODE_BLOCK) { 1070 rte_bbdev_log(ERR, 1071 "code_block_mode (%u) is out of range 0 <= value <= 1", 1072 ldpc_enc->code_block_mode); 1073 return -1; 1074 } 1075 1076 if (ldpc_enc->input.length > 1077 RTE_BBDEV_LDPC_MAX_CB_SIZE >> 3) { 1078 rte_bbdev_log(ERR, "CB size (%u) is too big, max: %d", 1079 ldpc_enc->input.length, 1080 RTE_BBDEV_LDPC_MAX_CB_SIZE); 1081 return -1; 1082 } 1083 int z_c = ldpc_enc->z_c; 1084 /* Check Zc is valid value */ 1085 if ((z_c > 384) || (z_c < 4)) { 1086 rte_bbdev_log(ERR, "Zc (%u) is out of range", z_c); 1087 return -1; 1088 } 1089 if (z_c > 256) { 1090 if ((z_c % 32) != 0) { 1091 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1092 return -1; 1093 } 1094 } else if (z_c > 128) { 1095 if ((z_c % 16) != 0) { 1096 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1097 return -1; 1098 } 1099 } else if (z_c > 64) { 1100 if ((z_c % 8) != 0) { 1101 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1102 return -1; 1103 } 1104 } else if (z_c > 32) { 1105 if ((z_c % 4) != 0) { 1106 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1107 return -1; 1108 } 1109 } else if (z_c > 16) { 1110 if ((z_c % 2) != 0) { 1111 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1112 return -1; 1113 } 1114 } 1115 1116 int n_filler = ldpc_enc->n_filler; 1117 int K = (ldpc_enc->basegraph == 1 ? 22 : 10) * ldpc_enc->z_c; 1118 int Kp = K - n_filler; 1119 int q_m = ldpc_enc->q_m; 1120 int n_cb = ldpc_enc->n_cb; 1121 int N = (ldpc_enc->basegraph == 1 ? N_ZC_1 : N_ZC_2) * z_c; 1122 int k0 = get_k0(n_cb, z_c, ldpc_enc->basegraph, 1123 ldpc_enc->rv_index); 1124 int crc24 = 0; 1125 int32_t L, Lcb, cw, cw_rm; 1126 int32_t e = ldpc_enc->cb_params.e; 1127 if (check_bit(op->ldpc_enc.op_flags, 1128 RTE_BBDEV_LDPC_CRC_24B_ATTACH)) 1129 crc24 = 24; 1130 1131 if (K < (int) (ldpc_enc->input.length * 8 + n_filler) + crc24) { 1132 rte_bbdev_log(ERR, "K and F not matching input size %u %u %u", 1133 K, n_filler, ldpc_enc->input.length); 1134 return -1; 1135 } 1136 if (ldpc_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1137 rte_bbdev_log(ERR, "TB mode not supported"); 1138 return -1; 1139 1140 } 1141 1142 /* K' range check */ 1143 if (Kp % 8 > 0) { 1144 rte_bbdev_log(ERR, "K' not byte aligned %u", Kp); 1145 return -1; 1146 } 1147 if ((crc24 > 0) && (Kp < 292)) { 1148 rte_bbdev_log(ERR, "Invalid CRC24 for small block %u", Kp); 1149 return -1; 1150 } 1151 if (Kp < 24) { 1152 rte_bbdev_log(ERR, "K' too small %u", Kp); 1153 return -1; 1154 } 1155 if (n_filler >= (K - 2 * z_c)) { 1156 rte_bbdev_log(ERR, "K - F invalid %u %u", K, n_filler); 1157 return -1; 1158 } 1159 /* Ncb range check */ 1160 if ((n_cb > N) || (n_cb < 32) || (n_cb <= (Kp - crc24))) { 1161 rte_bbdev_log(ERR, "Ncb (%u) is out of range K %d N %d", n_cb, K, N); 1162 return -1; 1163 } 1164 /* Qm range check */ 1165 if (!check_bit(op->ldpc_enc.op_flags, RTE_BBDEV_LDPC_INTERLEAVER_BYPASS) && 1166 ((q_m == 0) || ((q_m > 2) && ((q_m % 2) == 1)) || (q_m > 8))) { 1167 rte_bbdev_log(ERR, "Qm (%u) is out of range", q_m); 1168 return -1; 1169 } 1170 /* K0 range check */ 1171 if (((k0 % z_c) > 0) || (k0 >= n_cb) || ((k0 >= (Kp - 2 * z_c)) 1172 && (k0 < (K - 2 * z_c)))) { 1173 rte_bbdev_log(ERR, "K0 (%u) is out of range", k0); 1174 return -1; 1175 } 1176 /* E range check */ 1177 if (e <= RTE_MAX(32, z_c)) { 1178 rte_bbdev_log(ERR, "E is too small %"PRIu32"", e); 1179 return -1; 1180 } 1181 if ((e > 0xFFFF)) { 1182 rte_bbdev_log(ERR, "E is too large for N3000 %"PRIu32" > 64k", e); 1183 return -1; 1184 } 1185 if (q_m > 0) { 1186 if (e % q_m > 0) { 1187 rte_bbdev_log(ERR, "E %"PRIu32" not multiple of qm %d", e, q_m); 1188 return -1; 1189 } 1190 } 1191 /* Code word in RM range check */ 1192 if (k0 > (Kp - 2 * z_c)) 1193 L = k0 + e; 1194 else 1195 L = k0 + e + n_filler; 1196 Lcb = RTE_MIN(L, n_cb); 1197 if (ldpc_enc->basegraph == 1) { 1198 if (Lcb <= 25 * z_c) 1199 cw = 25 * z_c; 1200 else if (Lcb <= 27 * z_c) 1201 cw = 27 * z_c; 1202 else if (Lcb <= 30 * z_c) 1203 cw = 30 * z_c; 1204 else if (Lcb <= 33 * z_c) 1205 cw = 33 * z_c; 1206 else if (Lcb <= 44 * z_c) 1207 cw = 44 * z_c; 1208 else if (Lcb <= 55 * z_c) 1209 cw = 55 * z_c; 1210 else 1211 cw = 66 * z_c; 1212 } else { 1213 if (Lcb <= 15 * z_c) 1214 cw = 15 * z_c; 1215 else if (Lcb <= 20 * z_c) 1216 cw = 20 * z_c; 1217 else if (Lcb <= 25 * z_c) 1218 cw = 25 * z_c; 1219 else if (Lcb <= 30 * z_c) 1220 cw = 30 * z_c; 1221 else 1222 cw = 50 * z_c; 1223 } 1224 if (n_cb < Kp - 2 * z_c) 1225 cw_rm = n_cb; 1226 else if ((Kp - 2 * z_c <= n_cb) && (n_cb < K - 2 * z_c)) 1227 cw_rm = Kp - 2 * z_c; 1228 else if ((K - 2 * z_c <= n_cb) && (n_cb < cw)) 1229 cw_rm = n_cb - n_filler; 1230 else 1231 cw_rm = cw - n_filler; 1232 if (cw_rm <= 32) { 1233 rte_bbdev_log(ERR, 1234 "Invalid Ratematching"); 1235 return -1; 1236 } 1237 return 0; 1238 } 1239 1240 /* Validates LDPC decoder parameters */ 1241 static inline int 1242 validate_ldpc_dec_op(struct rte_bbdev_dec_op *op) 1243 { 1244 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &op->ldpc_dec; 1245 if (check_bit(ldpc_dec->op_flags, 1246 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) 1247 return 0; 1248 if (ldpc_dec->input.data == NULL) { 1249 rte_bbdev_log(ERR, "Invalid input pointer"); 1250 return -1; 1251 } 1252 if (ldpc_dec->hard_output.data == NULL) { 1253 rte_bbdev_log(ERR, "Invalid output pointer"); 1254 return -1; 1255 } 1256 if (ldpc_dec->input.length == 0) { 1257 rte_bbdev_log(ERR, "input is null"); 1258 return -1; 1259 } 1260 if ((ldpc_dec->basegraph > 2) || (ldpc_dec->basegraph == 0)) { 1261 rte_bbdev_log(ERR, 1262 "BG (%u) is out of range 1 <= value <= 2", 1263 ldpc_dec->basegraph); 1264 return -1; 1265 } 1266 if (ldpc_dec->iter_max == 0) { 1267 rte_bbdev_log(ERR, 1268 "iter_max (%u) is equal to 0", 1269 ldpc_dec->iter_max); 1270 return -1; 1271 } 1272 if (ldpc_dec->rv_index > 3) { 1273 rte_bbdev_log(ERR, 1274 "rv_index (%u) is out of range 0 <= value <= 3", 1275 ldpc_dec->rv_index); 1276 return -1; 1277 } 1278 if (ldpc_dec->code_block_mode > RTE_BBDEV_CODE_BLOCK) { 1279 rte_bbdev_log(ERR, 1280 "code_block_mode (%u) is out of range 0 <= value <= 1", 1281 ldpc_dec->code_block_mode); 1282 return -1; 1283 } 1284 if (check_bit(op->ldpc_dec.op_flags, 1285 RTE_BBDEV_LDPC_DECODE_BYPASS)) { 1286 rte_bbdev_log(ERR, "Avoid LDPC Decode bypass"); 1287 return -1; 1288 } 1289 int z_c = ldpc_dec->z_c; 1290 /* Check Zc is valid value */ 1291 if ((z_c > 384) || (z_c < 4)) { 1292 rte_bbdev_log(ERR, 1293 "Zc (%u) is out of range", 1294 z_c); 1295 return -1; 1296 } 1297 if (z_c > 256) { 1298 if ((z_c % 32) != 0) { 1299 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1300 return -1; 1301 } 1302 } else if (z_c > 128) { 1303 if ((z_c % 16) != 0) { 1304 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1305 return -1; 1306 } 1307 } else if (z_c > 64) { 1308 if ((z_c % 8) != 0) { 1309 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1310 return -1; 1311 } 1312 } else if (z_c > 32) { 1313 if ((z_c % 4) != 0) { 1314 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1315 return -1; 1316 } 1317 } else if (z_c > 16) { 1318 if ((z_c % 2) != 0) { 1319 rte_bbdev_log(ERR, "Invalid Zc %d", z_c); 1320 return -1; 1321 } 1322 } 1323 1324 int n_filler = ldpc_dec->n_filler; 1325 int K = (ldpc_dec->basegraph == 1 ? 22 : 10) * ldpc_dec->z_c; 1326 int Kp = K - n_filler; 1327 int q_m = ldpc_dec->q_m; 1328 int n_cb = ldpc_dec->n_cb; 1329 int N = (ldpc_dec->basegraph == 1 ? N_ZC_1 : N_ZC_2) * z_c; 1330 int k0 = get_k0(n_cb, z_c, ldpc_dec->basegraph, 1331 ldpc_dec->rv_index); 1332 int crc24 = 0; 1333 int32_t L, Lcb, cw, cw_rm; 1334 int32_t e = ldpc_dec->cb_params.e; 1335 if (check_bit(op->ldpc_dec.op_flags, 1336 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK)) 1337 crc24 = 24; 1338 1339 if (ldpc_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1340 rte_bbdev_log(ERR, 1341 "TB mode not supported"); 1342 return -1; 1343 } 1344 /* Enforce HARQ input length */ 1345 ldpc_dec->harq_combined_input.length = RTE_MIN((uint32_t) n_cb, 1346 ldpc_dec->harq_combined_input.length); 1347 if ((ldpc_dec->harq_combined_input.length == 0) && 1348 check_bit(ldpc_dec->op_flags, 1349 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) { 1350 rte_bbdev_log(ERR, 1351 "HARQ input length (%u) should not be null", 1352 ldpc_dec->harq_combined_input.length); 1353 return -1; 1354 } 1355 if ((ldpc_dec->harq_combined_input.length > 0) && 1356 !check_bit(ldpc_dec->op_flags, 1357 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) { 1358 ldpc_dec->harq_combined_input.length = 0; 1359 } 1360 1361 /* K' range check */ 1362 if (Kp % 8 > 0) { 1363 rte_bbdev_log(ERR, 1364 "K' not byte aligned %u", 1365 Kp); 1366 return -1; 1367 } 1368 if ((crc24 > 0) && (Kp < 292)) { 1369 rte_bbdev_log(ERR, 1370 "Invalid CRC24 for small block %u", 1371 Kp); 1372 return -1; 1373 } 1374 if (Kp < 24) { 1375 rte_bbdev_log(ERR, 1376 "K' too small %u", 1377 Kp); 1378 return -1; 1379 } 1380 if (n_filler >= (K - 2 * z_c)) { 1381 rte_bbdev_log(ERR, 1382 "K - F invalid %u %u", 1383 K, n_filler); 1384 return -1; 1385 } 1386 /* Ncb range check */ 1387 if (n_cb != N) { 1388 rte_bbdev_log(ERR, 1389 "Ncb (%u) is out of range K %d N %d", 1390 n_cb, K, N); 1391 return -1; 1392 } 1393 /* Qm range check */ 1394 if (!check_bit(op->ldpc_dec.op_flags, 1395 RTE_BBDEV_LDPC_INTERLEAVER_BYPASS) && 1396 ((q_m == 0) || ((q_m > 2) && ((q_m % 2) == 1)) 1397 || (q_m > 8))) { 1398 rte_bbdev_log(ERR, 1399 "Qm (%u) is out of range", 1400 q_m); 1401 return -1; 1402 } 1403 /* K0 range check */ 1404 if (((k0 % z_c) > 0) || (k0 >= n_cb) || ((k0 >= (Kp - 2 * z_c)) 1405 && (k0 < (K - 2 * z_c)))) { 1406 rte_bbdev_log(ERR, 1407 "K0 (%u) is out of range", 1408 k0); 1409 return -1; 1410 } 1411 /* E range check */ 1412 if (e <= RTE_MAX(32, z_c)) { 1413 rte_bbdev_log(ERR, 1414 "E is too small"); 1415 return -1; 1416 } 1417 if ((e > 0xFFFF)) { 1418 rte_bbdev_log(ERR, 1419 "E is too large"); 1420 return -1; 1421 } 1422 if (q_m > 0) { 1423 if (e % q_m > 0) { 1424 rte_bbdev_log(ERR, 1425 "E not multiple of qm %d", q_m); 1426 return -1; 1427 } 1428 } 1429 /* Code word in RM range check */ 1430 if (k0 > (Kp - 2 * z_c)) 1431 L = k0 + e; 1432 else 1433 L = k0 + e + n_filler; 1434 Lcb = RTE_MIN(n_cb, RTE_MAX(L, 1435 (int32_t) ldpc_dec->harq_combined_input.length)); 1436 if (ldpc_dec->basegraph == 1) { 1437 if (Lcb <= 25 * z_c) 1438 cw = 25 * z_c; 1439 else if (Lcb <= 27 * z_c) 1440 cw = 27 * z_c; 1441 else if (Lcb <= 30 * z_c) 1442 cw = 30 * z_c; 1443 else if (Lcb <= 33 * z_c) 1444 cw = 33 * z_c; 1445 else if (Lcb <= 44 * z_c) 1446 cw = 44 * z_c; 1447 else if (Lcb <= 55 * z_c) 1448 cw = 55 * z_c; 1449 else 1450 cw = 66 * z_c; 1451 } else { 1452 if (Lcb <= 15 * z_c) 1453 cw = 15 * z_c; 1454 else if (Lcb <= 20 * z_c) 1455 cw = 20 * z_c; 1456 else if (Lcb <= 25 * z_c) 1457 cw = 25 * z_c; 1458 else if (Lcb <= 30 * z_c) 1459 cw = 30 * z_c; 1460 else 1461 cw = 50 * z_c; 1462 } 1463 cw_rm = cw - n_filler; 1464 if (cw_rm <= 32) { 1465 rte_bbdev_log(ERR, 1466 "Invalid Ratematching"); 1467 return -1; 1468 } 1469 return 0; 1470 } 1471 1472 static inline char * 1473 mbuf_append(struct rte_mbuf *m_head, struct rte_mbuf *m, uint16_t len) 1474 { 1475 if (unlikely(len > rte_pktmbuf_tailroom(m))) 1476 return NULL; 1477 1478 char *tail = (char *)m->buf_addr + m->data_off + m->data_len; 1479 m->data_len = (uint16_t)(m->data_len + len); 1480 m_head->pkt_len = (m_head->pkt_len + len); 1481 return tail; 1482 } 1483 1484 static inline void 1485 fpga_mutex_acquisition(struct fpga_queue *q) 1486 { 1487 uint32_t mutex_ctrl, mutex_read, cnt = 0; 1488 /* Assign a unique id for the duration of the DDR access */ 1489 q->ddr_mutex_uuid = rand(); 1490 /* Request and wait for acquisition of the mutex */ 1491 mutex_ctrl = (q->ddr_mutex_uuid << 16) + 1; 1492 do { 1493 if (cnt > 0) 1494 usleep(FPGA_TIMEOUT_CHECK_INTERVAL); 1495 rte_bbdev_log_debug("Acquiring Mutex for %x\n", 1496 q->ddr_mutex_uuid); 1497 fpga_reg_write_32(q->d->mmio_base, 1498 FPGA_5GNR_FEC_MUTEX, 1499 mutex_ctrl); 1500 mutex_read = fpga_reg_read_32(q->d->mmio_base, 1501 FPGA_5GNR_FEC_MUTEX); 1502 rte_bbdev_log_debug("Mutex %x cnt %d owner %x\n", 1503 mutex_read, cnt, q->ddr_mutex_uuid); 1504 cnt++; 1505 } while ((mutex_read >> 16) != q->ddr_mutex_uuid); 1506 } 1507 1508 static inline void 1509 fpga_mutex_free(struct fpga_queue *q) 1510 { 1511 uint32_t mutex_ctrl = q->ddr_mutex_uuid << 16; 1512 fpga_reg_write_32(q->d->mmio_base, 1513 FPGA_5GNR_FEC_MUTEX, 1514 mutex_ctrl); 1515 } 1516 1517 static inline int 1518 fpga_harq_write_loopback(struct fpga_queue *q, 1519 struct rte_mbuf *harq_input, uint16_t harq_in_length, 1520 uint32_t harq_in_offset, uint32_t harq_out_offset) 1521 { 1522 fpga_mutex_acquisition(q); 1523 uint32_t out_offset = harq_out_offset; 1524 uint32_t in_offset = harq_in_offset; 1525 uint32_t left_length = harq_in_length; 1526 uint32_t reg_32, increment = 0; 1527 uint64_t *input = NULL; 1528 uint32_t last_transaction = left_length 1529 % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; 1530 uint64_t last_word; 1531 1532 if (last_transaction > 0) 1533 left_length -= last_transaction; 1534 1535 /* 1536 * Get HARQ buffer size for each VF/PF: When 0x00, there is no 1537 * available DDR space for the corresponding VF/PF. 1538 */ 1539 reg_32 = fpga_reg_read_32(q->d->mmio_base, 1540 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS); 1541 if (reg_32 < harq_in_length) { 1542 left_length = reg_32; 1543 rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n"); 1544 } 1545 1546 input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_input, 1547 uint8_t *, in_offset); 1548 1549 while (left_length > 0) { 1550 if (fpga_reg_read_8(q->d->mmio_base, 1551 FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) == 1) { 1552 fpga_reg_write_32(q->d->mmio_base, 1553 FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS, 1554 out_offset); 1555 fpga_reg_write_64(q->d->mmio_base, 1556 FPGA_5GNR_FEC_DDR4_WR_DATA_REGS, 1557 input[increment]); 1558 left_length -= FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; 1559 out_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; 1560 increment++; 1561 fpga_reg_write_8(q->d->mmio_base, 1562 FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1); 1563 } 1564 } 1565 while (last_transaction > 0) { 1566 if (fpga_reg_read_8(q->d->mmio_base, 1567 FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) == 1) { 1568 fpga_reg_write_32(q->d->mmio_base, 1569 FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS, 1570 out_offset); 1571 last_word = input[increment]; 1572 last_word &= (uint64_t)(1 << (last_transaction * 4)) 1573 - 1; 1574 fpga_reg_write_64(q->d->mmio_base, 1575 FPGA_5GNR_FEC_DDR4_WR_DATA_REGS, 1576 last_word); 1577 fpga_reg_write_8(q->d->mmio_base, 1578 FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1); 1579 last_transaction = 0; 1580 } 1581 } 1582 fpga_mutex_free(q); 1583 return 1; 1584 } 1585 1586 static inline int 1587 fpga_harq_read_loopback(struct fpga_queue *q, 1588 struct rte_mbuf *harq_output, uint16_t harq_in_length, 1589 uint32_t harq_in_offset, uint32_t harq_out_offset) 1590 { 1591 fpga_mutex_acquisition(q); 1592 uint32_t left_length, in_offset = harq_in_offset; 1593 uint64_t reg; 1594 uint32_t increment = 0; 1595 uint64_t *input = NULL; 1596 uint32_t last_transaction = harq_in_length 1597 % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; 1598 1599 if (last_transaction > 0) 1600 harq_in_length += (8 - last_transaction); 1601 1602 reg = fpga_reg_read_32(q->d->mmio_base, 1603 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS); 1604 if (reg < harq_in_length) { 1605 harq_in_length = reg; 1606 rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n"); 1607 } 1608 1609 if (!mbuf_append(harq_output, harq_output, harq_in_length)) { 1610 rte_bbdev_log(ERR, "HARQ output buffer warning %d %d\n", 1611 harq_output->buf_len - 1612 rte_pktmbuf_headroom(harq_output), 1613 harq_in_length); 1614 harq_in_length = harq_output->buf_len - 1615 rte_pktmbuf_headroom(harq_output); 1616 if (!mbuf_append(harq_output, harq_output, harq_in_length)) { 1617 rte_bbdev_log(ERR, "HARQ output buffer issue %d %d\n", 1618 harq_output->buf_len, harq_in_length); 1619 return -1; 1620 } 1621 } 1622 left_length = harq_in_length; 1623 1624 input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_output, 1625 uint8_t *, harq_out_offset); 1626 1627 while (left_length > 0) { 1628 fpga_reg_write_32(q->d->mmio_base, 1629 FPGA_5GNR_FEC_DDR4_RD_ADDR_REGS, in_offset); 1630 fpga_reg_write_8(q->d->mmio_base, 1631 FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 1); 1632 reg = fpga_reg_read_8(q->d->mmio_base, 1633 FPGA_5GNR_FEC_DDR4_RD_RDY_REGS); 1634 while (reg != 1) { 1635 reg = fpga_reg_read_8(q->d->mmio_base, 1636 FPGA_5GNR_FEC_DDR4_RD_RDY_REGS); 1637 if (reg == FPGA_DDR_OVERFLOW) { 1638 rte_bbdev_log(ERR, 1639 "Read address is overflow!\n"); 1640 return -1; 1641 } 1642 } 1643 input[increment] = fpga_reg_read_64(q->d->mmio_base, 1644 FPGA_5GNR_FEC_DDR4_RD_DATA_REGS); 1645 left_length -= FPGA_5GNR_FEC_DDR_RD_DATA_LEN_IN_BYTES; 1646 in_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; 1647 increment++; 1648 fpga_reg_write_8(q->d->mmio_base, 1649 FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 0); 1650 } 1651 fpga_mutex_free(q); 1652 return 1; 1653 } 1654 1655 static inline int 1656 enqueue_ldpc_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op *op, 1657 uint16_t desc_offset) 1658 { 1659 union fpga_dma_desc *desc; 1660 int ret; 1661 uint8_t c, crc24_bits = 0; 1662 struct rte_bbdev_op_ldpc_enc *enc = &op->ldpc_enc; 1663 uint16_t in_offset = enc->input.offset; 1664 uint16_t out_offset = enc->output.offset; 1665 struct rte_mbuf *m_in = enc->input.data; 1666 struct rte_mbuf *m_out = enc->output.data; 1667 struct rte_mbuf *m_out_head = enc->output.data; 1668 uint32_t in_length, out_length, e; 1669 uint16_t total_left = enc->input.length; 1670 uint16_t ring_offset; 1671 uint16_t K, k_; 1672 1673 1674 if (validate_ldpc_enc_op(op) == -1) { 1675 rte_bbdev_log(ERR, "LDPC encoder validation rejected"); 1676 return -EINVAL; 1677 } 1678 1679 /* Clear op status */ 1680 op->status = 0; 1681 1682 if (m_in == NULL || m_out == NULL) { 1683 rte_bbdev_log(ERR, "Invalid mbuf pointer"); 1684 op->status = 1 << RTE_BBDEV_DATA_ERROR; 1685 return -EINVAL; 1686 } 1687 1688 if (enc->op_flags & RTE_BBDEV_LDPC_CRC_24B_ATTACH) 1689 crc24_bits = 24; 1690 1691 if (enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1692 /* For Transport Block mode */ 1693 /* FIXME */ 1694 c = enc->tb_params.c; 1695 e = enc->tb_params.ea; 1696 } else { /* For Code Block mode */ 1697 c = 1; 1698 e = enc->cb_params.e; 1699 } 1700 1701 /* Update total_left */ 1702 K = (enc->basegraph == 1 ? 22 : 10) * enc->z_c; 1703 k_ = K - enc->n_filler; 1704 in_length = (k_ - crc24_bits) >> 3; 1705 out_length = (e + 7) >> 3; 1706 1707 total_left = rte_pktmbuf_data_len(m_in) - in_offset; 1708 1709 /* Update offsets */ 1710 if (total_left != in_length) { 1711 op->status |= 1 << RTE_BBDEV_DATA_ERROR; 1712 rte_bbdev_log(ERR, 1713 "Mismatch between mbuf length and included CBs sizes %d", 1714 total_left); 1715 } 1716 1717 mbuf_append(m_out_head, m_out, out_length); 1718 1719 /* Offset into the ring */ 1720 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1721 /* Setup DMA Descriptor */ 1722 desc = q->ring_addr + ring_offset; 1723 1724 ret = fpga_dma_desc_te_fill(op, &desc->enc_req, m_in, m_out, 1725 k_, e, in_offset, out_offset, ring_offset, c); 1726 if (unlikely(ret < 0)) 1727 return ret; 1728 1729 /* Update lengths */ 1730 total_left -= in_length; 1731 op->ldpc_enc.output.length += out_length; 1732 1733 if (total_left > 0) { 1734 rte_bbdev_log(ERR, 1735 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u", 1736 total_left, in_length); 1737 return -1; 1738 } 1739 1740 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1741 print_dma_enc_desc_debug_info(desc); 1742 #endif 1743 return 1; 1744 } 1745 1746 static inline int 1747 enqueue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op *op, 1748 uint16_t desc_offset) 1749 { 1750 union fpga_dma_desc *desc; 1751 int ret; 1752 uint16_t ring_offset; 1753 uint8_t c; 1754 uint16_t e, in_length, out_length, k0, l, seg_total_left, sys_cols; 1755 uint16_t K, parity_offset, harq_in_length = 0, harq_out_length = 0; 1756 uint16_t crc24_overlap = 0; 1757 struct rte_bbdev_op_ldpc_dec *dec = &op->ldpc_dec; 1758 struct rte_mbuf *m_in = dec->input.data; 1759 struct rte_mbuf *m_out = dec->hard_output.data; 1760 struct rte_mbuf *m_out_head = dec->hard_output.data; 1761 uint16_t in_offset = dec->input.offset; 1762 uint16_t out_offset = dec->hard_output.offset; 1763 uint32_t harq_offset = 0; 1764 1765 if (validate_ldpc_dec_op(op) == -1) { 1766 rte_bbdev_log(ERR, "LDPC decoder validation rejected"); 1767 return -EINVAL; 1768 } 1769 1770 /* Clear op status */ 1771 op->status = 0; 1772 1773 /* Setup DMA Descriptor */ 1774 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1775 desc = q->ring_addr + ring_offset; 1776 1777 if (check_bit(dec->op_flags, 1778 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 1779 struct rte_mbuf *harq_in = dec->harq_combined_input.data; 1780 struct rte_mbuf *harq_out = dec->harq_combined_output.data; 1781 harq_in_length = dec->harq_combined_input.length; 1782 uint32_t harq_in_offset = dec->harq_combined_input.offset; 1783 uint32_t harq_out_offset = dec->harq_combined_output.offset; 1784 1785 if (check_bit(dec->op_flags, 1786 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE 1787 )) { 1788 ret = fpga_harq_write_loopback(q, harq_in, 1789 harq_in_length, harq_in_offset, 1790 harq_out_offset); 1791 } else if (check_bit(dec->op_flags, 1792 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE 1793 )) { 1794 ret = fpga_harq_read_loopback(q, harq_out, 1795 harq_in_length, harq_in_offset, 1796 harq_out_offset); 1797 dec->harq_combined_output.length = harq_in_length; 1798 } else { 1799 rte_bbdev_log(ERR, "OP flag Err!"); 1800 ret = -1; 1801 } 1802 /* Set descriptor for dequeue */ 1803 desc->dec_req.done = 1; 1804 desc->dec_req.error = 0; 1805 desc->dec_req.op_addr = op; 1806 desc->dec_req.cbs_in_op = 1; 1807 /* Mark this dummy descriptor to be dropped by HW */ 1808 desc->dec_req.desc_idx = (ring_offset + 1) 1809 & q->sw_ring_wrap_mask; 1810 return ret; /* Error or number of CB */ 1811 } 1812 1813 if (m_in == NULL || m_out == NULL) { 1814 rte_bbdev_log(ERR, "Invalid mbuf pointer"); 1815 op->status = 1 << RTE_BBDEV_DATA_ERROR; 1816 return -1; 1817 } 1818 1819 c = 1; 1820 e = dec->cb_params.e; 1821 1822 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP)) 1823 crc24_overlap = 24; 1824 1825 sys_cols = (dec->basegraph == 1) ? 22 : 10; 1826 K = sys_cols * dec->z_c; 1827 parity_offset = K - 2 * dec->z_c; 1828 1829 out_length = ((K - crc24_overlap - dec->n_filler) >> 3); 1830 in_length = e; 1831 seg_total_left = dec->input.length; 1832 1833 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) { 1834 harq_in_length = RTE_MIN(dec->harq_combined_input.length, 1835 (uint32_t)dec->n_cb); 1836 } 1837 1838 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) { 1839 k0 = get_k0(dec->n_cb, dec->z_c, 1840 dec->basegraph, dec->rv_index); 1841 if (k0 > parity_offset) 1842 l = k0 + e; 1843 else 1844 l = k0 + e + dec->n_filler; 1845 harq_out_length = RTE_MIN(RTE_MAX(harq_in_length, l), 1846 dec->n_cb); 1847 dec->harq_combined_output.length = harq_out_length; 1848 } 1849 1850 mbuf_append(m_out_head, m_out, out_length); 1851 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) 1852 harq_offset = dec->harq_combined_input.offset; 1853 else if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) 1854 harq_offset = dec->harq_combined_output.offset; 1855 1856 if ((harq_offset & 0x3FF) > 0) { 1857 rte_bbdev_log(ERR, "Invalid HARQ offset %d", harq_offset); 1858 op->status = 1 << RTE_BBDEV_DATA_ERROR; 1859 return -1; 1860 } 1861 1862 ret = fpga_dma_desc_ld_fill(op, &desc->dec_req, m_in, m_out, 1863 harq_in_length, in_offset, out_offset, harq_offset, 1864 ring_offset, c); 1865 if (unlikely(ret < 0)) 1866 return ret; 1867 /* Update lengths */ 1868 seg_total_left -= in_length; 1869 op->ldpc_dec.hard_output.length += out_length; 1870 if (seg_total_left > 0) { 1871 rte_bbdev_log(ERR, 1872 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u", 1873 seg_total_left, in_length); 1874 return -1; 1875 } 1876 1877 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1878 print_dma_dec_desc_debug_info(desc); 1879 #endif 1880 1881 return 1; 1882 } 1883 1884 static uint16_t 1885 fpga_enqueue_ldpc_enc(struct rte_bbdev_queue_data *q_data, 1886 struct rte_bbdev_enc_op **ops, uint16_t num) 1887 { 1888 uint16_t i, total_enqueued_cbs = 0; 1889 int32_t avail; 1890 int enqueued_cbs; 1891 struct fpga_queue *q = q_data->queue_private; 1892 union fpga_dma_desc *desc; 1893 1894 /* Check if queue is not full */ 1895 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) == 1896 q->head_free_desc)) 1897 return 0; 1898 1899 /* Calculates available space */ 1900 avail = (q->head_free_desc > q->tail) ? 1901 q->head_free_desc - q->tail - 1 : 1902 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1; 1903 1904 for (i = 0; i < num; ++i) { 1905 1906 /* Check if there is available space for further 1907 * processing 1908 */ 1909 if (unlikely(avail - 1 < 0)) 1910 break; 1911 avail -= 1; 1912 enqueued_cbs = enqueue_ldpc_enc_one_op_cb(q, ops[i], 1913 total_enqueued_cbs); 1914 1915 if (enqueued_cbs < 0) 1916 break; 1917 1918 total_enqueued_cbs += enqueued_cbs; 1919 1920 rte_bbdev_log_debug("enqueuing enc ops [%d/%d] | head %d | tail %d", 1921 total_enqueued_cbs, num, 1922 q->head_free_desc, q->tail); 1923 } 1924 1925 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt 1926 * only when all previous CBs were already processed. 1927 */ 1928 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1) 1929 & q->sw_ring_wrap_mask); 1930 desc->enc_req.irq_en = q->irq_enable; 1931 1932 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats); 1933 1934 /* Update stats */ 1935 q_data->queue_stats.enqueued_count += i; 1936 q_data->queue_stats.enqueue_err_count += num - i; 1937 1938 return i; 1939 } 1940 1941 static uint16_t 1942 fpga_enqueue_ldpc_dec(struct rte_bbdev_queue_data *q_data, 1943 struct rte_bbdev_dec_op **ops, uint16_t num) 1944 { 1945 uint16_t i, total_enqueued_cbs = 0; 1946 int32_t avail; 1947 int enqueued_cbs; 1948 struct fpga_queue *q = q_data->queue_private; 1949 union fpga_dma_desc *desc; 1950 1951 /* Check if queue is not full */ 1952 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) == 1953 q->head_free_desc)) 1954 return 0; 1955 1956 /* Calculates available space */ 1957 avail = (q->head_free_desc > q->tail) ? 1958 q->head_free_desc - q->tail - 1 : 1959 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1; 1960 1961 for (i = 0; i < num; ++i) { 1962 1963 /* Check if there is available space for further 1964 * processing 1965 */ 1966 if (unlikely(avail - 1 < 0)) 1967 break; 1968 avail -= 1; 1969 enqueued_cbs = enqueue_ldpc_dec_one_op_cb(q, ops[i], 1970 total_enqueued_cbs); 1971 1972 if (enqueued_cbs < 0) 1973 break; 1974 1975 total_enqueued_cbs += enqueued_cbs; 1976 1977 rte_bbdev_log_debug("enqueuing dec ops [%d/%d] | head %d | tail %d", 1978 total_enqueued_cbs, num, 1979 q->head_free_desc, q->tail); 1980 } 1981 1982 /* Update stats */ 1983 q_data->queue_stats.enqueued_count += i; 1984 q_data->queue_stats.enqueue_err_count += num - i; 1985 1986 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt 1987 * only when all previous CBs were already processed. 1988 */ 1989 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1) 1990 & q->sw_ring_wrap_mask); 1991 desc->enc_req.irq_en = q->irq_enable; 1992 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats); 1993 return i; 1994 } 1995 1996 1997 static inline int 1998 dequeue_ldpc_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op **op, 1999 uint16_t desc_offset) 2000 { 2001 union fpga_dma_desc *desc; 2002 int desc_error; 2003 /* Set current desc */ 2004 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2005 & q->sw_ring_wrap_mask); 2006 2007 /*check if done */ 2008 if (desc->enc_req.done == 0) 2009 return -1; 2010 2011 /* make sure the response is read atomically */ 2012 rte_smp_rmb(); 2013 2014 rte_bbdev_log_debug("DMA response desc %p", desc); 2015 2016 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2017 print_dma_enc_desc_debug_info(desc); 2018 #endif 2019 2020 *op = desc->enc_req.op_addr; 2021 /* Check the descriptor error field, return 1 on error */ 2022 desc_error = check_desc_error(desc->enc_req.error); 2023 (*op)->status = desc_error << RTE_BBDEV_DATA_ERROR; 2024 2025 return 1; 2026 } 2027 2028 2029 static inline int 2030 dequeue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op **op, 2031 uint16_t desc_offset) 2032 { 2033 union fpga_dma_desc *desc; 2034 int desc_error; 2035 /* Set descriptor */ 2036 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2037 & q->sw_ring_wrap_mask); 2038 2039 /* Verify done bit is set */ 2040 if (desc->dec_req.done == 0) 2041 return -1; 2042 2043 /* make sure the response is read atomically */ 2044 rte_smp_rmb(); 2045 2046 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2047 print_dma_dec_desc_debug_info(desc); 2048 #endif 2049 2050 *op = desc->dec_req.op_addr; 2051 2052 if (check_bit((*op)->ldpc_dec.op_flags, 2053 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 2054 (*op)->status = 0; 2055 return 1; 2056 } 2057 2058 /* FPGA reports iterations based on round-up minus 1 */ 2059 (*op)->ldpc_dec.iter_count = desc->dec_req.iter + 1; 2060 /* CRC Check criteria */ 2061 if (desc->dec_req.crc24b_ind && !(desc->dec_req.crcb_pass)) 2062 (*op)->status = 1 << RTE_BBDEV_CRC_ERROR; 2063 /* et_pass = 0 when decoder fails */ 2064 (*op)->status |= !(desc->dec_req.et_pass) << RTE_BBDEV_SYNDROME_ERROR; 2065 /* Check the descriptor error field, return 1 on error */ 2066 desc_error = check_desc_error(desc->dec_req.error); 2067 (*op)->status |= desc_error << RTE_BBDEV_DATA_ERROR; 2068 return 1; 2069 } 2070 2071 static uint16_t 2072 fpga_dequeue_ldpc_enc(struct rte_bbdev_queue_data *q_data, 2073 struct rte_bbdev_enc_op **ops, uint16_t num) 2074 { 2075 struct fpga_queue *q = q_data->queue_private; 2076 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask; 2077 uint16_t i; 2078 uint16_t dequeued_cbs = 0; 2079 int ret; 2080 2081 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) { 2082 ret = dequeue_ldpc_enc_one_op_cb(q, &ops[i], dequeued_cbs); 2083 2084 if (ret < 0) 2085 break; 2086 2087 dequeued_cbs += ret; 2088 2089 rte_bbdev_log_debug("dequeuing enc ops [%d/%d] | head %d | tail %d", 2090 dequeued_cbs, num, q->head_free_desc, q->tail); 2091 } 2092 2093 /* Update head */ 2094 q->head_free_desc = (q->head_free_desc + dequeued_cbs) & 2095 q->sw_ring_wrap_mask; 2096 2097 /* Update stats */ 2098 q_data->queue_stats.dequeued_count += i; 2099 2100 return i; 2101 } 2102 2103 static uint16_t 2104 fpga_dequeue_ldpc_dec(struct rte_bbdev_queue_data *q_data, 2105 struct rte_bbdev_dec_op **ops, uint16_t num) 2106 { 2107 struct fpga_queue *q = q_data->queue_private; 2108 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask; 2109 uint16_t i; 2110 uint16_t dequeued_cbs = 0; 2111 int ret; 2112 2113 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) { 2114 ret = dequeue_ldpc_dec_one_op_cb(q, &ops[i], dequeued_cbs); 2115 2116 if (ret < 0) 2117 break; 2118 2119 dequeued_cbs += ret; 2120 2121 rte_bbdev_log_debug("dequeuing dec ops [%d/%d] | head %d | tail %d", 2122 dequeued_cbs, num, q->head_free_desc, q->tail); 2123 } 2124 2125 /* Update head */ 2126 q->head_free_desc = (q->head_free_desc + dequeued_cbs) & 2127 q->sw_ring_wrap_mask; 2128 2129 /* Update stats */ 2130 q_data->queue_stats.dequeued_count += i; 2131 2132 return i; 2133 } 2134 2135 2136 /* Initialization Function */ 2137 static void 2138 fpga_5gnr_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv) 2139 { 2140 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); 2141 2142 dev->dev_ops = &fpga_ops; 2143 dev->enqueue_ldpc_enc_ops = fpga_enqueue_ldpc_enc; 2144 dev->enqueue_ldpc_dec_ops = fpga_enqueue_ldpc_dec; 2145 dev->dequeue_ldpc_enc_ops = fpga_dequeue_ldpc_enc; 2146 dev->dequeue_ldpc_dec_ops = fpga_dequeue_ldpc_dec; 2147 2148 ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->pf_device = 2149 !strcmp(drv->driver.name, 2150 RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME)); 2151 ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->mmio_base = 2152 pci_dev->mem_resource[0].addr; 2153 2154 rte_bbdev_log_debug( 2155 "Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64, 2156 drv->driver.name, dev->data->name, 2157 (void *)pci_dev->mem_resource[0].addr, 2158 pci_dev->mem_resource[0].phys_addr); 2159 } 2160 2161 static int 2162 fpga_5gnr_fec_probe(struct rte_pci_driver *pci_drv, 2163 struct rte_pci_device *pci_dev) 2164 { 2165 struct rte_bbdev *bbdev = NULL; 2166 char dev_name[RTE_BBDEV_NAME_MAX_LEN]; 2167 2168 if (pci_dev == NULL) { 2169 rte_bbdev_log(ERR, "NULL PCI device"); 2170 return -EINVAL; 2171 } 2172 2173 rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name)); 2174 2175 /* Allocate memory to be used privately by drivers */ 2176 bbdev = rte_bbdev_allocate(pci_dev->device.name); 2177 if (bbdev == NULL) 2178 return -ENODEV; 2179 2180 /* allocate device private memory */ 2181 bbdev->data->dev_private = rte_zmalloc_socket(dev_name, 2182 sizeof(struct fpga_5gnr_fec_device), 2183 RTE_CACHE_LINE_SIZE, 2184 pci_dev->device.numa_node); 2185 2186 if (bbdev->data->dev_private == NULL) { 2187 rte_bbdev_log(CRIT, 2188 "Allocate of %zu bytes for device \"%s\" failed", 2189 sizeof(struct fpga_5gnr_fec_device), dev_name); 2190 rte_bbdev_release(bbdev); 2191 return -ENOMEM; 2192 } 2193 2194 /* Fill HW specific part of device structure */ 2195 bbdev->device = &pci_dev->device; 2196 bbdev->intr_handle = pci_dev->intr_handle; 2197 bbdev->data->socket_id = pci_dev->device.numa_node; 2198 2199 /* Invoke FEC FPGA device initialization function */ 2200 fpga_5gnr_fec_init(bbdev, pci_drv); 2201 2202 rte_bbdev_log_debug("bbdev id = %u [%s]", 2203 bbdev->data->dev_id, dev_name); 2204 2205 struct fpga_5gnr_fec_device *d = bbdev->data->dev_private; 2206 uint32_t version_id = fpga_reg_read_32(d->mmio_base, 2207 FPGA_5GNR_FEC_VERSION_ID); 2208 rte_bbdev_log(INFO, "FEC FPGA RTL v%u.%u", 2209 ((uint16_t)(version_id >> 16)), ((uint16_t)version_id)); 2210 2211 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2212 if (!strcmp(pci_drv->driver.name, 2213 RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME))) 2214 print_static_reg_debug_info(d->mmio_base); 2215 #endif 2216 return 0; 2217 } 2218 2219 static int 2220 fpga_5gnr_fec_remove(struct rte_pci_device *pci_dev) 2221 { 2222 struct rte_bbdev *bbdev; 2223 int ret; 2224 uint8_t dev_id; 2225 2226 if (pci_dev == NULL) 2227 return -EINVAL; 2228 2229 /* Find device */ 2230 bbdev = rte_bbdev_get_named_dev(pci_dev->device.name); 2231 if (bbdev == NULL) { 2232 rte_bbdev_log(CRIT, 2233 "Couldn't find HW dev \"%s\" to uninitialise it", 2234 pci_dev->device.name); 2235 return -ENODEV; 2236 } 2237 dev_id = bbdev->data->dev_id; 2238 2239 /* free device private memory before close */ 2240 rte_free(bbdev->data->dev_private); 2241 2242 /* Close device */ 2243 ret = rte_bbdev_close(dev_id); 2244 if (ret < 0) 2245 rte_bbdev_log(ERR, 2246 "Device %i failed to close during uninit: %i", 2247 dev_id, ret); 2248 2249 /* release bbdev from library */ 2250 ret = rte_bbdev_release(bbdev); 2251 if (ret) 2252 rte_bbdev_log(ERR, "Device %i failed to uninit: %i", dev_id, 2253 ret); 2254 2255 rte_bbdev_log_debug("Destroyed bbdev = %u", dev_id); 2256 2257 return 0; 2258 } 2259 2260 static inline void 2261 set_default_fpga_conf(struct rte_fpga_5gnr_fec_conf *def_conf) 2262 { 2263 /* clear default configuration before initialization */ 2264 memset(def_conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf)); 2265 /* Set pf mode to true */ 2266 def_conf->pf_mode_en = true; 2267 2268 /* Set ratio between UL and DL to 1:1 (unit of weight is 3 CBs) */ 2269 def_conf->ul_bandwidth = 3; 2270 def_conf->dl_bandwidth = 3; 2271 2272 /* Set Load Balance Factor to 64 */ 2273 def_conf->dl_load_balance = 64; 2274 def_conf->ul_load_balance = 64; 2275 } 2276 2277 /* Initial configuration of FPGA 5GNR FEC device */ 2278 int 2279 rte_fpga_5gnr_fec_configure(const char *dev_name, 2280 const struct rte_fpga_5gnr_fec_conf *conf) 2281 { 2282 uint32_t payload_32, address; 2283 uint16_t payload_16; 2284 uint8_t payload_8; 2285 uint16_t q_id, vf_id, total_q_id, total_ul_q_id, total_dl_q_id; 2286 struct rte_bbdev *bbdev = rte_bbdev_get_named_dev(dev_name); 2287 struct rte_fpga_5gnr_fec_conf def_conf; 2288 2289 if (bbdev == NULL) { 2290 rte_bbdev_log(ERR, 2291 "Invalid dev_name (%s), or device is not yet initialised", 2292 dev_name); 2293 return -ENODEV; 2294 } 2295 2296 struct fpga_5gnr_fec_device *d = bbdev->data->dev_private; 2297 2298 if (conf == NULL) { 2299 rte_bbdev_log(ERR, 2300 "FPGA Configuration was not provided. Default configuration will be loaded."); 2301 set_default_fpga_conf(&def_conf); 2302 conf = &def_conf; 2303 } 2304 2305 /* 2306 * Configure UL:DL ratio. 2307 * [7:0]: UL weight 2308 * [15:8]: DL weight 2309 */ 2310 payload_16 = (conf->dl_bandwidth << 8) | conf->ul_bandwidth; 2311 address = FPGA_5GNR_FEC_CONFIGURATION; 2312 fpga_reg_write_16(d->mmio_base, address, payload_16); 2313 2314 /* Clear all queues registers */ 2315 payload_32 = FPGA_INVALID_HW_QUEUE_ID; 2316 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 2317 address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP; 2318 fpga_reg_write_32(d->mmio_base, address, payload_32); 2319 } 2320 2321 /* 2322 * If PF mode is enabled allocate all queues for PF only. 2323 * 2324 * For VF mode each VF can have different number of UL and DL queues. 2325 * Total number of queues to configure cannot exceed FPGA 2326 * capabilities - 64 queues - 32 queues for UL and 32 queues for DL. 2327 * Queues mapping is done according to configuration: 2328 * 2329 * UL queues: 2330 * | Q_ID | VF_ID | 2331 * | 0 | 0 | 2332 * | ... | 0 | 2333 * | conf->vf_dl_queues_number[0] - 1 | 0 | 2334 * | conf->vf_dl_queues_number[0] | 1 | 2335 * | ... | 1 | 2336 * | conf->vf_dl_queues_number[1] - 1 | 1 | 2337 * | ... | ... | 2338 * | conf->vf_dl_queues_number[7] - 1 | 7 | 2339 * 2340 * DL queues: 2341 * | Q_ID | VF_ID | 2342 * | 32 | 0 | 2343 * | ... | 0 | 2344 * | conf->vf_ul_queues_number[0] - 1 | 0 | 2345 * | conf->vf_ul_queues_number[0] | 1 | 2346 * | ... | 1 | 2347 * | conf->vf_ul_queues_number[1] - 1 | 1 | 2348 * | ... | ... | 2349 * | conf->vf_ul_queues_number[7] - 1 | 7 | 2350 * 2351 * Example of configuration: 2352 * conf->vf_ul_queues_number[0] = 4; -> 4 UL queues for VF0 2353 * conf->vf_dl_queues_number[0] = 4; -> 4 DL queues for VF0 2354 * conf->vf_ul_queues_number[1] = 2; -> 2 UL queues for VF1 2355 * conf->vf_dl_queues_number[1] = 2; -> 2 DL queues for VF1 2356 * 2357 * UL: 2358 * | Q_ID | VF_ID | 2359 * | 0 | 0 | 2360 * | 1 | 0 | 2361 * | 2 | 0 | 2362 * | 3 | 0 | 2363 * | 4 | 1 | 2364 * | 5 | 1 | 2365 * 2366 * DL: 2367 * | Q_ID | VF_ID | 2368 * | 32 | 0 | 2369 * | 33 | 0 | 2370 * | 34 | 0 | 2371 * | 35 | 0 | 2372 * | 36 | 1 | 2373 * | 37 | 1 | 2374 */ 2375 if (conf->pf_mode_en) { 2376 payload_32 = 0x1; 2377 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 2378 address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP; 2379 fpga_reg_write_32(d->mmio_base, address, payload_32); 2380 } 2381 } else { 2382 /* Calculate total number of UL and DL queues to configure */ 2383 total_ul_q_id = total_dl_q_id = 0; 2384 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) { 2385 total_ul_q_id += conf->vf_ul_queues_number[vf_id]; 2386 total_dl_q_id += conf->vf_dl_queues_number[vf_id]; 2387 } 2388 total_q_id = total_dl_q_id + total_ul_q_id; 2389 /* 2390 * Check if total number of queues to configure does not exceed 2391 * FPGA capabilities (64 queues - 32 UL and 32 DL queues) 2392 */ 2393 if ((total_ul_q_id > FPGA_NUM_UL_QUEUES) || 2394 (total_dl_q_id > FPGA_NUM_DL_QUEUES) || 2395 (total_q_id > FPGA_TOTAL_NUM_QUEUES)) { 2396 rte_bbdev_log(ERR, 2397 "FPGA Configuration failed. Too many queues to configure: UL_Q %u, DL_Q %u, FPGA_Q %u", 2398 total_ul_q_id, total_dl_q_id, 2399 FPGA_TOTAL_NUM_QUEUES); 2400 return -EINVAL; 2401 } 2402 total_ul_q_id = 0; 2403 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) { 2404 for (q_id = 0; q_id < conf->vf_ul_queues_number[vf_id]; 2405 ++q_id, ++total_ul_q_id) { 2406 address = (total_ul_q_id << 2) + 2407 FPGA_5GNR_FEC_QUEUE_MAP; 2408 payload_32 = ((0x80 + vf_id) << 16) | 0x1; 2409 fpga_reg_write_32(d->mmio_base, address, 2410 payload_32); 2411 } 2412 } 2413 total_dl_q_id = 0; 2414 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) { 2415 for (q_id = 0; q_id < conf->vf_dl_queues_number[vf_id]; 2416 ++q_id, ++total_dl_q_id) { 2417 address = ((total_dl_q_id + FPGA_NUM_UL_QUEUES) 2418 << 2) + FPGA_5GNR_FEC_QUEUE_MAP; 2419 payload_32 = ((0x80 + vf_id) << 16) | 0x1; 2420 fpga_reg_write_32(d->mmio_base, address, 2421 payload_32); 2422 } 2423 } 2424 } 2425 2426 /* Setting Load Balance Factor */ 2427 payload_16 = (conf->dl_load_balance << 8) | (conf->ul_load_balance); 2428 address = FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR; 2429 fpga_reg_write_16(d->mmio_base, address, payload_16); 2430 2431 /* Setting length of ring descriptor entry */ 2432 payload_16 = FPGA_RING_DESC_ENTRY_LENGTH; 2433 address = FPGA_5GNR_FEC_RING_DESC_LEN; 2434 fpga_reg_write_16(d->mmio_base, address, payload_16); 2435 2436 /* Queue PF/VF mapping table is ready */ 2437 payload_8 = 0x1; 2438 address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE; 2439 fpga_reg_write_8(d->mmio_base, address, payload_8); 2440 2441 rte_bbdev_log_debug("PF FPGA 5GNR FEC configuration complete for %s", 2442 dev_name); 2443 2444 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2445 print_static_reg_debug_info(d->mmio_base); 2446 #endif 2447 return 0; 2448 } 2449 2450 /* FPGA 5GNR FEC PCI PF address map */ 2451 static struct rte_pci_id pci_id_fpga_5gnr_fec_pf_map[] = { 2452 { 2453 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID, 2454 FPGA_5GNR_FEC_PF_DEVICE_ID) 2455 }, 2456 {.device_id = 0}, 2457 }; 2458 2459 static struct rte_pci_driver fpga_5gnr_fec_pci_pf_driver = { 2460 .probe = fpga_5gnr_fec_probe, 2461 .remove = fpga_5gnr_fec_remove, 2462 .id_table = pci_id_fpga_5gnr_fec_pf_map, 2463 .drv_flags = RTE_PCI_DRV_NEED_MAPPING 2464 }; 2465 2466 /* FPGA 5GNR FEC PCI VF address map */ 2467 static struct rte_pci_id pci_id_fpga_5gnr_fec_vf_map[] = { 2468 { 2469 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID, 2470 FPGA_5GNR_FEC_VF_DEVICE_ID) 2471 }, 2472 {.device_id = 0}, 2473 }; 2474 2475 static struct rte_pci_driver fpga_5gnr_fec_pci_vf_driver = { 2476 .probe = fpga_5gnr_fec_probe, 2477 .remove = fpga_5gnr_fec_remove, 2478 .id_table = pci_id_fpga_5gnr_fec_vf_map, 2479 .drv_flags = RTE_PCI_DRV_NEED_MAPPING 2480 }; 2481 2482 2483 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_PF_DRIVER_NAME, fpga_5gnr_fec_pci_pf_driver); 2484 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_PF_DRIVER_NAME, 2485 pci_id_fpga_5gnr_fec_pf_map); 2486 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_VF_DRIVER_NAME, fpga_5gnr_fec_pci_vf_driver); 2487 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_VF_DRIVER_NAME, 2488 pci_id_fpga_5gnr_fec_vf_map); 2489