1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2020 Intel Corporation 3 */ 4 5 #include <unistd.h> 6 7 #include <rte_common.h> 8 #include <rte_log.h> 9 #include <rte_dev.h> 10 #include <rte_malloc.h> 11 #include <rte_mempool.h> 12 #include <rte_errno.h> 13 #include <rte_pci.h> 14 #include <rte_bus_pci.h> 15 #include <rte_byteorder.h> 16 #ifdef RTE_BBDEV_OFFLOAD_COST 17 #include <rte_cycles.h> 18 #endif 19 20 #include <rte_bbdev.h> 21 #include <rte_bbdev_pmd.h> 22 23 #include "fpga_5gnr_fec.h" 24 #include "rte_pmd_fpga_5gnr_fec.h" 25 26 #ifdef RTE_LIBRTE_BBDEV_DEBUG 27 RTE_LOG_REGISTER(fpga_5gnr_fec_logtype, pmd.bb.fpga_5gnr_fec, DEBUG); 28 #else 29 RTE_LOG_REGISTER(fpga_5gnr_fec_logtype, pmd.bb.fpga_5gnr_fec, NOTICE); 30 #endif 31 32 #ifdef RTE_LIBRTE_BBDEV_DEBUG 33 34 /* Read Ring Control Register of FPGA 5GNR FEC device */ 35 static inline void 36 print_ring_reg_debug_info(void *mmio_base, uint32_t offset) 37 { 38 rte_bbdev_log_debug( 39 "FPGA MMIO base address @ %p | Ring Control Register @ offset = 0x%08" 40 PRIx32, mmio_base, offset); 41 rte_bbdev_log_debug( 42 "RING_BASE_ADDR = 0x%016"PRIx64, 43 fpga_reg_read_64(mmio_base, offset)); 44 rte_bbdev_log_debug( 45 "RING_HEAD_ADDR = 0x%016"PRIx64, 46 fpga_reg_read_64(mmio_base, offset + 47 FPGA_5GNR_FEC_RING_HEAD_ADDR)); 48 rte_bbdev_log_debug( 49 "RING_SIZE = 0x%04"PRIx16, 50 fpga_reg_read_16(mmio_base, offset + 51 FPGA_5GNR_FEC_RING_SIZE)); 52 rte_bbdev_log_debug( 53 "RING_MISC = 0x%02"PRIx8, 54 fpga_reg_read_8(mmio_base, offset + 55 FPGA_5GNR_FEC_RING_MISC)); 56 rte_bbdev_log_debug( 57 "RING_ENABLE = 0x%02"PRIx8, 58 fpga_reg_read_8(mmio_base, offset + 59 FPGA_5GNR_FEC_RING_ENABLE)); 60 rte_bbdev_log_debug( 61 "RING_FLUSH_QUEUE_EN = 0x%02"PRIx8, 62 fpga_reg_read_8(mmio_base, offset + 63 FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN)); 64 rte_bbdev_log_debug( 65 "RING_SHADOW_TAIL = 0x%04"PRIx16, 66 fpga_reg_read_16(mmio_base, offset + 67 FPGA_5GNR_FEC_RING_SHADOW_TAIL)); 68 rte_bbdev_log_debug( 69 "RING_HEAD_POINT = 0x%04"PRIx16, 70 fpga_reg_read_16(mmio_base, offset + 71 FPGA_5GNR_FEC_RING_HEAD_POINT)); 72 } 73 74 /* Read Static Register of FPGA 5GNR FEC device */ 75 static inline void 76 print_static_reg_debug_info(void *mmio_base) 77 { 78 uint16_t config = fpga_reg_read_16(mmio_base, 79 FPGA_5GNR_FEC_CONFIGURATION); 80 uint8_t qmap_done = fpga_reg_read_8(mmio_base, 81 FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE); 82 uint16_t lb_factor = fpga_reg_read_16(mmio_base, 83 FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR); 84 uint16_t ring_desc_len = fpga_reg_read_16(mmio_base, 85 FPGA_5GNR_FEC_RING_DESC_LEN); 86 uint16_t flr_time_out = fpga_reg_read_16(mmio_base, 87 FPGA_5GNR_FEC_FLR_TIME_OUT); 88 89 rte_bbdev_log_debug("UL.DL Weights = %u.%u", 90 ((uint8_t)config), ((uint8_t)(config >> 8))); 91 rte_bbdev_log_debug("UL.DL Load Balance = %u.%u", 92 ((uint8_t)lb_factor), ((uint8_t)(lb_factor >> 8))); 93 rte_bbdev_log_debug("Queue-PF/VF Mapping Table = %s", 94 (qmap_done > 0) ? "READY" : "NOT-READY"); 95 rte_bbdev_log_debug("Ring Descriptor Size = %u bytes", 96 ring_desc_len*FPGA_RING_DESC_LEN_UNIT_BYTES); 97 rte_bbdev_log_debug("FLR Timeout = %f usec", 98 (float)flr_time_out*FPGA_FLR_TIMEOUT_UNIT); 99 } 100 101 /* Print decode DMA Descriptor of FPGA 5GNR Decoder device */ 102 static void 103 print_dma_dec_desc_debug_info(union fpga_dma_desc *desc) 104 { 105 rte_bbdev_log_debug("DMA response desc %p\n" 106 "\t-- done(%"PRIu32") | iter(%"PRIu32") | et_pass(%"PRIu32")" 107 " | crcb_pass (%"PRIu32") | error(%"PRIu32")\n" 108 "\t-- qm_idx(%"PRIu32") | max_iter(%"PRIu32") | " 109 "bg_idx (%"PRIu32") | harqin_en(%"PRIu32") | zc(%"PRIu32")\n" 110 "\t-- hbstroe_offset(%"PRIu32") | num_null (%"PRIu32") " 111 "| irq_en(%"PRIu32")\n" 112 "\t-- ncb(%"PRIu32") | desc_idx (%"PRIu32") | " 113 "drop_crc24b(%"PRIu32") | RV (%"PRIu32")\n" 114 "\t-- crc24b_ind(%"PRIu32") | et_dis (%"PRIu32")\n" 115 "\t-- harq_input_length(%"PRIu32") | rm_e(%"PRIu32")\n" 116 "\t-- cbs_in_op(%"PRIu32") | in_add (0x%08"PRIx32"%08"PRIx32")" 117 "| out_add (0x%08"PRIx32"%08"PRIx32")", 118 desc, 119 (uint32_t)desc->dec_req.done, 120 (uint32_t)desc->dec_req.iter, 121 (uint32_t)desc->dec_req.et_pass, 122 (uint32_t)desc->dec_req.crcb_pass, 123 (uint32_t)desc->dec_req.error, 124 (uint32_t)desc->dec_req.qm_idx, 125 (uint32_t)desc->dec_req.max_iter, 126 (uint32_t)desc->dec_req.bg_idx, 127 (uint32_t)desc->dec_req.harqin_en, 128 (uint32_t)desc->dec_req.zc, 129 (uint32_t)desc->dec_req.hbstroe_offset, 130 (uint32_t)desc->dec_req.num_null, 131 (uint32_t)desc->dec_req.irq_en, 132 (uint32_t)desc->dec_req.ncb, 133 (uint32_t)desc->dec_req.desc_idx, 134 (uint32_t)desc->dec_req.drop_crc24b, 135 (uint32_t)desc->dec_req.rv, 136 (uint32_t)desc->dec_req.crc24b_ind, 137 (uint32_t)desc->dec_req.et_dis, 138 (uint32_t)desc->dec_req.harq_input_length, 139 (uint32_t)desc->dec_req.rm_e, 140 (uint32_t)desc->dec_req.cbs_in_op, 141 (uint32_t)desc->dec_req.in_addr_hi, 142 (uint32_t)desc->dec_req.in_addr_lw, 143 (uint32_t)desc->dec_req.out_addr_hi, 144 (uint32_t)desc->dec_req.out_addr_lw); 145 uint32_t *word = (uint32_t *) desc; 146 rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n" 147 "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n", 148 word[0], word[1], word[2], word[3], 149 word[4], word[5], word[6], word[7]); 150 } 151 152 /* Print decode DMA Descriptor of FPGA 5GNR encoder device */ 153 static void 154 print_dma_enc_desc_debug_info(union fpga_dma_desc *desc) 155 { 156 rte_bbdev_log_debug("DMA response desc %p\n" 157 "%"PRIu32" %"PRIu32"\n" 158 "K' %"PRIu32" E %"PRIu32" desc %"PRIu32" Z %"PRIu32"\n" 159 "BG %"PRIu32" Qm %"PRIu32" CRC %"PRIu32" IRQ %"PRIu32"\n" 160 "k0 %"PRIu32" Ncb %"PRIu32" F %"PRIu32"\n", 161 desc, 162 (uint32_t)desc->enc_req.done, 163 (uint32_t)desc->enc_req.error, 164 165 (uint32_t)desc->enc_req.k_, 166 (uint32_t)desc->enc_req.rm_e, 167 (uint32_t)desc->enc_req.desc_idx, 168 (uint32_t)desc->enc_req.zc, 169 170 (uint32_t)desc->enc_req.bg_idx, 171 (uint32_t)desc->enc_req.qm_idx, 172 (uint32_t)desc->enc_req.crc_en, 173 (uint32_t)desc->enc_req.irq_en, 174 175 (uint32_t)desc->enc_req.k0, 176 (uint32_t)desc->enc_req.ncb, 177 (uint32_t)desc->enc_req.num_null); 178 uint32_t *word = (uint32_t *) desc; 179 rte_bbdev_log_debug("%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n" 180 "%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n%08"PRIx32"\n", 181 word[0], word[1], word[2], word[3], 182 word[4], word[5], word[6], word[7]); 183 } 184 185 #endif 186 187 static int 188 fpga_setup_queues(struct rte_bbdev *dev, uint16_t num_queues, int socket_id) 189 { 190 /* Number of queues bound to a PF/VF */ 191 uint32_t hw_q_num = 0; 192 uint32_t ring_size, payload, address, q_id, offset; 193 rte_iova_t phys_addr; 194 struct fpga_ring_ctrl_reg ring_reg; 195 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private; 196 197 address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE; 198 if (!(fpga_reg_read_32(fpga_dev->mmio_base, address) & 0x1)) { 199 rte_bbdev_log(ERR, 200 "Queue-PF/VF mapping is not set! Was PF configured for device (%s) ?", 201 dev->data->name); 202 return -EPERM; 203 } 204 205 /* Clear queue registers structure */ 206 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg)); 207 208 /* Scan queue map. 209 * If a queue is valid and mapped to a calling PF/VF the read value is 210 * replaced with a queue ID and if it's not then 211 * FPGA_INVALID_HW_QUEUE_ID is returned. 212 */ 213 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 214 uint32_t hw_q_id = fpga_reg_read_32(fpga_dev->mmio_base, 215 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2)); 216 217 rte_bbdev_log_debug("%s: queue ID: %u, registry queue ID: %u", 218 dev->device->name, q_id, hw_q_id); 219 220 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) { 221 fpga_dev->q_bound_bit_map |= (1ULL << q_id); 222 /* Clear queue register of found queue */ 223 offset = FPGA_5GNR_FEC_RING_CTRL_REGS + 224 (sizeof(struct fpga_ring_ctrl_reg) * q_id); 225 fpga_ring_reg_write(fpga_dev->mmio_base, 226 offset, ring_reg); 227 ++hw_q_num; 228 } 229 } 230 if (hw_q_num == 0) { 231 rte_bbdev_log(ERR, 232 "No HW queues assigned to this device. Probably this is a VF configured for PF mode. Check device configuration!"); 233 return -ENODEV; 234 } 235 236 if (num_queues > hw_q_num) { 237 rte_bbdev_log(ERR, 238 "Not enough queues for device %s! Requested: %u, available: %u", 239 dev->device->name, num_queues, hw_q_num); 240 return -EINVAL; 241 } 242 243 ring_size = FPGA_RING_MAX_SIZE * sizeof(struct fpga_dma_dec_desc); 244 245 /* Enforce 32 byte alignment */ 246 RTE_BUILD_BUG_ON((RTE_CACHE_LINE_SIZE % 32) != 0); 247 248 /* Allocate memory for SW descriptor rings */ 249 fpga_dev->sw_rings = rte_zmalloc_socket(dev->device->driver->name, 250 num_queues * ring_size, RTE_CACHE_LINE_SIZE, 251 socket_id); 252 if (fpga_dev->sw_rings == NULL) { 253 rte_bbdev_log(ERR, 254 "Failed to allocate memory for %s:%u sw_rings", 255 dev->device->driver->name, dev->data->dev_id); 256 return -ENOMEM; 257 } 258 259 fpga_dev->sw_rings_phys = rte_malloc_virt2iova(fpga_dev->sw_rings); 260 fpga_dev->sw_ring_size = ring_size; 261 fpga_dev->sw_ring_max_depth = FPGA_RING_MAX_SIZE; 262 263 /* Allocate memory for ring flush status */ 264 fpga_dev->flush_queue_status = rte_zmalloc_socket(NULL, 265 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, socket_id); 266 if (fpga_dev->flush_queue_status == NULL) { 267 rte_bbdev_log(ERR, 268 "Failed to allocate memory for %s:%u flush_queue_status", 269 dev->device->driver->name, dev->data->dev_id); 270 return -ENOMEM; 271 } 272 273 /* Set the flush status address registers */ 274 phys_addr = rte_malloc_virt2iova(fpga_dev->flush_queue_status); 275 276 address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_LW; 277 payload = (uint32_t)(phys_addr); 278 fpga_reg_write_32(fpga_dev->mmio_base, address, payload); 279 280 address = FPGA_5GNR_FEC_VFQ_FLUSH_STATUS_HI; 281 payload = (uint32_t)(phys_addr >> 32); 282 fpga_reg_write_32(fpga_dev->mmio_base, address, payload); 283 284 return 0; 285 } 286 287 static int 288 fpga_dev_close(struct rte_bbdev *dev) 289 { 290 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private; 291 292 rte_free(fpga_dev->sw_rings); 293 rte_free(fpga_dev->flush_queue_status); 294 295 return 0; 296 } 297 298 static void 299 fpga_dev_info_get(struct rte_bbdev *dev, 300 struct rte_bbdev_driver_info *dev_info) 301 { 302 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 303 uint32_t q_id = 0; 304 305 static const struct rte_bbdev_op_cap bbdev_capabilities[] = { 306 { 307 .type = RTE_BBDEV_OP_LDPC_ENC, 308 .cap.ldpc_enc = { 309 .capability_flags = 310 RTE_BBDEV_LDPC_RATE_MATCH | 311 RTE_BBDEV_LDPC_ENC_INTERRUPTS | 312 RTE_BBDEV_LDPC_CRC_24B_ATTACH, 313 .num_buffers_src = 314 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS, 315 .num_buffers_dst = 316 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS, 317 } 318 }, 319 { 320 .type = RTE_BBDEV_OP_LDPC_DEC, 321 .cap.ldpc_dec = { 322 .capability_flags = 323 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK | 324 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP | 325 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE | 326 RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE | 327 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE | 328 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE | 329 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE | 330 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK | 331 RTE_BBDEV_LDPC_DEC_INTERRUPTS | 332 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_FILLERS, 333 .llr_size = 6, 334 .llr_decimals = 2, 335 .num_buffers_src = 336 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS, 337 .num_buffers_hard_out = 338 RTE_BBDEV_LDPC_MAX_CODE_BLOCKS, 339 .num_buffers_soft_out = 0, 340 } 341 }, 342 RTE_BBDEV_END_OF_CAPABILITIES_LIST() 343 }; 344 345 /* Check the HARQ DDR size available */ 346 uint8_t timeout_counter = 0; 347 uint32_t harq_buf_ready = fpga_reg_read_32(d->mmio_base, 348 FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS); 349 while (harq_buf_ready != 1) { 350 usleep(FPGA_TIMEOUT_CHECK_INTERVAL); 351 timeout_counter++; 352 harq_buf_ready = fpga_reg_read_32(d->mmio_base, 353 FPGA_5GNR_FEC_HARQ_BUF_SIZE_RDY_REGS); 354 if (timeout_counter > FPGA_HARQ_RDY_TIMEOUT) { 355 rte_bbdev_log(ERR, "HARQ Buffer not ready %d", 356 harq_buf_ready); 357 harq_buf_ready = 1; 358 } 359 } 360 uint32_t harq_buf_size = fpga_reg_read_32(d->mmio_base, 361 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS); 362 363 static struct rte_bbdev_queue_conf default_queue_conf; 364 default_queue_conf.socket = dev->data->socket_id; 365 default_queue_conf.queue_size = FPGA_RING_MAX_SIZE; 366 367 dev_info->driver_name = dev->device->driver->name; 368 dev_info->queue_size_lim = FPGA_RING_MAX_SIZE; 369 dev_info->hardware_accelerated = true; 370 dev_info->min_alignment = 64; 371 dev_info->harq_buffer_size = (harq_buf_size >> 10) + 1; 372 dev_info->default_queue_conf = default_queue_conf; 373 dev_info->capabilities = bbdev_capabilities; 374 dev_info->cpu_flag_reqs = NULL; 375 376 /* Calculates number of queues assigned to device */ 377 dev_info->max_num_queues = 0; 378 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 379 uint32_t hw_q_id = fpga_reg_read_32(d->mmio_base, 380 FPGA_5GNR_FEC_QUEUE_MAP + (q_id << 2)); 381 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) 382 dev_info->max_num_queues++; 383 } 384 } 385 386 /** 387 * Find index of queue bound to current PF/VF which is unassigned. Return -1 388 * when there is no available queue 389 */ 390 static inline int 391 fpga_find_free_queue_idx(struct rte_bbdev *dev, 392 const struct rte_bbdev_queue_conf *conf) 393 { 394 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 395 uint64_t q_idx; 396 uint8_t i = 0; 397 uint8_t range = FPGA_TOTAL_NUM_QUEUES >> 1; 398 399 if (conf->op_type == RTE_BBDEV_OP_LDPC_ENC) { 400 i = FPGA_NUM_DL_QUEUES; 401 range = FPGA_TOTAL_NUM_QUEUES; 402 } 403 404 for (; i < range; ++i) { 405 q_idx = 1ULL << i; 406 /* Check if index of queue is bound to current PF/VF */ 407 if (d->q_bound_bit_map & q_idx) 408 /* Check if found queue was not already assigned */ 409 if (!(d->q_assigned_bit_map & q_idx)) { 410 d->q_assigned_bit_map |= q_idx; 411 return i; 412 } 413 } 414 415 rte_bbdev_log(INFO, "Failed to find free queue on %s", dev->data->name); 416 417 return -1; 418 } 419 420 static int 421 fpga_queue_setup(struct rte_bbdev *dev, uint16_t queue_id, 422 const struct rte_bbdev_queue_conf *conf) 423 { 424 uint32_t address, ring_offset; 425 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 426 struct fpga_queue *q; 427 int8_t q_idx; 428 429 /* Check if there is a free queue to assign */ 430 q_idx = fpga_find_free_queue_idx(dev, conf); 431 if (q_idx == -1) 432 return -1; 433 434 /* Allocate the queue data structure. */ 435 q = rte_zmalloc_socket(dev->device->driver->name, sizeof(*q), 436 RTE_CACHE_LINE_SIZE, conf->socket); 437 if (q == NULL) { 438 /* Mark queue as un-assigned */ 439 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 440 rte_bbdev_log(ERR, "Failed to allocate queue memory"); 441 return -ENOMEM; 442 } 443 444 q->d = d; 445 q->q_idx = q_idx; 446 447 /* Set ring_base_addr */ 448 q->ring_addr = RTE_PTR_ADD(d->sw_rings, (d->sw_ring_size * queue_id)); 449 q->ring_ctrl_reg.ring_base_addr = d->sw_rings_phys + 450 (d->sw_ring_size * queue_id); 451 452 /* Allocate memory for Completion Head variable*/ 453 q->ring_head_addr = rte_zmalloc_socket(dev->device->driver->name, 454 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, conf->socket); 455 if (q->ring_head_addr == NULL) { 456 /* Mark queue as un-assigned */ 457 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 458 rte_free(q); 459 rte_bbdev_log(ERR, 460 "Failed to allocate memory for %s:%u completion_head", 461 dev->device->driver->name, dev->data->dev_id); 462 return -ENOMEM; 463 } 464 /* Set ring_head_addr */ 465 q->ring_ctrl_reg.ring_head_addr = 466 rte_malloc_virt2iova(q->ring_head_addr); 467 468 /* Clear shadow_completion_head */ 469 q->shadow_completion_head = 0; 470 471 /* Set ring_size */ 472 if (conf->queue_size > FPGA_RING_MAX_SIZE) { 473 /* Mark queue as un-assigned */ 474 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 475 rte_free(q->ring_head_addr); 476 rte_free(q); 477 rte_bbdev_log(ERR, 478 "Size of queue is too big %d (MAX: %d ) for %s:%u", 479 conf->queue_size, FPGA_RING_MAX_SIZE, 480 dev->device->driver->name, dev->data->dev_id); 481 return -EINVAL; 482 } 483 q->ring_ctrl_reg.ring_size = conf->queue_size; 484 485 /* Set Miscellaneous FPGA register*/ 486 /* Max iteration number for TTI mitigation - todo */ 487 q->ring_ctrl_reg.max_ul_dec = 0; 488 /* Enable max iteration number for TTI - todo */ 489 q->ring_ctrl_reg.max_ul_dec_en = 0; 490 491 /* Enable the ring */ 492 q->ring_ctrl_reg.enable = 1; 493 494 /* Set FPGA head_point and tail registers */ 495 q->ring_ctrl_reg.head_point = q->tail = 0; 496 497 /* Set FPGA shadow_tail register */ 498 q->ring_ctrl_reg.shadow_tail = q->tail; 499 500 /* Calculates the ring offset for found queue */ 501 ring_offset = FPGA_5GNR_FEC_RING_CTRL_REGS + 502 (sizeof(struct fpga_ring_ctrl_reg) * q_idx); 503 504 /* Set FPGA Ring Control Registers */ 505 fpga_ring_reg_write(d->mmio_base, ring_offset, q->ring_ctrl_reg); 506 507 /* Store MMIO register of shadow_tail */ 508 address = ring_offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL; 509 q->shadow_tail_addr = RTE_PTR_ADD(d->mmio_base, address); 510 511 q->head_free_desc = q->tail; 512 513 /* Set wrap mask */ 514 q->sw_ring_wrap_mask = conf->queue_size - 1; 515 516 rte_bbdev_log_debug("Setup dev%u q%u: queue_idx=%u", 517 dev->data->dev_id, queue_id, q->q_idx); 518 519 dev->data->queues[queue_id].queue_private = q; 520 521 rte_bbdev_log_debug("BBDEV queue[%d] set up for FPGA queue[%d]", 522 queue_id, q_idx); 523 524 #ifdef RTE_LIBRTE_BBDEV_DEBUG 525 /* Read FPGA Ring Control Registers after configuration*/ 526 print_ring_reg_debug_info(d->mmio_base, ring_offset); 527 #endif 528 return 0; 529 } 530 531 static int 532 fpga_queue_release(struct rte_bbdev *dev, uint16_t queue_id) 533 { 534 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 535 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 536 struct fpga_ring_ctrl_reg ring_reg; 537 uint32_t offset; 538 539 rte_bbdev_log_debug("FPGA Queue[%d] released", queue_id); 540 541 if (q != NULL) { 542 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg)); 543 offset = FPGA_5GNR_FEC_RING_CTRL_REGS + 544 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 545 /* Disable queue */ 546 fpga_reg_write_8(d->mmio_base, 547 offset + FPGA_5GNR_FEC_RING_ENABLE, 0x00); 548 /* Clear queue registers */ 549 fpga_ring_reg_write(d->mmio_base, offset, ring_reg); 550 551 /* Mark the Queue as un-assigned */ 552 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q->q_idx)); 553 rte_free(q->ring_head_addr); 554 rte_free(q); 555 dev->data->queues[queue_id].queue_private = NULL; 556 } 557 558 return 0; 559 } 560 561 /* Function starts a device queue. */ 562 static int 563 fpga_queue_start(struct rte_bbdev *dev, uint16_t queue_id) 564 { 565 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 566 #ifdef RTE_LIBRTE_BBDEV_DEBUG 567 if (d == NULL) { 568 rte_bbdev_log(ERR, "Invalid device pointer"); 569 return -1; 570 } 571 #endif 572 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 573 uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS + 574 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 575 uint8_t enable = 0x01; 576 uint16_t zero = 0x0000; 577 578 /* Clear queue head and tail variables */ 579 q->tail = q->head_free_desc = 0; 580 581 /* Clear FPGA head_point and tail registers */ 582 fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_HEAD_POINT, 583 zero); 584 fpga_reg_write_16(d->mmio_base, offset + FPGA_5GNR_FEC_RING_SHADOW_TAIL, 585 zero); 586 587 /* Enable queue */ 588 fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE, 589 enable); 590 591 rte_bbdev_log_debug("FPGA Queue[%d] started", queue_id); 592 return 0; 593 } 594 595 /* Function stops a device queue. */ 596 static int 597 fpga_queue_stop(struct rte_bbdev *dev, uint16_t queue_id) 598 { 599 struct fpga_5gnr_fec_device *d = dev->data->dev_private; 600 #ifdef RTE_LIBRTE_BBDEV_DEBUG 601 if (d == NULL) { 602 rte_bbdev_log(ERR, "Invalid device pointer"); 603 return -1; 604 } 605 #endif 606 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 607 uint32_t offset = FPGA_5GNR_FEC_RING_CTRL_REGS + 608 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 609 uint8_t payload = 0x01; 610 uint8_t counter = 0; 611 uint8_t timeout = FPGA_QUEUE_FLUSH_TIMEOUT_US / 612 FPGA_TIMEOUT_CHECK_INTERVAL; 613 614 /* Set flush_queue_en bit to trigger queue flushing */ 615 fpga_reg_write_8(d->mmio_base, 616 offset + FPGA_5GNR_FEC_RING_FLUSH_QUEUE_EN, payload); 617 618 /** Check if queue flush is completed. 619 * FPGA will update the completion flag after queue flushing is 620 * completed. If completion flag is not updated within 1ms it is 621 * considered as a failure. 622 */ 623 while (!(*((volatile uint8_t *)d->flush_queue_status + q->q_idx) 624 & payload)) { 625 if (counter > timeout) { 626 rte_bbdev_log(ERR, "FPGA Queue Flush failed for queue %d", 627 queue_id); 628 return -1; 629 } 630 usleep(FPGA_TIMEOUT_CHECK_INTERVAL); 631 counter++; 632 } 633 634 /* Disable queue */ 635 payload = 0x00; 636 fpga_reg_write_8(d->mmio_base, offset + FPGA_5GNR_FEC_RING_ENABLE, 637 payload); 638 639 rte_bbdev_log_debug("FPGA Queue[%d] stopped", queue_id); 640 return 0; 641 } 642 643 static inline uint16_t 644 get_queue_id(struct rte_bbdev_data *data, uint8_t q_idx) 645 { 646 uint16_t queue_id; 647 648 for (queue_id = 0; queue_id < data->num_queues; ++queue_id) { 649 struct fpga_queue *q = data->queues[queue_id].queue_private; 650 if (q != NULL && q->q_idx == q_idx) 651 return queue_id; 652 } 653 654 return -1; 655 } 656 657 /* Interrupt handler triggered by FPGA dev for handling specific interrupt */ 658 static void 659 fpga_dev_interrupt_handler(void *cb_arg) 660 { 661 struct rte_bbdev *dev = cb_arg; 662 struct fpga_5gnr_fec_device *fpga_dev = dev->data->dev_private; 663 struct fpga_queue *q; 664 uint64_t ring_head; 665 uint64_t q_idx; 666 uint16_t queue_id; 667 uint8_t i; 668 669 /* Scan queue assigned to this device */ 670 for (i = 0; i < FPGA_TOTAL_NUM_QUEUES; ++i) { 671 q_idx = 1ULL << i; 672 if (fpga_dev->q_bound_bit_map & q_idx) { 673 queue_id = get_queue_id(dev->data, i); 674 if (queue_id == (uint16_t) -1) 675 continue; 676 677 /* Check if completion head was changed */ 678 q = dev->data->queues[queue_id].queue_private; 679 ring_head = *q->ring_head_addr; 680 if (q->shadow_completion_head != ring_head && 681 q->irq_enable == 1) { 682 q->shadow_completion_head = ring_head; 683 rte_bbdev_pmd_callback_process( 684 dev, 685 RTE_BBDEV_EVENT_DEQUEUE, 686 &queue_id); 687 } 688 } 689 } 690 } 691 692 static int 693 fpga_queue_intr_enable(struct rte_bbdev *dev, uint16_t queue_id) 694 { 695 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 696 697 if (!rte_intr_cap_multiple(dev->intr_handle)) 698 return -ENOTSUP; 699 700 q->irq_enable = 1; 701 702 return 0; 703 } 704 705 static int 706 fpga_queue_intr_disable(struct rte_bbdev *dev, uint16_t queue_id) 707 { 708 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 709 q->irq_enable = 0; 710 711 return 0; 712 } 713 714 static int 715 fpga_intr_enable(struct rte_bbdev *dev) 716 { 717 int ret; 718 uint8_t i; 719 720 if (!rte_intr_cap_multiple(dev->intr_handle)) { 721 rte_bbdev_log(ERR, "Multiple intr vector is not supported by FPGA (%s)", 722 dev->data->name); 723 return -ENOTSUP; 724 } 725 726 /* Create event file descriptors for each of 64 queue. Event fds will be 727 * mapped to FPGA IRQs in rte_intr_enable(). This is a 1:1 mapping where 728 * the IRQ number is a direct translation to the queue number. 729 * 730 * 63 (FPGA_NUM_INTR_VEC) event fds are created as rte_intr_enable() 731 * mapped the first IRQ to already created interrupt event file 732 * descriptor (intr_handle->fd). 733 */ 734 if (rte_intr_efd_enable(dev->intr_handle, FPGA_NUM_INTR_VEC)) { 735 rte_bbdev_log(ERR, "Failed to create fds for %u queues", 736 dev->data->num_queues); 737 return -1; 738 } 739 740 /* TODO Each event file descriptor is overwritten by interrupt event 741 * file descriptor. That descriptor is added to epoll observed list. 742 * It ensures that callback function assigned to that descriptor will 743 * invoked when any FPGA queue issues interrupt. 744 */ 745 for (i = 0; i < FPGA_NUM_INTR_VEC; ++i) 746 dev->intr_handle->efds[i] = dev->intr_handle->fd; 747 748 if (!dev->intr_handle->intr_vec) { 749 dev->intr_handle->intr_vec = rte_zmalloc("intr_vec", 750 dev->data->num_queues * sizeof(int), 0); 751 if (!dev->intr_handle->intr_vec) { 752 rte_bbdev_log(ERR, "Failed to allocate %u vectors", 753 dev->data->num_queues); 754 return -ENOMEM; 755 } 756 } 757 758 ret = rte_intr_enable(dev->intr_handle); 759 if (ret < 0) { 760 rte_bbdev_log(ERR, 761 "Couldn't enable interrupts for device: %s", 762 dev->data->name); 763 return ret; 764 } 765 766 ret = rte_intr_callback_register(dev->intr_handle, 767 fpga_dev_interrupt_handler, dev); 768 if (ret < 0) { 769 rte_bbdev_log(ERR, 770 "Couldn't register interrupt callback for device: %s", 771 dev->data->name); 772 return ret; 773 } 774 775 return 0; 776 } 777 778 static const struct rte_bbdev_ops fpga_ops = { 779 .setup_queues = fpga_setup_queues, 780 .intr_enable = fpga_intr_enable, 781 .close = fpga_dev_close, 782 .info_get = fpga_dev_info_get, 783 .queue_setup = fpga_queue_setup, 784 .queue_stop = fpga_queue_stop, 785 .queue_start = fpga_queue_start, 786 .queue_release = fpga_queue_release, 787 .queue_intr_enable = fpga_queue_intr_enable, 788 .queue_intr_disable = fpga_queue_intr_disable 789 }; 790 791 static inline void 792 fpga_dma_enqueue(struct fpga_queue *q, uint16_t num_desc, 793 struct rte_bbdev_stats *queue_stats) 794 { 795 #ifdef RTE_BBDEV_OFFLOAD_COST 796 uint64_t start_time = 0; 797 queue_stats->acc_offload_cycles = 0; 798 #else 799 RTE_SET_USED(queue_stats); 800 #endif 801 802 /* Update tail and shadow_tail register */ 803 q->tail = (q->tail + num_desc) & q->sw_ring_wrap_mask; 804 805 rte_wmb(); 806 807 #ifdef RTE_BBDEV_OFFLOAD_COST 808 /* Start time measurement for enqueue function offload. */ 809 start_time = rte_rdtsc_precise(); 810 #endif 811 mmio_write_16(q->shadow_tail_addr, q->tail); 812 813 #ifdef RTE_BBDEV_OFFLOAD_COST 814 rte_wmb(); 815 queue_stats->acc_offload_cycles += rte_rdtsc_precise() - start_time; 816 #endif 817 } 818 819 /* Read flag value 0/1/ from bitmap */ 820 static inline bool 821 check_bit(uint32_t bitmap, uint32_t bitmask) 822 { 823 return bitmap & bitmask; 824 } 825 826 /* Print an error if a descriptor error has occurred. 827 * Return 0 on success, 1 on failure 828 */ 829 static inline int 830 check_desc_error(uint32_t error_code) { 831 switch (error_code) { 832 case DESC_ERR_NO_ERR: 833 return 0; 834 case DESC_ERR_K_P_OUT_OF_RANGE: 835 rte_bbdev_log(ERR, "Encode block size K' is out of range"); 836 break; 837 case DESC_ERR_Z_C_NOT_LEGAL: 838 rte_bbdev_log(ERR, "Zc is illegal"); 839 break; 840 case DESC_ERR_DESC_OFFSET_ERR: 841 rte_bbdev_log(ERR, 842 "Queue offset does not meet the expectation in the FPGA" 843 ); 844 break; 845 case DESC_ERR_DESC_READ_FAIL: 846 rte_bbdev_log(ERR, "Unsuccessful completion for descriptor read"); 847 break; 848 case DESC_ERR_DESC_READ_TIMEOUT: 849 rte_bbdev_log(ERR, "Descriptor read time-out"); 850 break; 851 case DESC_ERR_DESC_READ_TLP_POISONED: 852 rte_bbdev_log(ERR, "Descriptor read TLP poisoned"); 853 break; 854 case DESC_ERR_CB_READ_FAIL: 855 rte_bbdev_log(ERR, "Unsuccessful completion for code block"); 856 break; 857 case DESC_ERR_CB_READ_TIMEOUT: 858 rte_bbdev_log(ERR, "Code block read time-out"); 859 break; 860 case DESC_ERR_CB_READ_TLP_POISONED: 861 rte_bbdev_log(ERR, "Code block read TLP poisoned"); 862 break; 863 case DESC_ERR_HBSTORE_ERR: 864 rte_bbdev_log(ERR, "Hbstroe exceeds HARQ buffer size."); 865 break; 866 default: 867 rte_bbdev_log(ERR, "Descriptor error unknown error code %u", 868 error_code); 869 break; 870 } 871 return 1; 872 } 873 874 /* Compute value of k0. 875 * Based on 3GPP 38.212 Table 5.4.2.1-2 876 * Starting position of different redundancy versions, k0 877 */ 878 static inline uint16_t 879 get_k0(uint16_t n_cb, uint16_t z_c, uint8_t bg, uint8_t rv_index) 880 { 881 if (rv_index == 0) 882 return 0; 883 uint16_t n = (bg == 1 ? N_ZC_1 : N_ZC_2) * z_c; 884 if (n_cb == n) { 885 if (rv_index == 1) 886 return (bg == 1 ? K0_1_1 : K0_1_2) * z_c; 887 else if (rv_index == 2) 888 return (bg == 1 ? K0_2_1 : K0_2_2) * z_c; 889 else 890 return (bg == 1 ? K0_3_1 : K0_3_2) * z_c; 891 } 892 /* LBRM case - includes a division by N */ 893 if (rv_index == 1) 894 return (((bg == 1 ? K0_1_1 : K0_1_2) * n_cb) 895 / n) * z_c; 896 else if (rv_index == 2) 897 return (((bg == 1 ? K0_2_1 : K0_2_2) * n_cb) 898 / n) * z_c; 899 else 900 return (((bg == 1 ? K0_3_1 : K0_3_2) * n_cb) 901 / n) * z_c; 902 } 903 904 /** 905 * Set DMA descriptor for encode operation (1 Code Block) 906 * 907 * @param op 908 * Pointer to a single encode operation. 909 * @param desc 910 * Pointer to DMA descriptor. 911 * @param input 912 * Pointer to pointer to input data which will be decoded. 913 * @param e 914 * E value (length of output in bits). 915 * @param ncb 916 * Ncb value (size of the soft buffer). 917 * @param out_length 918 * Length of output buffer 919 * @param in_offset 920 * Input offset in rte_mbuf structure. It is used for calculating the point 921 * where data is starting. 922 * @param out_offset 923 * Output offset in rte_mbuf structure. It is used for calculating the point 924 * where hard output data will be stored. 925 * @param cbs_in_op 926 * Number of CBs contained in one operation. 927 */ 928 static inline int 929 fpga_dma_desc_te_fill(struct rte_bbdev_enc_op *op, 930 struct fpga_dma_enc_desc *desc, struct rte_mbuf *input, 931 struct rte_mbuf *output, uint16_t k_, uint16_t e, 932 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset, 933 uint8_t cbs_in_op) 934 { 935 /* reset */ 936 desc->done = 0; 937 desc->error = 0; 938 desc->k_ = k_; 939 desc->rm_e = e; 940 desc->desc_idx = desc_offset; 941 desc->zc = op->ldpc_enc.z_c; 942 desc->bg_idx = op->ldpc_enc.basegraph - 1; 943 desc->qm_idx = op->ldpc_enc.q_m / 2; 944 desc->crc_en = check_bit(op->ldpc_enc.op_flags, 945 RTE_BBDEV_LDPC_CRC_24B_ATTACH); 946 desc->irq_en = 0; 947 desc->k0 = get_k0(op->ldpc_enc.n_cb, op->ldpc_enc.z_c, 948 op->ldpc_enc.basegraph, op->ldpc_enc.rv_index); 949 desc->ncb = op->ldpc_enc.n_cb; 950 desc->num_null = op->ldpc_enc.n_filler; 951 /* Set inbound data buffer address */ 952 desc->in_addr_hi = (uint32_t)( 953 rte_pktmbuf_iova_offset(input, in_offset) >> 32); 954 desc->in_addr_lw = (uint32_t)( 955 rte_pktmbuf_iova_offset(input, in_offset)); 956 957 desc->out_addr_hi = (uint32_t)( 958 rte_pktmbuf_iova_offset(output, out_offset) >> 32); 959 desc->out_addr_lw = (uint32_t)( 960 rte_pktmbuf_iova_offset(output, out_offset)); 961 /* Save software context needed for dequeue */ 962 desc->op_addr = op; 963 /* Set total number of CBs in an op */ 964 desc->cbs_in_op = cbs_in_op; 965 return 0; 966 } 967 968 /** 969 * Set DMA descriptor for decode operation (1 Code Block) 970 * 971 * @param op 972 * Pointer to a single encode operation. 973 * @param desc 974 * Pointer to DMA descriptor. 975 * @param input 976 * Pointer to pointer to input data which will be decoded. 977 * @param in_offset 978 * Input offset in rte_mbuf structure. It is used for calculating the point 979 * where data is starting. 980 * @param out_offset 981 * Output offset in rte_mbuf structure. It is used for calculating the point 982 * where hard output data will be stored. 983 * @param cbs_in_op 984 * Number of CBs contained in one operation. 985 */ 986 static inline int 987 fpga_dma_desc_ld_fill(struct rte_bbdev_dec_op *op, 988 struct fpga_dma_dec_desc *desc, 989 struct rte_mbuf *input, struct rte_mbuf *output, 990 uint16_t harq_in_length, 991 uint32_t in_offset, uint32_t out_offset, 992 uint32_t harq_offset, 993 uint16_t desc_offset, 994 uint8_t cbs_in_op) 995 { 996 /* reset */ 997 desc->done = 0; 998 desc->error = 0; 999 /* Set inbound data buffer address */ 1000 desc->in_addr_hi = (uint32_t)( 1001 rte_pktmbuf_iova_offset(input, in_offset) >> 32); 1002 desc->in_addr_lw = (uint32_t)( 1003 rte_pktmbuf_iova_offset(input, in_offset)); 1004 desc->rm_e = op->ldpc_dec.cb_params.e; 1005 desc->harq_input_length = harq_in_length; 1006 desc->et_dis = !check_bit(op->ldpc_dec.op_flags, 1007 RTE_BBDEV_LDPC_ITERATION_STOP_ENABLE); 1008 desc->rv = op->ldpc_dec.rv_index; 1009 desc->crc24b_ind = check_bit(op->ldpc_dec.op_flags, 1010 RTE_BBDEV_LDPC_CRC_TYPE_24B_CHECK); 1011 desc->drop_crc24b = check_bit(op->ldpc_dec.op_flags, 1012 RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP); 1013 desc->desc_idx = desc_offset; 1014 desc->ncb = op->ldpc_dec.n_cb; 1015 desc->num_null = op->ldpc_dec.n_filler; 1016 desc->hbstroe_offset = harq_offset >> 10; 1017 desc->zc = op->ldpc_dec.z_c; 1018 desc->harqin_en = check_bit(op->ldpc_dec.op_flags, 1019 RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE); 1020 desc->bg_idx = op->ldpc_dec.basegraph - 1; 1021 desc->max_iter = op->ldpc_dec.iter_max; 1022 desc->qm_idx = op->ldpc_dec.q_m / 2; 1023 desc->out_addr_hi = (uint32_t)( 1024 rte_pktmbuf_iova_offset(output, out_offset) >> 32); 1025 desc->out_addr_lw = (uint32_t)( 1026 rte_pktmbuf_iova_offset(output, out_offset)); 1027 /* Save software context needed for dequeue */ 1028 desc->op_addr = op; 1029 /* Set total number of CBs in an op */ 1030 desc->cbs_in_op = cbs_in_op; 1031 1032 return 0; 1033 } 1034 1035 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1036 /* Validates LDPC encoder parameters */ 1037 static int 1038 validate_enc_op(struct rte_bbdev_enc_op *op __rte_unused) 1039 { 1040 struct rte_bbdev_op_ldpc_enc *ldpc_enc = &op->ldpc_enc; 1041 struct rte_bbdev_op_enc_ldpc_cb_params *cb = NULL; 1042 struct rte_bbdev_op_enc_ldpc_tb_params *tb = NULL; 1043 1044 1045 if (ldpc_enc->input.length > 1046 RTE_BBDEV_LDPC_MAX_CB_SIZE >> 3) { 1047 rte_bbdev_log(ERR, "CB size (%u) is too big, max: %d", 1048 ldpc_enc->input.length, 1049 RTE_BBDEV_LDPC_MAX_CB_SIZE); 1050 return -1; 1051 } 1052 1053 if (op->mempool == NULL) { 1054 rte_bbdev_log(ERR, "Invalid mempool pointer"); 1055 return -1; 1056 } 1057 if (ldpc_enc->input.data == NULL) { 1058 rte_bbdev_log(ERR, "Invalid input pointer"); 1059 return -1; 1060 } 1061 if (ldpc_enc->output.data == NULL) { 1062 rte_bbdev_log(ERR, "Invalid output pointer"); 1063 return -1; 1064 } 1065 if ((ldpc_enc->basegraph > 2) || (ldpc_enc->basegraph == 0)) { 1066 rte_bbdev_log(ERR, 1067 "basegraph (%u) is out of range 1 <= value <= 2", 1068 ldpc_enc->basegraph); 1069 return -1; 1070 } 1071 if (ldpc_enc->code_block_mode > 1) { 1072 rte_bbdev_log(ERR, 1073 "code_block_mode (%u) is out of range 0:Tb 1:CB", 1074 ldpc_enc->code_block_mode); 1075 return -1; 1076 } 1077 1078 if (ldpc_enc->code_block_mode == 0) { 1079 tb = &ldpc_enc->tb_params; 1080 if (tb->c == 0) { 1081 rte_bbdev_log(ERR, 1082 "c (%u) is out of range 1 <= value <= %u", 1083 tb->c, RTE_BBDEV_LDPC_MAX_CODE_BLOCKS); 1084 return -1; 1085 } 1086 if (tb->cab > tb->c) { 1087 rte_bbdev_log(ERR, 1088 "cab (%u) is greater than c (%u)", 1089 tb->cab, tb->c); 1090 return -1; 1091 } 1092 if ((tb->ea < RTE_BBDEV_LDPC_MIN_CB_SIZE) 1093 && tb->r < tb->cab) { 1094 rte_bbdev_log(ERR, 1095 "ea (%u) is less than %u or it is not even", 1096 tb->ea, RTE_BBDEV_LDPC_MIN_CB_SIZE); 1097 return -1; 1098 } 1099 if ((tb->eb < RTE_BBDEV_LDPC_MIN_CB_SIZE) 1100 && tb->c > tb->cab) { 1101 rte_bbdev_log(ERR, 1102 "eb (%u) is less than %u", 1103 tb->eb, RTE_BBDEV_LDPC_MIN_CB_SIZE); 1104 return -1; 1105 } 1106 if (tb->r > (tb->c - 1)) { 1107 rte_bbdev_log(ERR, 1108 "r (%u) is greater than c - 1 (%u)", 1109 tb->r, tb->c - 1); 1110 return -1; 1111 } 1112 } else { 1113 cb = &ldpc_enc->cb_params; 1114 if (cb->e < RTE_BBDEV_LDPC_MIN_CB_SIZE) { 1115 rte_bbdev_log(ERR, 1116 "e (%u) is less than %u or it is not even", 1117 cb->e, RTE_BBDEV_LDPC_MIN_CB_SIZE); 1118 return -1; 1119 } 1120 } 1121 return 0; 1122 } 1123 #endif 1124 1125 static inline char * 1126 mbuf_append(struct rte_mbuf *m_head, struct rte_mbuf *m, uint16_t len) 1127 { 1128 if (unlikely(len > rte_pktmbuf_tailroom(m))) 1129 return NULL; 1130 1131 char *tail = (char *)m->buf_addr + m->data_off + m->data_len; 1132 m->data_len = (uint16_t)(m->data_len + len); 1133 m_head->pkt_len = (m_head->pkt_len + len); 1134 return tail; 1135 } 1136 1137 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1138 /* Validates LDPC decoder parameters */ 1139 static int 1140 validate_dec_op(struct rte_bbdev_dec_op *op __rte_unused) 1141 { 1142 struct rte_bbdev_op_ldpc_dec *ldpc_dec = &op->ldpc_dec; 1143 struct rte_bbdev_op_dec_ldpc_cb_params *cb = NULL; 1144 struct rte_bbdev_op_dec_ldpc_tb_params *tb = NULL; 1145 1146 if (op->mempool == NULL) { 1147 rte_bbdev_log(ERR, "Invalid mempool pointer"); 1148 return -1; 1149 } 1150 if (ldpc_dec->rv_index > 3) { 1151 rte_bbdev_log(ERR, 1152 "rv_index (%u) is out of range 0 <= value <= 3", 1153 ldpc_dec->rv_index); 1154 return -1; 1155 } 1156 1157 if (ldpc_dec->iter_max == 0) { 1158 rte_bbdev_log(ERR, 1159 "iter_max (%u) is equal to 0", 1160 ldpc_dec->iter_max); 1161 return -1; 1162 } 1163 1164 if (ldpc_dec->code_block_mode > 1) { 1165 rte_bbdev_log(ERR, 1166 "code_block_mode (%u) is out of range 0 <= value <= 1", 1167 ldpc_dec->code_block_mode); 1168 return -1; 1169 } 1170 1171 if (ldpc_dec->code_block_mode == 0) { 1172 tb = &ldpc_dec->tb_params; 1173 if (tb->c < 1) { 1174 rte_bbdev_log(ERR, 1175 "c (%u) is out of range 1 <= value <= %u", 1176 tb->c, RTE_BBDEV_LDPC_MAX_CODE_BLOCKS); 1177 return -1; 1178 } 1179 if (tb->cab > tb->c) { 1180 rte_bbdev_log(ERR, 1181 "cab (%u) is greater than c (%u)", 1182 tb->cab, tb->c); 1183 return -1; 1184 } 1185 } else { 1186 cb = &ldpc_dec->cb_params; 1187 if (cb->e < RTE_BBDEV_LDPC_MIN_CB_SIZE) { 1188 rte_bbdev_log(ERR, 1189 "e (%u) is out of range %u <= value <= %u", 1190 cb->e, RTE_BBDEV_LDPC_MIN_CB_SIZE, 1191 RTE_BBDEV_LDPC_MAX_CB_SIZE); 1192 return -1; 1193 } 1194 } 1195 1196 return 0; 1197 } 1198 #endif 1199 1200 static inline int 1201 fpga_harq_write_loopback(struct fpga_5gnr_fec_device *fpga_dev, 1202 struct rte_mbuf *harq_input, uint16_t harq_in_length, 1203 uint32_t harq_in_offset, uint32_t harq_out_offset) 1204 { 1205 uint32_t out_offset = harq_out_offset; 1206 uint32_t in_offset = harq_in_offset; 1207 uint32_t left_length = harq_in_length; 1208 uint32_t reg_32, increment = 0; 1209 uint64_t *input = NULL; 1210 uint32_t last_transaction = left_length 1211 % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; 1212 uint64_t last_word; 1213 1214 if (last_transaction > 0) 1215 left_length -= last_transaction; 1216 1217 /* 1218 * Get HARQ buffer size for each VF/PF: When 0x00, there is no 1219 * available DDR space for the corresponding VF/PF. 1220 */ 1221 reg_32 = fpga_reg_read_32(fpga_dev->mmio_base, 1222 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS); 1223 if (reg_32 < harq_in_length) { 1224 left_length = reg_32; 1225 rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n"); 1226 } 1227 1228 input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_input, 1229 uint8_t *, in_offset); 1230 1231 while (left_length > 0) { 1232 if (fpga_reg_read_8(fpga_dev->mmio_base, 1233 FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) == 1) { 1234 fpga_reg_write_32(fpga_dev->mmio_base, 1235 FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS, 1236 out_offset); 1237 fpga_reg_write_64(fpga_dev->mmio_base, 1238 FPGA_5GNR_FEC_DDR4_WR_DATA_REGS, 1239 input[increment]); 1240 left_length -= FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; 1241 out_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; 1242 increment++; 1243 fpga_reg_write_8(fpga_dev->mmio_base, 1244 FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1); 1245 } 1246 } 1247 while (last_transaction > 0) { 1248 if (fpga_reg_read_8(fpga_dev->mmio_base, 1249 FPGA_5GNR_FEC_DDR4_ADDR_RDY_REGS) == 1) { 1250 fpga_reg_write_32(fpga_dev->mmio_base, 1251 FPGA_5GNR_FEC_DDR4_WR_ADDR_REGS, 1252 out_offset); 1253 last_word = input[increment]; 1254 last_word &= (uint64_t)(1 << (last_transaction * 4)) 1255 - 1; 1256 fpga_reg_write_64(fpga_dev->mmio_base, 1257 FPGA_5GNR_FEC_DDR4_WR_DATA_REGS, 1258 last_word); 1259 fpga_reg_write_8(fpga_dev->mmio_base, 1260 FPGA_5GNR_FEC_DDR4_WR_DONE_REGS, 1); 1261 last_transaction = 0; 1262 } 1263 } 1264 return 1; 1265 } 1266 1267 static inline int 1268 fpga_harq_read_loopback(struct fpga_5gnr_fec_device *fpga_dev, 1269 struct rte_mbuf *harq_output, uint16_t harq_in_length, 1270 uint32_t harq_in_offset, uint32_t harq_out_offset) 1271 { 1272 uint32_t left_length, in_offset = harq_in_offset; 1273 uint64_t reg; 1274 uint32_t increment = 0; 1275 uint64_t *input = NULL; 1276 uint32_t last_transaction = harq_in_length 1277 % FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; 1278 1279 if (last_transaction > 0) 1280 harq_in_length += (8 - last_transaction); 1281 1282 reg = fpga_reg_read_32(fpga_dev->mmio_base, 1283 FPGA_5GNR_FEC_HARQ_BUF_SIZE_REGS); 1284 if (reg < harq_in_length) { 1285 harq_in_length = reg; 1286 rte_bbdev_log(ERR, "HARQ in length > HARQ buffer size\n"); 1287 } 1288 1289 if (!mbuf_append(harq_output, harq_output, harq_in_length)) { 1290 rte_bbdev_log(ERR, "HARQ output buffer warning %d %d\n", 1291 harq_output->buf_len - 1292 rte_pktmbuf_headroom(harq_output), 1293 harq_in_length); 1294 harq_in_length = harq_output->buf_len - 1295 rte_pktmbuf_headroom(harq_output); 1296 if (!mbuf_append(harq_output, harq_output, harq_in_length)) { 1297 rte_bbdev_log(ERR, "HARQ output buffer issue %d %d\n", 1298 harq_output->buf_len, harq_in_length); 1299 return -1; 1300 } 1301 } 1302 left_length = harq_in_length; 1303 1304 input = (uint64_t *)rte_pktmbuf_mtod_offset(harq_output, 1305 uint8_t *, harq_out_offset); 1306 1307 while (left_length > 0) { 1308 fpga_reg_write_32(fpga_dev->mmio_base, 1309 FPGA_5GNR_FEC_DDR4_RD_ADDR_REGS, in_offset); 1310 fpga_reg_write_8(fpga_dev->mmio_base, 1311 FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 1); 1312 reg = fpga_reg_read_8(fpga_dev->mmio_base, 1313 FPGA_5GNR_FEC_DDR4_RD_RDY_REGS); 1314 while (reg != 1) { 1315 reg = fpga_reg_read_8(fpga_dev->mmio_base, 1316 FPGA_5GNR_FEC_DDR4_RD_RDY_REGS); 1317 if (reg == FPGA_DDR_OVERFLOW) { 1318 rte_bbdev_log(ERR, 1319 "Read address is overflow!\n"); 1320 return -1; 1321 } 1322 } 1323 input[increment] = fpga_reg_read_64(fpga_dev->mmio_base, 1324 FPGA_5GNR_FEC_DDR4_RD_DATA_REGS); 1325 left_length -= FPGA_5GNR_FEC_DDR_RD_DATA_LEN_IN_BYTES; 1326 in_offset += FPGA_5GNR_FEC_DDR_WR_DATA_LEN_IN_BYTES; 1327 increment++; 1328 fpga_reg_write_8(fpga_dev->mmio_base, 1329 FPGA_5GNR_FEC_DDR4_RD_DONE_REGS, 0); 1330 } 1331 return 1; 1332 } 1333 1334 static inline int 1335 enqueue_ldpc_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op *op, 1336 uint16_t desc_offset) 1337 { 1338 union fpga_dma_desc *desc; 1339 int ret; 1340 uint8_t c, crc24_bits = 0; 1341 struct rte_bbdev_op_ldpc_enc *enc = &op->ldpc_enc; 1342 uint16_t in_offset = enc->input.offset; 1343 uint16_t out_offset = enc->output.offset; 1344 struct rte_mbuf *m_in = enc->input.data; 1345 struct rte_mbuf *m_out = enc->output.data; 1346 struct rte_mbuf *m_out_head = enc->output.data; 1347 uint32_t in_length, out_length, e; 1348 uint16_t total_left = enc->input.length; 1349 uint16_t ring_offset; 1350 uint16_t K, k_; 1351 1352 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1353 /* Validate op structure */ 1354 /* FIXME */ 1355 if (validate_enc_op(op) == -1) { 1356 rte_bbdev_log(ERR, "LDPC encoder validation failed"); 1357 return -EINVAL; 1358 } 1359 #endif 1360 1361 /* Clear op status */ 1362 op->status = 0; 1363 1364 if (m_in == NULL || m_out == NULL) { 1365 rte_bbdev_log(ERR, "Invalid mbuf pointer"); 1366 op->status = 1 << RTE_BBDEV_DATA_ERROR; 1367 return -EINVAL; 1368 } 1369 1370 if (enc->op_flags & RTE_BBDEV_LDPC_CRC_24B_ATTACH) 1371 crc24_bits = 24; 1372 1373 if (enc->code_block_mode == 0) { 1374 /* For Transport Block mode */ 1375 /* FIXME */ 1376 c = enc->tb_params.c; 1377 e = enc->tb_params.ea; 1378 } else { /* For Code Block mode */ 1379 c = 1; 1380 e = enc->cb_params.e; 1381 } 1382 1383 /* Update total_left */ 1384 K = (enc->basegraph == 1 ? 22 : 10) * enc->z_c; 1385 k_ = K - enc->n_filler; 1386 in_length = (k_ - crc24_bits) >> 3; 1387 out_length = (e + 7) >> 3; 1388 1389 total_left = rte_pktmbuf_data_len(m_in) - in_offset; 1390 1391 /* Update offsets */ 1392 if (total_left != in_length) { 1393 op->status |= 1 << RTE_BBDEV_DATA_ERROR; 1394 rte_bbdev_log(ERR, 1395 "Mismatch between mbuf length and included CBs sizes %d", 1396 total_left); 1397 } 1398 1399 mbuf_append(m_out_head, m_out, out_length); 1400 1401 /* Offset into the ring */ 1402 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1403 /* Setup DMA Descriptor */ 1404 desc = q->ring_addr + ring_offset; 1405 1406 ret = fpga_dma_desc_te_fill(op, &desc->enc_req, m_in, m_out, 1407 k_, e, in_offset, out_offset, ring_offset, c); 1408 if (unlikely(ret < 0)) 1409 return ret; 1410 1411 /* Update lengths */ 1412 total_left -= in_length; 1413 op->ldpc_enc.output.length += out_length; 1414 1415 if (total_left > 0) { 1416 rte_bbdev_log(ERR, 1417 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u", 1418 total_left, in_length); 1419 return -1; 1420 } 1421 1422 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1423 print_dma_enc_desc_debug_info(desc); 1424 #endif 1425 return 1; 1426 } 1427 1428 static inline int 1429 enqueue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op *op, 1430 uint16_t desc_offset) 1431 { 1432 union fpga_dma_desc *desc; 1433 int ret; 1434 uint16_t ring_offset; 1435 uint8_t c; 1436 uint16_t e, in_length, out_length, k0, l, seg_total_left, sys_cols; 1437 uint16_t K, parity_offset, harq_in_length = 0, harq_out_length = 0; 1438 uint16_t crc24_overlap = 0; 1439 struct rte_bbdev_op_ldpc_dec *dec = &op->ldpc_dec; 1440 struct rte_mbuf *m_in = dec->input.data; 1441 struct rte_mbuf *m_out = dec->hard_output.data; 1442 struct rte_mbuf *m_out_head = dec->hard_output.data; 1443 uint16_t in_offset = dec->input.offset; 1444 uint16_t out_offset = dec->hard_output.offset; 1445 uint32_t harq_offset = 0; 1446 1447 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1448 /* Validate op structure */ 1449 if (validate_dec_op(op) == -1) { 1450 rte_bbdev_log(ERR, "LDPC decoder validation failed"); 1451 return -EINVAL; 1452 } 1453 #endif 1454 1455 /* Clear op status */ 1456 op->status = 0; 1457 1458 /* Setup DMA Descriptor */ 1459 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1460 desc = q->ring_addr + ring_offset; 1461 1462 if (check_bit(dec->op_flags, 1463 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 1464 struct rte_mbuf *harq_in = dec->harq_combined_input.data; 1465 struct rte_mbuf *harq_out = dec->harq_combined_output.data; 1466 harq_in_length = dec->harq_combined_input.length; 1467 uint32_t harq_in_offset = dec->harq_combined_input.offset; 1468 uint32_t harq_out_offset = dec->harq_combined_output.offset; 1469 1470 if (check_bit(dec->op_flags, 1471 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_OUT_ENABLE 1472 )) { 1473 ret = fpga_harq_write_loopback(q->d, harq_in, 1474 harq_in_length, harq_in_offset, 1475 harq_out_offset); 1476 } else if (check_bit(dec->op_flags, 1477 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_IN_ENABLE 1478 )) { 1479 ret = fpga_harq_read_loopback(q->d, harq_out, 1480 harq_in_length, harq_in_offset, 1481 harq_out_offset); 1482 dec->harq_combined_output.length = harq_in_length; 1483 } else { 1484 rte_bbdev_log(ERR, "OP flag Err!"); 1485 ret = -1; 1486 } 1487 /* Set descriptor for dequeue */ 1488 desc->dec_req.done = 1; 1489 desc->dec_req.error = 0; 1490 desc->dec_req.op_addr = op; 1491 desc->dec_req.cbs_in_op = 1; 1492 /* Mark this dummy descriptor to be dropped by HW */ 1493 desc->dec_req.desc_idx = (ring_offset + 1) 1494 & q->sw_ring_wrap_mask; 1495 return ret; /* Error or number of CB */ 1496 } 1497 1498 if (m_in == NULL || m_out == NULL) { 1499 rte_bbdev_log(ERR, "Invalid mbuf pointer"); 1500 op->status = 1 << RTE_BBDEV_DATA_ERROR; 1501 return -1; 1502 } 1503 1504 c = 1; 1505 e = dec->cb_params.e; 1506 1507 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_CRC_TYPE_24B_DROP)) 1508 crc24_overlap = 24; 1509 1510 sys_cols = (dec->basegraph == 1) ? 22 : 10; 1511 K = sys_cols * dec->z_c; 1512 parity_offset = K - 2 * dec->z_c; 1513 1514 out_length = ((K - crc24_overlap - dec->n_filler) >> 3); 1515 in_length = e; 1516 seg_total_left = dec->input.length; 1517 1518 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) { 1519 harq_in_length = RTE_MIN(dec->harq_combined_input.length, 1520 (uint32_t)dec->n_cb); 1521 } 1522 1523 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) { 1524 k0 = get_k0(dec->n_cb, dec->z_c, 1525 dec->basegraph, dec->rv_index); 1526 if (k0 > parity_offset) 1527 l = k0 + e; 1528 else 1529 l = k0 + e + dec->n_filler; 1530 harq_out_length = RTE_MIN(RTE_MAX(harq_in_length, l), 1531 dec->n_cb - dec->n_filler); 1532 dec->harq_combined_output.length = harq_out_length; 1533 } 1534 1535 mbuf_append(m_out_head, m_out, out_length); 1536 if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_IN_ENABLE)) 1537 harq_offset = dec->harq_combined_input.offset; 1538 else if (check_bit(dec->op_flags, RTE_BBDEV_LDPC_HQ_COMBINE_OUT_ENABLE)) 1539 harq_offset = dec->harq_combined_output.offset; 1540 1541 if ((harq_offset & 0x3FF) > 0) { 1542 rte_bbdev_log(ERR, "Invalid HARQ offset %d", harq_offset); 1543 op->status = 1 << RTE_BBDEV_DATA_ERROR; 1544 return -1; 1545 } 1546 1547 ret = fpga_dma_desc_ld_fill(op, &desc->dec_req, m_in, m_out, 1548 harq_in_length, in_offset, out_offset, harq_offset, 1549 ring_offset, c); 1550 if (unlikely(ret < 0)) 1551 return ret; 1552 /* Update lengths */ 1553 seg_total_left -= in_length; 1554 op->ldpc_dec.hard_output.length += out_length; 1555 if (seg_total_left > 0) { 1556 rte_bbdev_log(ERR, 1557 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u", 1558 seg_total_left, in_length); 1559 return -1; 1560 } 1561 1562 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1563 print_dma_dec_desc_debug_info(desc); 1564 #endif 1565 1566 return 1; 1567 } 1568 1569 static uint16_t 1570 fpga_enqueue_ldpc_enc(struct rte_bbdev_queue_data *q_data, 1571 struct rte_bbdev_enc_op **ops, uint16_t num) 1572 { 1573 uint16_t i, total_enqueued_cbs = 0; 1574 int32_t avail; 1575 int enqueued_cbs; 1576 struct fpga_queue *q = q_data->queue_private; 1577 union fpga_dma_desc *desc; 1578 1579 /* Check if queue is not full */ 1580 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) == 1581 q->head_free_desc)) 1582 return 0; 1583 1584 /* Calculates available space */ 1585 avail = (q->head_free_desc > q->tail) ? 1586 q->head_free_desc - q->tail - 1 : 1587 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1; 1588 1589 for (i = 0; i < num; ++i) { 1590 1591 /* Check if there is available space for further 1592 * processing 1593 */ 1594 if (unlikely(avail - 1 < 0)) 1595 break; 1596 avail -= 1; 1597 enqueued_cbs = enqueue_ldpc_enc_one_op_cb(q, ops[i], 1598 total_enqueued_cbs); 1599 1600 if (enqueued_cbs < 0) 1601 break; 1602 1603 total_enqueued_cbs += enqueued_cbs; 1604 1605 rte_bbdev_log_debug("enqueuing enc ops [%d/%d] | head %d | tail %d", 1606 total_enqueued_cbs, num, 1607 q->head_free_desc, q->tail); 1608 } 1609 1610 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt 1611 * only when all previous CBs were already processed. 1612 */ 1613 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1) 1614 & q->sw_ring_wrap_mask); 1615 desc->enc_req.irq_en = q->irq_enable; 1616 1617 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats); 1618 1619 /* Update stats */ 1620 q_data->queue_stats.enqueued_count += i; 1621 q_data->queue_stats.enqueue_err_count += num - i; 1622 1623 return i; 1624 } 1625 1626 static uint16_t 1627 fpga_enqueue_ldpc_dec(struct rte_bbdev_queue_data *q_data, 1628 struct rte_bbdev_dec_op **ops, uint16_t num) 1629 { 1630 uint16_t i, total_enqueued_cbs = 0; 1631 int32_t avail; 1632 int enqueued_cbs; 1633 struct fpga_queue *q = q_data->queue_private; 1634 union fpga_dma_desc *desc; 1635 1636 /* Check if queue is not full */ 1637 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) == 1638 q->head_free_desc)) 1639 return 0; 1640 1641 /* Calculates available space */ 1642 avail = (q->head_free_desc > q->tail) ? 1643 q->head_free_desc - q->tail - 1 : 1644 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1; 1645 1646 for (i = 0; i < num; ++i) { 1647 1648 /* Check if there is available space for further 1649 * processing 1650 */ 1651 if (unlikely(avail - 1 < 0)) 1652 break; 1653 avail -= 1; 1654 enqueued_cbs = enqueue_ldpc_dec_one_op_cb(q, ops[i], 1655 total_enqueued_cbs); 1656 1657 if (enqueued_cbs < 0) 1658 break; 1659 1660 total_enqueued_cbs += enqueued_cbs; 1661 1662 rte_bbdev_log_debug("enqueuing dec ops [%d/%d] | head %d | tail %d", 1663 total_enqueued_cbs, num, 1664 q->head_free_desc, q->tail); 1665 } 1666 1667 /* Update stats */ 1668 q_data->queue_stats.enqueued_count += i; 1669 q_data->queue_stats.enqueue_err_count += num - i; 1670 1671 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt 1672 * only when all previous CBs were already processed. 1673 */ 1674 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1) 1675 & q->sw_ring_wrap_mask); 1676 desc->enc_req.irq_en = q->irq_enable; 1677 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats); 1678 return i; 1679 } 1680 1681 1682 static inline int 1683 dequeue_ldpc_enc_one_op_cb(struct fpga_queue *q, 1684 struct rte_bbdev_enc_op **op, 1685 uint16_t desc_offset) 1686 { 1687 union fpga_dma_desc *desc; 1688 int desc_error; 1689 /* Set current desc */ 1690 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 1691 & q->sw_ring_wrap_mask); 1692 1693 /*check if done */ 1694 if (desc->enc_req.done == 0) 1695 return -1; 1696 1697 /* make sure the response is read atomically */ 1698 rte_smp_rmb(); 1699 1700 rte_bbdev_log_debug("DMA response desc %p", desc); 1701 1702 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1703 print_dma_enc_desc_debug_info(desc); 1704 #endif 1705 1706 *op = desc->enc_req.op_addr; 1707 /* Check the descriptor error field, return 1 on error */ 1708 desc_error = check_desc_error(desc->enc_req.error); 1709 (*op)->status = desc_error << RTE_BBDEV_DATA_ERROR; 1710 1711 return 1; 1712 } 1713 1714 1715 static inline int 1716 dequeue_ldpc_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op **op, 1717 uint16_t desc_offset) 1718 { 1719 union fpga_dma_desc *desc; 1720 int desc_error; 1721 /* Set descriptor */ 1722 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 1723 & q->sw_ring_wrap_mask); 1724 1725 /* Verify done bit is set */ 1726 if (desc->dec_req.done == 0) 1727 return -1; 1728 1729 /* make sure the response is read atomically */ 1730 rte_smp_rmb(); 1731 1732 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1733 print_dma_dec_desc_debug_info(desc); 1734 #endif 1735 1736 *op = desc->dec_req.op_addr; 1737 1738 if (check_bit((*op)->ldpc_dec.op_flags, 1739 RTE_BBDEV_LDPC_INTERNAL_HARQ_MEMORY_LOOPBACK)) { 1740 (*op)->status = 0; 1741 return 1; 1742 } 1743 1744 /* FPGA reports iterations based on round-up minus 1 */ 1745 (*op)->ldpc_dec.iter_count = desc->dec_req.iter + 1; 1746 /* CRC Check criteria */ 1747 if (desc->dec_req.crc24b_ind && !(desc->dec_req.crcb_pass)) 1748 (*op)->status = 1 << RTE_BBDEV_CRC_ERROR; 1749 /* et_pass = 0 when decoder fails */ 1750 (*op)->status |= !(desc->dec_req.et_pass) << RTE_BBDEV_SYNDROME_ERROR; 1751 /* Check the descriptor error field, return 1 on error */ 1752 desc_error = check_desc_error(desc->dec_req.error); 1753 (*op)->status |= desc_error << RTE_BBDEV_DATA_ERROR; 1754 return 1; 1755 } 1756 1757 static uint16_t 1758 fpga_dequeue_ldpc_enc(struct rte_bbdev_queue_data *q_data, 1759 struct rte_bbdev_enc_op **ops, uint16_t num) 1760 { 1761 struct fpga_queue *q = q_data->queue_private; 1762 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask; 1763 uint16_t i; 1764 uint16_t dequeued_cbs = 0; 1765 int ret; 1766 1767 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) { 1768 ret = dequeue_ldpc_enc_one_op_cb(q, &ops[i], dequeued_cbs); 1769 1770 if (ret < 0) 1771 break; 1772 1773 dequeued_cbs += ret; 1774 1775 rte_bbdev_log_debug("dequeuing enc ops [%d/%d] | head %d | tail %d", 1776 dequeued_cbs, num, q->head_free_desc, q->tail); 1777 } 1778 1779 /* Update head */ 1780 q->head_free_desc = (q->head_free_desc + dequeued_cbs) & 1781 q->sw_ring_wrap_mask; 1782 1783 /* Update stats */ 1784 q_data->queue_stats.dequeued_count += i; 1785 1786 return i; 1787 } 1788 1789 static uint16_t 1790 fpga_dequeue_ldpc_dec(struct rte_bbdev_queue_data *q_data, 1791 struct rte_bbdev_dec_op **ops, uint16_t num) 1792 { 1793 struct fpga_queue *q = q_data->queue_private; 1794 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask; 1795 uint16_t i; 1796 uint16_t dequeued_cbs = 0; 1797 int ret; 1798 1799 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) { 1800 ret = dequeue_ldpc_dec_one_op_cb(q, &ops[i], dequeued_cbs); 1801 1802 if (ret < 0) 1803 break; 1804 1805 dequeued_cbs += ret; 1806 1807 rte_bbdev_log_debug("dequeuing dec ops [%d/%d] | head %d | tail %d", 1808 dequeued_cbs, num, q->head_free_desc, q->tail); 1809 } 1810 1811 /* Update head */ 1812 q->head_free_desc = (q->head_free_desc + dequeued_cbs) & 1813 q->sw_ring_wrap_mask; 1814 1815 /* Update stats */ 1816 q_data->queue_stats.dequeued_count += i; 1817 1818 return i; 1819 } 1820 1821 1822 /* Initialization Function */ 1823 static void 1824 fpga_5gnr_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv) 1825 { 1826 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); 1827 1828 dev->dev_ops = &fpga_ops; 1829 dev->enqueue_ldpc_enc_ops = fpga_enqueue_ldpc_enc; 1830 dev->enqueue_ldpc_dec_ops = fpga_enqueue_ldpc_dec; 1831 dev->dequeue_ldpc_enc_ops = fpga_dequeue_ldpc_enc; 1832 dev->dequeue_ldpc_dec_ops = fpga_dequeue_ldpc_dec; 1833 1834 ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->pf_device = 1835 !strcmp(drv->driver.name, 1836 RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME)); 1837 ((struct fpga_5gnr_fec_device *) dev->data->dev_private)->mmio_base = 1838 pci_dev->mem_resource[0].addr; 1839 1840 rte_bbdev_log_debug( 1841 "Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64, 1842 drv->driver.name, dev->data->name, 1843 (void *)pci_dev->mem_resource[0].addr, 1844 pci_dev->mem_resource[0].phys_addr); 1845 } 1846 1847 static int 1848 fpga_5gnr_fec_probe(struct rte_pci_driver *pci_drv, 1849 struct rte_pci_device *pci_dev) 1850 { 1851 struct rte_bbdev *bbdev = NULL; 1852 char dev_name[RTE_BBDEV_NAME_MAX_LEN]; 1853 1854 if (pci_dev == NULL) { 1855 rte_bbdev_log(ERR, "NULL PCI device"); 1856 return -EINVAL; 1857 } 1858 1859 rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name)); 1860 1861 /* Allocate memory to be used privately by drivers */ 1862 bbdev = rte_bbdev_allocate(pci_dev->device.name); 1863 if (bbdev == NULL) 1864 return -ENODEV; 1865 1866 /* allocate device private memory */ 1867 bbdev->data->dev_private = rte_zmalloc_socket(dev_name, 1868 sizeof(struct fpga_5gnr_fec_device), 1869 RTE_CACHE_LINE_SIZE, 1870 pci_dev->device.numa_node); 1871 1872 if (bbdev->data->dev_private == NULL) { 1873 rte_bbdev_log(CRIT, 1874 "Allocate of %zu bytes for device \"%s\" failed", 1875 sizeof(struct fpga_5gnr_fec_device), dev_name); 1876 rte_bbdev_release(bbdev); 1877 return -ENOMEM; 1878 } 1879 1880 /* Fill HW specific part of device structure */ 1881 bbdev->device = &pci_dev->device; 1882 bbdev->intr_handle = &pci_dev->intr_handle; 1883 bbdev->data->socket_id = pci_dev->device.numa_node; 1884 1885 /* Invoke FEC FPGA device initialization function */ 1886 fpga_5gnr_fec_init(bbdev, pci_drv); 1887 1888 rte_bbdev_log_debug("bbdev id = %u [%s]", 1889 bbdev->data->dev_id, dev_name); 1890 1891 struct fpga_5gnr_fec_device *d = bbdev->data->dev_private; 1892 uint32_t version_id = fpga_reg_read_32(d->mmio_base, 1893 FPGA_5GNR_FEC_VERSION_ID); 1894 rte_bbdev_log(INFO, "FEC FPGA RTL v%u.%u", 1895 ((uint16_t)(version_id >> 16)), ((uint16_t)version_id)); 1896 1897 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1898 if (!strcmp(pci_drv->driver.name, 1899 RTE_STR(FPGA_5GNR_FEC_PF_DRIVER_NAME))) 1900 print_static_reg_debug_info(d->mmio_base); 1901 #endif 1902 return 0; 1903 } 1904 1905 static int 1906 fpga_5gnr_fec_remove(struct rte_pci_device *pci_dev) 1907 { 1908 struct rte_bbdev *bbdev; 1909 int ret; 1910 uint8_t dev_id; 1911 1912 if (pci_dev == NULL) 1913 return -EINVAL; 1914 1915 /* Find device */ 1916 bbdev = rte_bbdev_get_named_dev(pci_dev->device.name); 1917 if (bbdev == NULL) { 1918 rte_bbdev_log(CRIT, 1919 "Couldn't find HW dev \"%s\" to uninitialise it", 1920 pci_dev->device.name); 1921 return -ENODEV; 1922 } 1923 dev_id = bbdev->data->dev_id; 1924 1925 /* free device private memory before close */ 1926 rte_free(bbdev->data->dev_private); 1927 1928 /* Close device */ 1929 ret = rte_bbdev_close(dev_id); 1930 if (ret < 0) 1931 rte_bbdev_log(ERR, 1932 "Device %i failed to close during uninit: %i", 1933 dev_id, ret); 1934 1935 /* release bbdev from library */ 1936 ret = rte_bbdev_release(bbdev); 1937 if (ret) 1938 rte_bbdev_log(ERR, "Device %i failed to uninit: %i", dev_id, 1939 ret); 1940 1941 rte_bbdev_log_debug("Destroyed bbdev = %u", dev_id); 1942 1943 return 0; 1944 } 1945 1946 static inline void 1947 set_default_fpga_conf(struct rte_fpga_5gnr_fec_conf *def_conf) 1948 { 1949 /* clear default configuration before initialization */ 1950 memset(def_conf, 0, sizeof(struct rte_fpga_5gnr_fec_conf)); 1951 /* Set pf mode to true */ 1952 def_conf->pf_mode_en = true; 1953 1954 /* Set ratio between UL and DL to 1:1 (unit of weight is 3 CBs) */ 1955 def_conf->ul_bandwidth = 3; 1956 def_conf->dl_bandwidth = 3; 1957 1958 /* Set Load Balance Factor to 64 */ 1959 def_conf->dl_load_balance = 64; 1960 def_conf->ul_load_balance = 64; 1961 } 1962 1963 /* Initial configuration of FPGA 5GNR FEC device */ 1964 int 1965 rte_fpga_5gnr_fec_configure(const char *dev_name, 1966 const struct rte_fpga_5gnr_fec_conf *conf) 1967 { 1968 uint32_t payload_32, address; 1969 uint16_t payload_16; 1970 uint8_t payload_8; 1971 uint16_t q_id, vf_id, total_q_id, total_ul_q_id, total_dl_q_id; 1972 struct rte_bbdev *bbdev = rte_bbdev_get_named_dev(dev_name); 1973 struct rte_fpga_5gnr_fec_conf def_conf; 1974 1975 if (bbdev == NULL) { 1976 rte_bbdev_log(ERR, 1977 "Invalid dev_name (%s), or device is not yet initialised", 1978 dev_name); 1979 return -ENODEV; 1980 } 1981 1982 struct fpga_5gnr_fec_device *d = bbdev->data->dev_private; 1983 1984 if (conf == NULL) { 1985 rte_bbdev_log(ERR, 1986 "FPGA Configuration was not provided. Default configuration will be loaded."); 1987 set_default_fpga_conf(&def_conf); 1988 conf = &def_conf; 1989 } 1990 1991 /* 1992 * Configure UL:DL ratio. 1993 * [7:0]: UL weight 1994 * [15:8]: DL weight 1995 */ 1996 payload_16 = (conf->dl_bandwidth << 8) | conf->ul_bandwidth; 1997 address = FPGA_5GNR_FEC_CONFIGURATION; 1998 fpga_reg_write_16(d->mmio_base, address, payload_16); 1999 2000 /* Clear all queues registers */ 2001 payload_32 = FPGA_INVALID_HW_QUEUE_ID; 2002 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 2003 address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP; 2004 fpga_reg_write_32(d->mmio_base, address, payload_32); 2005 } 2006 2007 /* 2008 * If PF mode is enabled allocate all queues for PF only. 2009 * 2010 * For VF mode each VF can have different number of UL and DL queues. 2011 * Total number of queues to configure cannot exceed FPGA 2012 * capabilities - 64 queues - 32 queues for UL and 32 queues for DL. 2013 * Queues mapping is done according to configuration: 2014 * 2015 * UL queues: 2016 * | Q_ID | VF_ID | 2017 * | 0 | 0 | 2018 * | ... | 0 | 2019 * | conf->vf_dl_queues_number[0] - 1 | 0 | 2020 * | conf->vf_dl_queues_number[0] | 1 | 2021 * | ... | 1 | 2022 * | conf->vf_dl_queues_number[1] - 1 | 1 | 2023 * | ... | ... | 2024 * | conf->vf_dl_queues_number[7] - 1 | 7 | 2025 * 2026 * DL queues: 2027 * | Q_ID | VF_ID | 2028 * | 32 | 0 | 2029 * | ... | 0 | 2030 * | conf->vf_ul_queues_number[0] - 1 | 0 | 2031 * | conf->vf_ul_queues_number[0] | 1 | 2032 * | ... | 1 | 2033 * | conf->vf_ul_queues_number[1] - 1 | 1 | 2034 * | ... | ... | 2035 * | conf->vf_ul_queues_number[7] - 1 | 7 | 2036 * 2037 * Example of configuration: 2038 * conf->vf_ul_queues_number[0] = 4; -> 4 UL queues for VF0 2039 * conf->vf_dl_queues_number[0] = 4; -> 4 DL queues for VF0 2040 * conf->vf_ul_queues_number[1] = 2; -> 2 UL queues for VF1 2041 * conf->vf_dl_queues_number[1] = 2; -> 2 DL queues for VF1 2042 * 2043 * UL: 2044 * | Q_ID | VF_ID | 2045 * | 0 | 0 | 2046 * | 1 | 0 | 2047 * | 2 | 0 | 2048 * | 3 | 0 | 2049 * | 4 | 1 | 2050 * | 5 | 1 | 2051 * 2052 * DL: 2053 * | Q_ID | VF_ID | 2054 * | 32 | 0 | 2055 * | 33 | 0 | 2056 * | 34 | 0 | 2057 * | 35 | 0 | 2058 * | 36 | 1 | 2059 * | 37 | 1 | 2060 */ 2061 if (conf->pf_mode_en) { 2062 payload_32 = 0x1; 2063 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 2064 address = (q_id << 2) + FPGA_5GNR_FEC_QUEUE_MAP; 2065 fpga_reg_write_32(d->mmio_base, address, payload_32); 2066 } 2067 } else { 2068 /* Calculate total number of UL and DL queues to configure */ 2069 total_ul_q_id = total_dl_q_id = 0; 2070 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) { 2071 total_ul_q_id += conf->vf_ul_queues_number[vf_id]; 2072 total_dl_q_id += conf->vf_dl_queues_number[vf_id]; 2073 } 2074 total_q_id = total_dl_q_id + total_ul_q_id; 2075 /* 2076 * Check if total number of queues to configure does not exceed 2077 * FPGA capabilities (64 queues - 32 UL and 32 DL queues) 2078 */ 2079 if ((total_ul_q_id > FPGA_NUM_UL_QUEUES) || 2080 (total_dl_q_id > FPGA_NUM_DL_QUEUES) || 2081 (total_q_id > FPGA_TOTAL_NUM_QUEUES)) { 2082 rte_bbdev_log(ERR, 2083 "FPGA Configuration failed. Too many queues to configure: UL_Q %u, DL_Q %u, FPGA_Q %u", 2084 total_ul_q_id, total_dl_q_id, 2085 FPGA_TOTAL_NUM_QUEUES); 2086 return -EINVAL; 2087 } 2088 total_ul_q_id = 0; 2089 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) { 2090 for (q_id = 0; q_id < conf->vf_ul_queues_number[vf_id]; 2091 ++q_id, ++total_ul_q_id) { 2092 address = (total_ul_q_id << 2) + 2093 FPGA_5GNR_FEC_QUEUE_MAP; 2094 payload_32 = ((0x80 + vf_id) << 16) | 0x1; 2095 fpga_reg_write_32(d->mmio_base, address, 2096 payload_32); 2097 } 2098 } 2099 total_dl_q_id = 0; 2100 for (vf_id = 0; vf_id < FPGA_5GNR_FEC_NUM_VFS; ++vf_id) { 2101 for (q_id = 0; q_id < conf->vf_dl_queues_number[vf_id]; 2102 ++q_id, ++total_dl_q_id) { 2103 address = ((total_dl_q_id + FPGA_NUM_UL_QUEUES) 2104 << 2) + FPGA_5GNR_FEC_QUEUE_MAP; 2105 payload_32 = ((0x80 + vf_id) << 16) | 0x1; 2106 fpga_reg_write_32(d->mmio_base, address, 2107 payload_32); 2108 } 2109 } 2110 } 2111 2112 /* Setting Load Balance Factor */ 2113 payload_16 = (conf->dl_load_balance << 8) | (conf->ul_load_balance); 2114 address = FPGA_5GNR_FEC_LOAD_BALANCE_FACTOR; 2115 fpga_reg_write_16(d->mmio_base, address, payload_16); 2116 2117 /* Setting length of ring descriptor entry */ 2118 payload_16 = FPGA_RING_DESC_ENTRY_LENGTH; 2119 address = FPGA_5GNR_FEC_RING_DESC_LEN; 2120 fpga_reg_write_16(d->mmio_base, address, payload_16); 2121 2122 /* Setting FLR timeout value */ 2123 payload_16 = conf->flr_time_out; 2124 address = FPGA_5GNR_FEC_FLR_TIME_OUT; 2125 fpga_reg_write_16(d->mmio_base, address, payload_16); 2126 2127 /* Queue PF/VF mapping table is ready */ 2128 payload_8 = 0x1; 2129 address = FPGA_5GNR_FEC_QUEUE_PF_VF_MAP_DONE; 2130 fpga_reg_write_8(d->mmio_base, address, payload_8); 2131 2132 rte_bbdev_log_debug("PF FPGA 5GNR FEC configuration complete for %s", 2133 dev_name); 2134 2135 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2136 print_static_reg_debug_info(d->mmio_base); 2137 #endif 2138 return 0; 2139 } 2140 2141 /* FPGA 5GNR FEC PCI PF address map */ 2142 static struct rte_pci_id pci_id_fpga_5gnr_fec_pf_map[] = { 2143 { 2144 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID, 2145 FPGA_5GNR_FEC_PF_DEVICE_ID) 2146 }, 2147 {.device_id = 0}, 2148 }; 2149 2150 static struct rte_pci_driver fpga_5gnr_fec_pci_pf_driver = { 2151 .probe = fpga_5gnr_fec_probe, 2152 .remove = fpga_5gnr_fec_remove, 2153 .id_table = pci_id_fpga_5gnr_fec_pf_map, 2154 .drv_flags = RTE_PCI_DRV_NEED_MAPPING 2155 }; 2156 2157 /* FPGA 5GNR FEC PCI VF address map */ 2158 static struct rte_pci_id pci_id_fpga_5gnr_fec_vf_map[] = { 2159 { 2160 RTE_PCI_DEVICE(FPGA_5GNR_FEC_VENDOR_ID, 2161 FPGA_5GNR_FEC_VF_DEVICE_ID) 2162 }, 2163 {.device_id = 0}, 2164 }; 2165 2166 static struct rte_pci_driver fpga_5gnr_fec_pci_vf_driver = { 2167 .probe = fpga_5gnr_fec_probe, 2168 .remove = fpga_5gnr_fec_remove, 2169 .id_table = pci_id_fpga_5gnr_fec_vf_map, 2170 .drv_flags = RTE_PCI_DRV_NEED_MAPPING 2171 }; 2172 2173 2174 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_PF_DRIVER_NAME, fpga_5gnr_fec_pci_pf_driver); 2175 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_PF_DRIVER_NAME, 2176 pci_id_fpga_5gnr_fec_pf_map); 2177 RTE_PMD_REGISTER_PCI(FPGA_5GNR_FEC_VF_DRIVER_NAME, fpga_5gnr_fec_pci_vf_driver); 2178 RTE_PMD_REGISTER_PCI_TABLE(FPGA_5GNR_FEC_VF_DRIVER_NAME, 2179 pci_id_fpga_5gnr_fec_vf_map); 2180