1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #include <unistd.h> 6 7 #include <rte_common.h> 8 #include <rte_log.h> 9 #include <rte_dev.h> 10 #include <rte_malloc.h> 11 #include <rte_mempool.h> 12 #include <rte_errno.h> 13 #include <rte_pci.h> 14 #include <rte_bus_pci.h> 15 #include <rte_byteorder.h> 16 #ifdef RTE_BBDEV_OFFLOAD_COST 17 #include <rte_cycles.h> 18 #endif 19 20 #include <rte_bbdev.h> 21 #include <rte_bbdev_pmd.h> 22 23 #include "fpga_lte_fec.h" 24 25 #ifdef RTE_LIBRTE_BBDEV_DEBUG 26 RTE_LOG_REGISTER(fpga_lte_fec_logtype, pmd.bb.fpga_lte_fec, DEBUG); 27 #else 28 RTE_LOG_REGISTER(fpga_lte_fec_logtype, pmd.bb.fpga_lte_fec, NOTICE); 29 #endif 30 31 /* Helper macro for logging */ 32 #define rte_bbdev_log(level, fmt, ...) \ 33 rte_log(RTE_LOG_ ## level, fpga_lte_fec_logtype, fmt "\n", \ 34 ##__VA_ARGS__) 35 36 #ifdef RTE_LIBRTE_BBDEV_DEBUG 37 #define rte_bbdev_log_debug(fmt, ...) \ 38 rte_bbdev_log(DEBUG, "fpga_lte_fec: " fmt, \ 39 ##__VA_ARGS__) 40 #else 41 #define rte_bbdev_log_debug(fmt, ...) 42 #endif 43 44 /* FPGA LTE FEC driver names */ 45 #define FPGA_LTE_FEC_PF_DRIVER_NAME intel_fpga_lte_fec_pf 46 #define FPGA_LTE_FEC_VF_DRIVER_NAME intel_fpga_lte_fec_vf 47 48 /* FPGA LTE FEC PCI vendor & device IDs */ 49 #define FPGA_LTE_FEC_VENDOR_ID (0x1172) 50 #define FPGA_LTE_FEC_PF_DEVICE_ID (0x5052) 51 #define FPGA_LTE_FEC_VF_DEVICE_ID (0x5050) 52 53 /* Align DMA descriptors to 256 bytes - cache-aligned */ 54 #define FPGA_RING_DESC_ENTRY_LENGTH (8) 55 /* Ring size is in 256 bits (32 bytes) units */ 56 #define FPGA_RING_DESC_LEN_UNIT_BYTES (32) 57 /* Maximum size of queue */ 58 #define FPGA_RING_MAX_SIZE (1024) 59 #define FPGA_FLR_TIMEOUT_UNIT (16.384) 60 61 #define FPGA_NUM_UL_QUEUES (32) 62 #define FPGA_NUM_DL_QUEUES (32) 63 #define FPGA_TOTAL_NUM_QUEUES (FPGA_NUM_UL_QUEUES + FPGA_NUM_DL_QUEUES) 64 #define FPGA_NUM_INTR_VEC (FPGA_TOTAL_NUM_QUEUES - RTE_INTR_VEC_RXTX_OFFSET) 65 66 #define FPGA_INVALID_HW_QUEUE_ID (0xFFFFFFFF) 67 68 #define FPGA_QUEUE_FLUSH_TIMEOUT_US (1000) 69 #define FPGA_TIMEOUT_CHECK_INTERVAL (5) 70 71 /* FPGA LTE FEC Register mapping on BAR0 */ 72 enum { 73 FPGA_LTE_FEC_VERSION_ID = 0x00000000, /* len: 4B */ 74 FPGA_LTE_FEC_CONFIGURATION = 0x00000004, /* len: 2B */ 75 FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE = 0x00000008, /* len: 1B */ 76 FPGA_LTE_FEC_LOAD_BALANCE_FACTOR = 0x0000000a, /* len: 2B */ 77 FPGA_LTE_FEC_RING_DESC_LEN = 0x0000000c, /* len: 2B */ 78 FPGA_LTE_FEC_FLR_TIME_OUT = 0x0000000e, /* len: 2B */ 79 FPGA_LTE_FEC_VFQ_FLUSH_STATUS_LW = 0x00000018, /* len: 4B */ 80 FPGA_LTE_FEC_VFQ_FLUSH_STATUS_HI = 0x0000001c, /* len: 4B */ 81 FPGA_LTE_FEC_VF0_DEBUG = 0x00000020, /* len: 4B */ 82 FPGA_LTE_FEC_VF1_DEBUG = 0x00000024, /* len: 4B */ 83 FPGA_LTE_FEC_VF2_DEBUG = 0x00000028, /* len: 4B */ 84 FPGA_LTE_FEC_VF3_DEBUG = 0x0000002c, /* len: 4B */ 85 FPGA_LTE_FEC_VF4_DEBUG = 0x00000030, /* len: 4B */ 86 FPGA_LTE_FEC_VF5_DEBUG = 0x00000034, /* len: 4B */ 87 FPGA_LTE_FEC_VF6_DEBUG = 0x00000038, /* len: 4B */ 88 FPGA_LTE_FEC_VF7_DEBUG = 0x0000003c, /* len: 4B */ 89 FPGA_LTE_FEC_QUEUE_MAP = 0x00000040, /* len: 256B */ 90 FPGA_LTE_FEC_RING_CTRL_REGS = 0x00000200 /* len: 2048B */ 91 }; 92 93 /* FPGA LTE FEC Ring Control Registers */ 94 enum { 95 FPGA_LTE_FEC_RING_HEAD_ADDR = 0x00000008, 96 FPGA_LTE_FEC_RING_SIZE = 0x00000010, 97 FPGA_LTE_FEC_RING_MISC = 0x00000014, 98 FPGA_LTE_FEC_RING_ENABLE = 0x00000015, 99 FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN = 0x00000016, 100 FPGA_LTE_FEC_RING_SHADOW_TAIL = 0x00000018, 101 FPGA_LTE_FEC_RING_HEAD_POINT = 0x0000001C 102 }; 103 104 /* FPGA LTE FEC DESCRIPTOR ERROR */ 105 enum { 106 DESC_ERR_NO_ERR = 0x0, 107 DESC_ERR_K_OUT_OF_RANGE = 0x1, 108 DESC_ERR_K_NOT_NORMAL = 0x2, 109 DESC_ERR_KPAI_NOT_NORMAL = 0x3, 110 DESC_ERR_DESC_OFFSET_ERR = 0x4, 111 DESC_ERR_DESC_READ_FAIL = 0x8, 112 DESC_ERR_DESC_READ_TIMEOUT = 0x9, 113 DESC_ERR_DESC_READ_TLP_POISONED = 0xA, 114 DESC_ERR_CB_READ_FAIL = 0xC, 115 DESC_ERR_CB_READ_TIMEOUT = 0xD, 116 DESC_ERR_CB_READ_TLP_POISONED = 0xE 117 }; 118 119 /* FPGA LTE FEC DMA Encoding Request Descriptor */ 120 struct __rte_packed fpga_dma_enc_desc { 121 uint32_t done:1, 122 rsrvd0:11, 123 error:4, 124 rsrvd1:16; 125 uint32_t ncb:16, 126 rsrvd2:14, 127 rv:2; 128 uint32_t bypass_rm:1, 129 irq_en:1, 130 crc_en:1, 131 rsrvd3:13, 132 offset:10, 133 rsrvd4:6; 134 uint16_t e; 135 uint16_t k; 136 uint32_t out_addr_lw; 137 uint32_t out_addr_hi; 138 uint32_t in_addr_lw; 139 uint32_t in_addr_hi; 140 141 union { 142 struct { 143 /* Virtual addresses used to retrieve SW context info */ 144 void *op_addr; 145 /* Stores information about total number of Code Blocks 146 * in currently processed Transport Block 147 */ 148 uint64_t cbs_in_op; 149 }; 150 151 uint8_t sw_ctxt[FPGA_RING_DESC_LEN_UNIT_BYTES * 152 (FPGA_RING_DESC_ENTRY_LENGTH - 1)]; 153 }; 154 }; 155 156 /* FPGA LTE FEC DMA Decoding Request Descriptor */ 157 struct __rte_packed fpga_dma_dec_desc { 158 uint32_t done:1, 159 iter:5, 160 rsrvd0:2, 161 crc_pass:1, 162 rsrvd1:3, 163 error:4, 164 crc_type:1, 165 rsrvd2:7, 166 max_iter:5, 167 rsrvd3:3; 168 uint32_t rsrvd4; 169 uint32_t bypass_rm:1, 170 irq_en:1, 171 drop_crc:1, 172 rsrvd5:13, 173 offset:10, 174 rsrvd6:6; 175 uint16_t k; 176 uint16_t in_len; 177 uint32_t out_addr_lw; 178 uint32_t out_addr_hi; 179 uint32_t in_addr_lw; 180 uint32_t in_addr_hi; 181 182 union { 183 struct { 184 /* Virtual addresses used to retrieve SW context info */ 185 void *op_addr; 186 /* Stores information about total number of Code Blocks 187 * in currently processed Transport Block 188 */ 189 uint8_t cbs_in_op; 190 }; 191 192 uint32_t sw_ctxt[8 * (FPGA_RING_DESC_ENTRY_LENGTH - 1)]; 193 }; 194 }; 195 196 /* FPGA LTE DMA Descriptor */ 197 union fpga_dma_desc { 198 struct fpga_dma_enc_desc enc_req; 199 struct fpga_dma_dec_desc dec_req; 200 }; 201 202 /* FPGA LTE FEC Ring Control Register */ 203 struct __rte_packed fpga_ring_ctrl_reg { 204 uint64_t ring_base_addr; 205 uint64_t ring_head_addr; 206 uint16_t ring_size:11; 207 uint16_t rsrvd0; 208 union { /* Miscellaneous register */ 209 uint8_t misc; 210 uint8_t max_ul_dec:5, 211 max_ul_dec_en:1, 212 rsrvd1:2; 213 }; 214 uint8_t enable; 215 uint8_t flush_queue_en; 216 uint8_t rsrvd2; 217 uint16_t shadow_tail; 218 uint16_t rsrvd3; 219 uint16_t head_point; 220 uint16_t rsrvd4; 221 222 }; 223 224 /* Private data structure for each FPGA FEC device */ 225 struct fpga_lte_fec_device { 226 /** Base address of MMIO registers (BAR0) */ 227 void *mmio_base; 228 /** Base address of memory for sw rings */ 229 void *sw_rings; 230 /** Physical address of sw_rings */ 231 rte_iova_t sw_rings_phys; 232 /** Number of bytes available for each queue in device. */ 233 uint32_t sw_ring_size; 234 /** Max number of entries available for each queue in device */ 235 uint32_t sw_ring_max_depth; 236 /** Base address of response tail pointer buffer */ 237 uint32_t *tail_ptrs; 238 /** Physical address of tail pointers */ 239 rte_iova_t tail_ptr_phys; 240 /** Queues flush completion flag */ 241 uint64_t *flush_queue_status; 242 /* Bitmap capturing which Queues are bound to the PF/VF */ 243 uint64_t q_bound_bit_map; 244 /* Bitmap capturing which Queues have already been assigned */ 245 uint64_t q_assigned_bit_map; 246 /** True if this is a PF FPGA FEC device */ 247 bool pf_device; 248 }; 249 250 /* Structure associated with each queue. */ 251 struct __rte_cache_aligned fpga_queue { 252 struct fpga_ring_ctrl_reg ring_ctrl_reg; /* Ring Control Register */ 253 union fpga_dma_desc *ring_addr; /* Virtual address of software ring */ 254 uint64_t *ring_head_addr; /* Virtual address of completion_head */ 255 uint64_t shadow_completion_head; /* Shadow completion head value */ 256 uint16_t head_free_desc; /* Ring head */ 257 uint16_t tail; /* Ring tail */ 258 /* Mask used to wrap enqueued descriptors on the sw ring */ 259 uint32_t sw_ring_wrap_mask; 260 uint32_t irq_enable; /* Enable ops dequeue interrupts if set to 1 */ 261 uint8_t q_idx; /* Queue index */ 262 struct fpga_lte_fec_device *d; 263 /* MMIO register of shadow_tail used to enqueue descriptors */ 264 void *shadow_tail_addr; 265 }; 266 267 /* Write to 16 bit MMIO register address */ 268 static inline void 269 mmio_write_16(void *addr, uint16_t value) 270 { 271 *((volatile uint16_t *)(addr)) = rte_cpu_to_le_16(value); 272 } 273 274 /* Write to 32 bit MMIO register address */ 275 static inline void 276 mmio_write_32(void *addr, uint32_t value) 277 { 278 *((volatile uint32_t *)(addr)) = rte_cpu_to_le_32(value); 279 } 280 281 /* Write to 64 bit MMIO register address */ 282 static inline void 283 mmio_write_64(void *addr, uint64_t value) 284 { 285 *((volatile uint64_t *)(addr)) = rte_cpu_to_le_64(value); 286 } 287 288 /* Write a 8 bit register of a FPGA LTE FEC device */ 289 static inline void 290 fpga_reg_write_8(void *mmio_base, uint32_t offset, uint8_t payload) 291 { 292 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 293 *((volatile uint8_t *)(reg_addr)) = payload; 294 } 295 296 /* Write a 16 bit register of a FPGA LTE FEC device */ 297 static inline void 298 fpga_reg_write_16(void *mmio_base, uint32_t offset, uint16_t payload) 299 { 300 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 301 mmio_write_16(reg_addr, payload); 302 } 303 304 /* Write a 32 bit register of a FPGA LTE FEC device */ 305 static inline void 306 fpga_reg_write_32(void *mmio_base, uint32_t offset, uint32_t payload) 307 { 308 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 309 mmio_write_32(reg_addr, payload); 310 } 311 312 /* Write a 64 bit register of a FPGA LTE FEC device */ 313 static inline void 314 fpga_reg_write_64(void *mmio_base, uint32_t offset, uint64_t payload) 315 { 316 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 317 mmio_write_64(reg_addr, payload); 318 } 319 320 /* Write a ring control register of a FPGA LTE FEC device */ 321 static inline void 322 fpga_ring_reg_write(void *mmio_base, uint32_t offset, 323 struct fpga_ring_ctrl_reg payload) 324 { 325 fpga_reg_write_64(mmio_base, offset, payload.ring_base_addr); 326 fpga_reg_write_64(mmio_base, offset + FPGA_LTE_FEC_RING_HEAD_ADDR, 327 payload.ring_head_addr); 328 fpga_reg_write_16(mmio_base, offset + FPGA_LTE_FEC_RING_SIZE, 329 payload.ring_size); 330 fpga_reg_write_16(mmio_base, offset + FPGA_LTE_FEC_RING_HEAD_POINT, 331 payload.head_point); 332 fpga_reg_write_8(mmio_base, offset + FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN, 333 payload.flush_queue_en); 334 fpga_reg_write_16(mmio_base, offset + FPGA_LTE_FEC_RING_SHADOW_TAIL, 335 payload.shadow_tail); 336 fpga_reg_write_8(mmio_base, offset + FPGA_LTE_FEC_RING_MISC, 337 payload.misc); 338 fpga_reg_write_8(mmio_base, offset + FPGA_LTE_FEC_RING_ENABLE, 339 payload.enable); 340 } 341 342 /* Read a register of FPGA LTE FEC device */ 343 static uint32_t 344 fpga_reg_read_32(void *mmio_base, uint32_t offset) 345 { 346 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 347 uint32_t ret = *((volatile uint32_t *)(reg_addr)); 348 return rte_le_to_cpu_32(ret); 349 } 350 351 #ifdef RTE_LIBRTE_BBDEV_DEBUG 352 /* Read a register of FPGA LTE FEC device */ 353 static uint8_t 354 fpga_reg_read_8(void *mmio_base, uint32_t offset) 355 { 356 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 357 return *((volatile uint8_t *)(reg_addr)); 358 } 359 360 /* Read a register of FPGA LTE FEC device */ 361 static uint16_t 362 fpga_reg_read_16(void *mmio_base, uint32_t offset) 363 { 364 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 365 uint16_t ret = *((volatile uint16_t *)(reg_addr)); 366 return rte_le_to_cpu_16(ret); 367 } 368 369 /* Read a register of FPGA LTE FEC device */ 370 static uint64_t 371 fpga_reg_read_64(void *mmio_base, uint32_t offset) 372 { 373 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 374 uint64_t ret = *((volatile uint64_t *)(reg_addr)); 375 return rte_le_to_cpu_64(ret); 376 } 377 378 /* Read Ring Control Register of FPGA LTE FEC device */ 379 static inline void 380 print_ring_reg_debug_info(void *mmio_base, uint32_t offset) 381 { 382 rte_bbdev_log_debug( 383 "FPGA MMIO base address @ %p | Ring Control Register @ offset = 0x%08" 384 PRIx32, mmio_base, offset); 385 rte_bbdev_log_debug( 386 "RING_BASE_ADDR = 0x%016"PRIx64, 387 fpga_reg_read_64(mmio_base, offset)); 388 rte_bbdev_log_debug( 389 "RING_HEAD_ADDR = 0x%016"PRIx64, 390 fpga_reg_read_64(mmio_base, offset + 391 FPGA_LTE_FEC_RING_HEAD_ADDR)); 392 rte_bbdev_log_debug( 393 "RING_SIZE = 0x%04"PRIx16, 394 fpga_reg_read_16(mmio_base, offset + 395 FPGA_LTE_FEC_RING_SIZE)); 396 rte_bbdev_log_debug( 397 "RING_MISC = 0x%02"PRIx8, 398 fpga_reg_read_8(mmio_base, offset + 399 FPGA_LTE_FEC_RING_MISC)); 400 rte_bbdev_log_debug( 401 "RING_ENABLE = 0x%02"PRIx8, 402 fpga_reg_read_8(mmio_base, offset + 403 FPGA_LTE_FEC_RING_ENABLE)); 404 rte_bbdev_log_debug( 405 "RING_FLUSH_QUEUE_EN = 0x%02"PRIx8, 406 fpga_reg_read_8(mmio_base, offset + 407 FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN)); 408 rte_bbdev_log_debug( 409 "RING_SHADOW_TAIL = 0x%04"PRIx16, 410 fpga_reg_read_16(mmio_base, offset + 411 FPGA_LTE_FEC_RING_SHADOW_TAIL)); 412 rte_bbdev_log_debug( 413 "RING_HEAD_POINT = 0x%04"PRIx16, 414 fpga_reg_read_16(mmio_base, offset + 415 FPGA_LTE_FEC_RING_HEAD_POINT)); 416 } 417 418 /* Read Static Register of FPGA LTE FEC device */ 419 static inline void 420 print_static_reg_debug_info(void *mmio_base) 421 { 422 uint16_t config = fpga_reg_read_16(mmio_base, 423 FPGA_LTE_FEC_CONFIGURATION); 424 uint8_t qmap_done = fpga_reg_read_8(mmio_base, 425 FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE); 426 uint16_t lb_factor = fpga_reg_read_16(mmio_base, 427 FPGA_LTE_FEC_LOAD_BALANCE_FACTOR); 428 uint16_t ring_desc_len = fpga_reg_read_16(mmio_base, 429 FPGA_LTE_FEC_RING_DESC_LEN); 430 uint16_t flr_time_out = fpga_reg_read_16(mmio_base, 431 FPGA_LTE_FEC_FLR_TIME_OUT); 432 433 rte_bbdev_log_debug("UL.DL Weights = %u.%u", 434 ((uint8_t)config), ((uint8_t)(config >> 8))); 435 rte_bbdev_log_debug("UL.DL Load Balance = %u.%u", 436 ((uint8_t)lb_factor), ((uint8_t)(lb_factor >> 8))); 437 rte_bbdev_log_debug("Queue-PF/VF Mapping Table = %s", 438 (qmap_done > 0) ? "READY" : "NOT-READY"); 439 rte_bbdev_log_debug("Ring Descriptor Size = %u bytes", 440 ring_desc_len*FPGA_RING_DESC_LEN_UNIT_BYTES); 441 rte_bbdev_log_debug("FLR Timeout = %f usec", 442 (float)flr_time_out*FPGA_FLR_TIMEOUT_UNIT); 443 } 444 445 /* Print decode DMA Descriptor of FPGA LTE FEC device */ 446 static void 447 print_dma_dec_desc_debug_info(union fpga_dma_desc *desc) 448 { 449 rte_bbdev_log_debug("DMA response desc %p\n" 450 "\t-- done(%"PRIu32") | iter(%"PRIu32") | crc_pass(%"PRIu32")" 451 " | error (%"PRIu32") | crc_type(%"PRIu32")\n" 452 "\t-- max_iter(%"PRIu32") | bypass_rm(%"PRIu32") | " 453 "irq_en (%"PRIu32") | drop_crc(%"PRIu32") | offset(%"PRIu32")\n" 454 "\t-- k(%"PRIu32") | in_len (%"PRIu16") | op_add(%p)\n" 455 "\t-- cbs_in_op(%"PRIu32") | in_add (0x%08"PRIx32"%08"PRIx32") | " 456 "out_add (0x%08"PRIx32"%08"PRIx32")", 457 desc, 458 (uint32_t)desc->dec_req.done, 459 (uint32_t)desc->dec_req.iter, 460 (uint32_t)desc->dec_req.crc_pass, 461 (uint32_t)desc->dec_req.error, 462 (uint32_t)desc->dec_req.crc_type, 463 (uint32_t)desc->dec_req.max_iter, 464 (uint32_t)desc->dec_req.bypass_rm, 465 (uint32_t)desc->dec_req.irq_en, 466 (uint32_t)desc->dec_req.drop_crc, 467 (uint32_t)desc->dec_req.offset, 468 (uint32_t)desc->dec_req.k, 469 (uint16_t)desc->dec_req.in_len, 470 desc->dec_req.op_addr, 471 (uint32_t)desc->dec_req.cbs_in_op, 472 (uint32_t)desc->dec_req.in_addr_hi, 473 (uint32_t)desc->dec_req.in_addr_lw, 474 (uint32_t)desc->dec_req.out_addr_hi, 475 (uint32_t)desc->dec_req.out_addr_lw); 476 } 477 #endif 478 479 static int 480 fpga_setup_queues(struct rte_bbdev *dev, uint16_t num_queues, int socket_id) 481 { 482 /* Number of queues bound to a PF/VF */ 483 uint32_t hw_q_num = 0; 484 uint32_t ring_size, payload, address, q_id, offset; 485 rte_iova_t phys_addr; 486 struct fpga_ring_ctrl_reg ring_reg; 487 struct fpga_lte_fec_device *fpga_dev = dev->data->dev_private; 488 489 address = FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE; 490 if (!(fpga_reg_read_32(fpga_dev->mmio_base, address) & 0x1)) { 491 rte_bbdev_log(ERR, 492 "Queue-PF/VF mapping is not set! Was PF configured for device (%s) ?", 493 dev->data->name); 494 return -EPERM; 495 } 496 497 /* Clear queue registers structure */ 498 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg)); 499 500 /* Scan queue map. 501 * If a queue is valid and mapped to a calling PF/VF the read value is 502 * replaced with a queue ID and if it's not then 503 * FPGA_INVALID_HW_QUEUE_ID is returned. 504 */ 505 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 506 uint32_t hw_q_id = fpga_reg_read_32(fpga_dev->mmio_base, 507 FPGA_LTE_FEC_QUEUE_MAP + (q_id << 2)); 508 509 rte_bbdev_log_debug("%s: queue ID: %u, registry queue ID: %u", 510 dev->device->name, q_id, hw_q_id); 511 512 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) { 513 fpga_dev->q_bound_bit_map |= (1ULL << q_id); 514 /* Clear queue register of found queue */ 515 offset = FPGA_LTE_FEC_RING_CTRL_REGS + 516 (sizeof(struct fpga_ring_ctrl_reg) * q_id); 517 fpga_ring_reg_write(fpga_dev->mmio_base, 518 offset, ring_reg); 519 ++hw_q_num; 520 } 521 } 522 if (hw_q_num == 0) { 523 rte_bbdev_log(ERR, 524 "No HW queues assigned to this device. Probably this is a VF configured for PF mode. Check device configuration!"); 525 return -ENODEV; 526 } 527 528 if (num_queues > hw_q_num) { 529 rte_bbdev_log(ERR, 530 "Not enough queues for device %s! Requested: %u, available: %u", 531 dev->device->name, num_queues, hw_q_num); 532 return -EINVAL; 533 } 534 535 ring_size = FPGA_RING_MAX_SIZE * sizeof(struct fpga_dma_dec_desc); 536 537 /* Enforce 32 byte alignment */ 538 RTE_BUILD_BUG_ON((RTE_CACHE_LINE_SIZE % 32) != 0); 539 540 /* Allocate memory for SW descriptor rings */ 541 fpga_dev->sw_rings = rte_zmalloc_socket(dev->device->driver->name, 542 num_queues * ring_size, RTE_CACHE_LINE_SIZE, 543 socket_id); 544 if (fpga_dev->sw_rings == NULL) { 545 rte_bbdev_log(ERR, 546 "Failed to allocate memory for %s:%u sw_rings", 547 dev->device->driver->name, dev->data->dev_id); 548 return -ENOMEM; 549 } 550 551 fpga_dev->sw_rings_phys = rte_malloc_virt2iova(fpga_dev->sw_rings); 552 fpga_dev->sw_ring_size = ring_size; 553 fpga_dev->sw_ring_max_depth = FPGA_RING_MAX_SIZE; 554 555 /* Allocate memory for ring flush status */ 556 fpga_dev->flush_queue_status = rte_zmalloc_socket(NULL, 557 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, socket_id); 558 if (fpga_dev->flush_queue_status == NULL) { 559 rte_bbdev_log(ERR, 560 "Failed to allocate memory for %s:%u flush_queue_status", 561 dev->device->driver->name, dev->data->dev_id); 562 return -ENOMEM; 563 } 564 565 /* Set the flush status address registers */ 566 phys_addr = rte_malloc_virt2iova(fpga_dev->flush_queue_status); 567 568 address = FPGA_LTE_FEC_VFQ_FLUSH_STATUS_LW; 569 payload = (uint32_t)(phys_addr); 570 fpga_reg_write_32(fpga_dev->mmio_base, address, payload); 571 572 address = FPGA_LTE_FEC_VFQ_FLUSH_STATUS_HI; 573 payload = (uint32_t)(phys_addr >> 32); 574 fpga_reg_write_32(fpga_dev->mmio_base, address, payload); 575 576 return 0; 577 } 578 579 static int 580 fpga_dev_close(struct rte_bbdev *dev) 581 { 582 struct fpga_lte_fec_device *fpga_dev = dev->data->dev_private; 583 584 rte_free(fpga_dev->sw_rings); 585 rte_free(fpga_dev->flush_queue_status); 586 587 return 0; 588 } 589 590 static void 591 fpga_dev_info_get(struct rte_bbdev *dev, 592 struct rte_bbdev_driver_info *dev_info) 593 { 594 struct fpga_lte_fec_device *d = dev->data->dev_private; 595 uint32_t q_id = 0; 596 597 /* TODO RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN and numbers of buffers are set 598 * to temporary values as they are required by test application while 599 * validation phase. 600 */ 601 static const struct rte_bbdev_op_cap bbdev_capabilities[] = { 602 { 603 .type = RTE_BBDEV_OP_TURBO_DEC, 604 .cap.turbo_dec = { 605 .capability_flags = 606 RTE_BBDEV_TURBO_CRC_TYPE_24B | 607 RTE_BBDEV_TURBO_SUBBLOCK_DEINTERLEAVE | 608 RTE_BBDEV_TURBO_DEC_INTERRUPTS | 609 RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN | 610 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP, 611 .max_llr_modulus = INT8_MAX, 612 .num_buffers_src = 613 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS, 614 .num_buffers_hard_out = 615 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS, 616 .num_buffers_soft_out = 0 617 } 618 }, 619 { 620 .type = RTE_BBDEV_OP_TURBO_ENC, 621 .cap.turbo_enc = { 622 .capability_flags = 623 RTE_BBDEV_TURBO_CRC_24B_ATTACH | 624 RTE_BBDEV_TURBO_RATE_MATCH | 625 RTE_BBDEV_TURBO_ENC_INTERRUPTS, 626 .num_buffers_src = 627 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS, 628 .num_buffers_dst = 629 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS 630 } 631 }, 632 RTE_BBDEV_END_OF_CAPABILITIES_LIST() 633 }; 634 635 static struct rte_bbdev_queue_conf default_queue_conf; 636 default_queue_conf.socket = dev->data->socket_id; 637 default_queue_conf.queue_size = FPGA_RING_MAX_SIZE; 638 639 640 dev_info->driver_name = dev->device->driver->name; 641 dev_info->queue_size_lim = FPGA_RING_MAX_SIZE; 642 dev_info->hardware_accelerated = true; 643 dev_info->min_alignment = 64; 644 dev_info->default_queue_conf = default_queue_conf; 645 dev_info->capabilities = bbdev_capabilities; 646 dev_info->cpu_flag_reqs = NULL; 647 648 /* Calculates number of queues assigned to device */ 649 dev_info->max_num_queues = 0; 650 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 651 uint32_t hw_q_id = fpga_reg_read_32(d->mmio_base, 652 FPGA_LTE_FEC_QUEUE_MAP + (q_id << 2)); 653 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) 654 dev_info->max_num_queues++; 655 } 656 } 657 658 /** 659 * Find index of queue bound to current PF/VF which is unassigned. Return -1 660 * when there is no available queue 661 */ 662 static int 663 fpga_find_free_queue_idx(struct rte_bbdev *dev, 664 const struct rte_bbdev_queue_conf *conf) 665 { 666 struct fpga_lte_fec_device *d = dev->data->dev_private; 667 uint64_t q_idx; 668 uint8_t i = 0; 669 uint8_t range = FPGA_TOTAL_NUM_QUEUES >> 1; 670 671 if (conf->op_type == RTE_BBDEV_OP_TURBO_ENC) { 672 i = FPGA_NUM_DL_QUEUES; 673 range = FPGA_TOTAL_NUM_QUEUES; 674 } 675 676 for (; i < range; ++i) { 677 q_idx = 1ULL << i; 678 /* Check if index of queue is bound to current PF/VF */ 679 if (d->q_bound_bit_map & q_idx) 680 /* Check if found queue was not already assigned */ 681 if (!(d->q_assigned_bit_map & q_idx)) { 682 d->q_assigned_bit_map |= q_idx; 683 return i; 684 } 685 } 686 687 rte_bbdev_log(INFO, "Failed to find free queue on %s", dev->data->name); 688 689 return -1; 690 } 691 692 static int 693 fpga_queue_setup(struct rte_bbdev *dev, uint16_t queue_id, 694 const struct rte_bbdev_queue_conf *conf) 695 { 696 uint32_t address, ring_offset; 697 struct fpga_lte_fec_device *d = dev->data->dev_private; 698 struct fpga_queue *q; 699 int8_t q_idx; 700 701 /* Check if there is a free queue to assign */ 702 q_idx = fpga_find_free_queue_idx(dev, conf); 703 if (q_idx == -1) 704 return -1; 705 706 /* Allocate the queue data structure. */ 707 q = rte_zmalloc_socket(dev->device->driver->name, sizeof(*q), 708 RTE_CACHE_LINE_SIZE, conf->socket); 709 if (q == NULL) { 710 /* Mark queue as un-assigned */ 711 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 712 rte_bbdev_log(ERR, "Failed to allocate queue memory"); 713 return -ENOMEM; 714 } 715 716 q->d = d; 717 q->q_idx = q_idx; 718 719 /* Set ring_base_addr */ 720 q->ring_addr = RTE_PTR_ADD(d->sw_rings, (d->sw_ring_size * queue_id)); 721 q->ring_ctrl_reg.ring_base_addr = d->sw_rings_phys + 722 (d->sw_ring_size * queue_id); 723 724 /* Allocate memory for Completion Head variable*/ 725 q->ring_head_addr = rte_zmalloc_socket(dev->device->driver->name, 726 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, conf->socket); 727 if (q->ring_head_addr == NULL) { 728 /* Mark queue as un-assigned */ 729 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 730 rte_free(q); 731 rte_bbdev_log(ERR, 732 "Failed to allocate memory for %s:%u completion_head", 733 dev->device->driver->name, dev->data->dev_id); 734 return -ENOMEM; 735 } 736 /* Set ring_head_addr */ 737 q->ring_ctrl_reg.ring_head_addr = 738 rte_malloc_virt2iova(q->ring_head_addr); 739 740 /* Clear shadow_completion_head */ 741 q->shadow_completion_head = 0; 742 743 /* Set ring_size */ 744 if (conf->queue_size > FPGA_RING_MAX_SIZE) { 745 /* Mark queue as un-assigned */ 746 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 747 rte_free(q->ring_head_addr); 748 rte_free(q); 749 rte_bbdev_log(ERR, 750 "Size of queue is too big %d (MAX: %d ) for %s:%u", 751 conf->queue_size, FPGA_RING_MAX_SIZE, 752 dev->device->driver->name, dev->data->dev_id); 753 return -EINVAL; 754 } 755 q->ring_ctrl_reg.ring_size = conf->queue_size; 756 757 /* Set Miscellaneous FPGA register*/ 758 /* Max iteration number for TTI mitigation - todo */ 759 q->ring_ctrl_reg.max_ul_dec = 0; 760 /* Enable max iteration number for TTI - todo */ 761 q->ring_ctrl_reg.max_ul_dec_en = 0; 762 763 /* Enable the ring */ 764 q->ring_ctrl_reg.enable = 1; 765 766 /* Set FPGA head_point and tail registers */ 767 q->ring_ctrl_reg.head_point = q->tail = 0; 768 769 /* Set FPGA shadow_tail register */ 770 q->ring_ctrl_reg.shadow_tail = q->tail; 771 772 /* Calculates the ring offset for found queue */ 773 ring_offset = FPGA_LTE_FEC_RING_CTRL_REGS + 774 (sizeof(struct fpga_ring_ctrl_reg) * q_idx); 775 776 /* Set FPGA Ring Control Registers */ 777 fpga_ring_reg_write(d->mmio_base, ring_offset, q->ring_ctrl_reg); 778 779 /* Store MMIO register of shadow_tail */ 780 address = ring_offset + FPGA_LTE_FEC_RING_SHADOW_TAIL; 781 q->shadow_tail_addr = RTE_PTR_ADD(d->mmio_base, address); 782 783 q->head_free_desc = q->tail; 784 785 /* Set wrap mask */ 786 q->sw_ring_wrap_mask = conf->queue_size - 1; 787 788 rte_bbdev_log_debug("Setup dev%u q%u: queue_idx=%u", 789 dev->data->dev_id, queue_id, q->q_idx); 790 791 dev->data->queues[queue_id].queue_private = q; 792 793 rte_bbdev_log_debug("BBDEV queue[%d] set up for FPGA queue[%d]", 794 queue_id, q_idx); 795 796 #ifdef RTE_LIBRTE_BBDEV_DEBUG 797 /* Read FPGA Ring Control Registers after configuration*/ 798 print_ring_reg_debug_info(d->mmio_base, ring_offset); 799 #endif 800 return 0; 801 } 802 803 static int 804 fpga_queue_release(struct rte_bbdev *dev, uint16_t queue_id) 805 { 806 struct fpga_lte_fec_device *d = dev->data->dev_private; 807 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 808 struct fpga_ring_ctrl_reg ring_reg; 809 uint32_t offset; 810 811 rte_bbdev_log_debug("FPGA Queue[%d] released", queue_id); 812 813 if (q != NULL) { 814 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg)); 815 offset = FPGA_LTE_FEC_RING_CTRL_REGS + 816 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 817 /* Disable queue */ 818 fpga_reg_write_8(d->mmio_base, 819 offset + FPGA_LTE_FEC_RING_ENABLE, 0x00); 820 /* Clear queue registers */ 821 fpga_ring_reg_write(d->mmio_base, offset, ring_reg); 822 823 /* Mark the Queue as un-assigned */ 824 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q->q_idx)); 825 rte_free(q->ring_head_addr); 826 rte_free(q); 827 dev->data->queues[queue_id].queue_private = NULL; 828 } 829 830 return 0; 831 } 832 833 /* Function starts a device queue. */ 834 static int 835 fpga_queue_start(struct rte_bbdev *dev, uint16_t queue_id) 836 { 837 struct fpga_lte_fec_device *d = dev->data->dev_private; 838 #ifdef RTE_LIBRTE_BBDEV_DEBUG 839 if (d == NULL) { 840 rte_bbdev_log(ERR, "Invalid device pointer"); 841 return -1; 842 } 843 #endif 844 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 845 uint32_t offset = FPGA_LTE_FEC_RING_CTRL_REGS + 846 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 847 uint8_t enable = 0x01; 848 uint16_t zero = 0x0000; 849 850 /* Clear queue head and tail variables */ 851 q->tail = q->head_free_desc = 0; 852 853 /* Clear FPGA head_point and tail registers */ 854 fpga_reg_write_16(d->mmio_base, offset + FPGA_LTE_FEC_RING_HEAD_POINT, 855 zero); 856 fpga_reg_write_16(d->mmio_base, offset + FPGA_LTE_FEC_RING_SHADOW_TAIL, 857 zero); 858 859 /* Enable queue */ 860 fpga_reg_write_8(d->mmio_base, offset + FPGA_LTE_FEC_RING_ENABLE, 861 enable); 862 863 rte_bbdev_log_debug("FPGA Queue[%d] started", queue_id); 864 return 0; 865 } 866 867 /* Function stops a device queue. */ 868 static int 869 fpga_queue_stop(struct rte_bbdev *dev, uint16_t queue_id) 870 { 871 struct fpga_lte_fec_device *d = dev->data->dev_private; 872 #ifdef RTE_LIBRTE_BBDEV_DEBUG 873 if (d == NULL) { 874 rte_bbdev_log(ERR, "Invalid device pointer"); 875 return -1; 876 } 877 #endif 878 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 879 uint32_t offset = FPGA_LTE_FEC_RING_CTRL_REGS + 880 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 881 uint8_t payload = 0x01; 882 uint8_t counter = 0; 883 uint8_t timeout = FPGA_QUEUE_FLUSH_TIMEOUT_US / 884 FPGA_TIMEOUT_CHECK_INTERVAL; 885 886 /* Set flush_queue_en bit to trigger queue flushing */ 887 fpga_reg_write_8(d->mmio_base, 888 offset + FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN, payload); 889 890 /** Check if queue flush is completed. 891 * FPGA will update the completion flag after queue flushing is 892 * completed. If completion flag is not updated within 1ms it is 893 * considered as a failure. 894 */ 895 while (!(*((volatile uint8_t *)d->flush_queue_status + q->q_idx) & payload)) { 896 if (counter > timeout) { 897 rte_bbdev_log(ERR, "FPGA Queue Flush failed for queue %d", 898 queue_id); 899 return -1; 900 } 901 usleep(FPGA_TIMEOUT_CHECK_INTERVAL); 902 counter++; 903 } 904 905 /* Disable queue */ 906 payload = 0x00; 907 fpga_reg_write_8(d->mmio_base, offset + FPGA_LTE_FEC_RING_ENABLE, 908 payload); 909 910 rte_bbdev_log_debug("FPGA Queue[%d] stopped", queue_id); 911 return 0; 912 } 913 914 static inline uint16_t 915 get_queue_id(struct rte_bbdev_data *data, uint8_t q_idx) 916 { 917 uint16_t queue_id; 918 919 for (queue_id = 0; queue_id < data->num_queues; ++queue_id) { 920 struct fpga_queue *q = data->queues[queue_id].queue_private; 921 if (q != NULL && q->q_idx == q_idx) 922 return queue_id; 923 } 924 925 return -1; 926 } 927 928 /* Interrupt handler triggered by FPGA dev for handling specific interrupt */ 929 static void 930 fpga_dev_interrupt_handler(void *cb_arg) 931 { 932 struct rte_bbdev *dev = cb_arg; 933 struct fpga_lte_fec_device *fpga_dev = dev->data->dev_private; 934 struct fpga_queue *q; 935 uint64_t ring_head; 936 uint64_t q_idx; 937 uint16_t queue_id; 938 uint8_t i; 939 940 /* Scan queue assigned to this device */ 941 for (i = 0; i < FPGA_TOTAL_NUM_QUEUES; ++i) { 942 q_idx = 1ULL << i; 943 if (fpga_dev->q_bound_bit_map & q_idx) { 944 queue_id = get_queue_id(dev->data, i); 945 if (queue_id == (uint16_t) -1) 946 continue; 947 948 /* Check if completion head was changed */ 949 q = dev->data->queues[queue_id].queue_private; 950 ring_head = *q->ring_head_addr; 951 if (q->shadow_completion_head != ring_head && 952 q->irq_enable == 1) { 953 q->shadow_completion_head = ring_head; 954 rte_bbdev_pmd_callback_process( 955 dev, 956 RTE_BBDEV_EVENT_DEQUEUE, 957 &queue_id); 958 } 959 } 960 } 961 } 962 963 static int 964 fpga_queue_intr_enable(struct rte_bbdev *dev, uint16_t queue_id) 965 { 966 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 967 968 if (!rte_intr_cap_multiple(dev->intr_handle)) 969 return -ENOTSUP; 970 971 q->irq_enable = 1; 972 973 return 0; 974 } 975 976 static int 977 fpga_queue_intr_disable(struct rte_bbdev *dev, uint16_t queue_id) 978 { 979 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 980 q->irq_enable = 0; 981 982 return 0; 983 } 984 985 static int 986 fpga_intr_enable(struct rte_bbdev *dev) 987 { 988 int ret; 989 uint8_t i; 990 991 if (!rte_intr_cap_multiple(dev->intr_handle)) { 992 rte_bbdev_log(ERR, "Multiple intr vector is not supported by FPGA (%s)", 993 dev->data->name); 994 return -ENOTSUP; 995 } 996 997 /* Create event file descriptors for each of 64 queue. Event fds will be 998 * mapped to FPGA IRQs in rte_intr_enable(). This is a 1:1 mapping where 999 * the IRQ number is a direct translation to the queue number. 1000 * 1001 * 63 (FPGA_NUM_INTR_VEC) event fds are created as rte_intr_enable() 1002 * mapped the first IRQ to already created interrupt event file 1003 * descriptor (intr_handle->fd). 1004 */ 1005 if (rte_intr_efd_enable(dev->intr_handle, FPGA_NUM_INTR_VEC)) { 1006 rte_bbdev_log(ERR, "Failed to create fds for %u queues", 1007 dev->data->num_queues); 1008 return -1; 1009 } 1010 1011 /* TODO Each event file descriptor is overwritten by interrupt event 1012 * file descriptor. That descriptor is added to epoll observed list. 1013 * It ensures that callback function assigned to that descriptor will 1014 * invoked when any FPGA queue issues interrupt. 1015 */ 1016 for (i = 0; i < FPGA_NUM_INTR_VEC; ++i) 1017 dev->intr_handle->efds[i] = dev->intr_handle->fd; 1018 1019 if (!dev->intr_handle->intr_vec) { 1020 dev->intr_handle->intr_vec = rte_zmalloc("intr_vec", 1021 dev->data->num_queues * sizeof(int), 0); 1022 if (!dev->intr_handle->intr_vec) { 1023 rte_bbdev_log(ERR, "Failed to allocate %u vectors", 1024 dev->data->num_queues); 1025 return -ENOMEM; 1026 } 1027 } 1028 1029 ret = rte_intr_enable(dev->intr_handle); 1030 if (ret < 0) { 1031 rte_bbdev_log(ERR, 1032 "Couldn't enable interrupts for device: %s", 1033 dev->data->name); 1034 return ret; 1035 } 1036 1037 ret = rte_intr_callback_register(dev->intr_handle, 1038 fpga_dev_interrupt_handler, dev); 1039 if (ret < 0) { 1040 rte_bbdev_log(ERR, 1041 "Couldn't register interrupt callback for device: %s", 1042 dev->data->name); 1043 return ret; 1044 } 1045 1046 return 0; 1047 } 1048 1049 static const struct rte_bbdev_ops fpga_ops = { 1050 .setup_queues = fpga_setup_queues, 1051 .intr_enable = fpga_intr_enable, 1052 .close = fpga_dev_close, 1053 .info_get = fpga_dev_info_get, 1054 .queue_setup = fpga_queue_setup, 1055 .queue_stop = fpga_queue_stop, 1056 .queue_start = fpga_queue_start, 1057 .queue_release = fpga_queue_release, 1058 .queue_intr_enable = fpga_queue_intr_enable, 1059 .queue_intr_disable = fpga_queue_intr_disable 1060 }; 1061 1062 static inline void 1063 fpga_dma_enqueue(struct fpga_queue *q, uint16_t num_desc, 1064 struct rte_bbdev_stats *queue_stats) 1065 { 1066 #ifdef RTE_BBDEV_OFFLOAD_COST 1067 uint64_t start_time = 0; 1068 queue_stats->acc_offload_cycles = 0; 1069 #else 1070 RTE_SET_USED(queue_stats); 1071 #endif 1072 1073 /* Update tail and shadow_tail register */ 1074 q->tail = (q->tail + num_desc) & q->sw_ring_wrap_mask; 1075 1076 rte_wmb(); 1077 1078 #ifdef RTE_BBDEV_OFFLOAD_COST 1079 /* Start time measurement for enqueue function offload. */ 1080 start_time = rte_rdtsc_precise(); 1081 #endif 1082 mmio_write_16(q->shadow_tail_addr, q->tail); 1083 1084 #ifdef RTE_BBDEV_OFFLOAD_COST 1085 rte_wmb(); 1086 queue_stats->acc_offload_cycles += rte_rdtsc_precise() - start_time; 1087 #endif 1088 } 1089 1090 /* Calculates number of CBs in processed encoder TB based on 'r' and input 1091 * length. 1092 */ 1093 static inline uint8_t 1094 get_num_cbs_in_op_enc(struct rte_bbdev_op_turbo_enc *turbo_enc) 1095 { 1096 uint8_t c, c_neg, r, crc24_bits = 0; 1097 uint16_t k, k_neg, k_pos; 1098 uint8_t cbs_in_op = 0; 1099 int32_t length; 1100 1101 length = turbo_enc->input.length; 1102 r = turbo_enc->tb_params.r; 1103 c = turbo_enc->tb_params.c; 1104 c_neg = turbo_enc->tb_params.c_neg; 1105 k_neg = turbo_enc->tb_params.k_neg; 1106 k_pos = turbo_enc->tb_params.k_pos; 1107 crc24_bits = 24; 1108 while (length > 0 && r < c) { 1109 k = (r < c_neg) ? k_neg : k_pos; 1110 length -= (k - crc24_bits) >> 3; 1111 r++; 1112 cbs_in_op++; 1113 } 1114 1115 return cbs_in_op; 1116 } 1117 1118 /* Calculates number of CBs in processed decoder TB based on 'r' and input 1119 * length. 1120 */ 1121 static inline uint16_t 1122 get_num_cbs_in_op_dec(struct rte_bbdev_op_turbo_dec *turbo_dec) 1123 { 1124 uint8_t c, c_neg, r = 0; 1125 uint16_t kw, k, k_neg, k_pos, cbs_in_op = 0; 1126 int32_t length; 1127 1128 length = turbo_dec->input.length; 1129 r = turbo_dec->tb_params.r; 1130 c = turbo_dec->tb_params.c; 1131 c_neg = turbo_dec->tb_params.c_neg; 1132 k_neg = turbo_dec->tb_params.k_neg; 1133 k_pos = turbo_dec->tb_params.k_pos; 1134 while (length > 0 && r < c) { 1135 k = (r < c_neg) ? k_neg : k_pos; 1136 kw = RTE_ALIGN_CEIL(k + 4, 32) * 3; 1137 length -= kw; 1138 r++; 1139 cbs_in_op++; 1140 } 1141 1142 return cbs_in_op; 1143 } 1144 1145 /* Read flag value 0/1/ from bitmap */ 1146 static inline bool 1147 check_bit(uint32_t bitmap, uint32_t bitmask) 1148 { 1149 return bitmap & bitmask; 1150 } 1151 1152 /* Print an error if a descriptor error has occurred. 1153 * Return 0 on success, 1 on failure 1154 */ 1155 static inline int 1156 check_desc_error(uint32_t error_code) { 1157 switch (error_code) { 1158 case DESC_ERR_NO_ERR: 1159 return 0; 1160 case DESC_ERR_K_OUT_OF_RANGE: 1161 rte_bbdev_log(ERR, "Block_size_k is out of range (k<40 or k>6144)"); 1162 break; 1163 case DESC_ERR_K_NOT_NORMAL: 1164 rte_bbdev_log(ERR, "Block_size_k is not a normal value within normal range"); 1165 break; 1166 case DESC_ERR_KPAI_NOT_NORMAL: 1167 rte_bbdev_log(ERR, "Three_kpai is not a normal value for UL only"); 1168 break; 1169 case DESC_ERR_DESC_OFFSET_ERR: 1170 rte_bbdev_log(ERR, "Queue offset does not meet the expectation in the FPGA"); 1171 break; 1172 case (DESC_ERR_K_OUT_OF_RANGE | DESC_ERR_DESC_OFFSET_ERR): 1173 rte_bbdev_log(ERR, "Block_size_k is out of range (k<40 or k>6144) and queue offset error"); 1174 break; 1175 case (DESC_ERR_K_NOT_NORMAL | DESC_ERR_DESC_OFFSET_ERR): 1176 rte_bbdev_log(ERR, "Block_size_k is not a normal value within normal range and queue offset error"); 1177 break; 1178 case (DESC_ERR_KPAI_NOT_NORMAL | DESC_ERR_DESC_OFFSET_ERR): 1179 rte_bbdev_log(ERR, "Three_kpai is not a normal value for UL only and queue offset error"); 1180 break; 1181 case DESC_ERR_DESC_READ_FAIL: 1182 rte_bbdev_log(ERR, "Unsuccessful completion for descriptor read"); 1183 break; 1184 case DESC_ERR_DESC_READ_TIMEOUT: 1185 rte_bbdev_log(ERR, "Descriptor read time-out"); 1186 break; 1187 case DESC_ERR_DESC_READ_TLP_POISONED: 1188 rte_bbdev_log(ERR, "Descriptor read TLP poisoned"); 1189 break; 1190 case DESC_ERR_CB_READ_FAIL: 1191 rte_bbdev_log(ERR, "Unsuccessful completion for code block"); 1192 break; 1193 case DESC_ERR_CB_READ_TIMEOUT: 1194 rte_bbdev_log(ERR, "Code block read time-out"); 1195 break; 1196 case DESC_ERR_CB_READ_TLP_POISONED: 1197 rte_bbdev_log(ERR, "Code block read TLP poisoned"); 1198 break; 1199 default: 1200 rte_bbdev_log(ERR, "Descriptor error unknown error code %u", 1201 error_code); 1202 break; 1203 } 1204 return 1; 1205 } 1206 1207 /** 1208 * Set DMA descriptor for encode operation (1 Code Block) 1209 * 1210 * @param op 1211 * Pointer to a single encode operation. 1212 * @param desc 1213 * Pointer to DMA descriptor. 1214 * @param input 1215 * Pointer to pointer to input data which will be decoded. 1216 * @param k 1217 * K value (length of input in bits). 1218 * @param e 1219 * E value (length of output in bits). 1220 * @param ncb 1221 * Ncb value (size of the soft buffer). 1222 * @param out_length 1223 * Length of output buffer 1224 * @param in_offset 1225 * Input offset in rte_mbuf structure. It is used for calculating the point 1226 * where data is starting. 1227 * @param out_offset 1228 * Output offset in rte_mbuf structure. It is used for calculating the point 1229 * where hard output data will be stored. 1230 * @param cbs_in_op 1231 * Number of CBs contained in one operation. 1232 */ 1233 static inline int 1234 fpga_dma_desc_te_fill(struct rte_bbdev_enc_op *op, 1235 struct fpga_dma_enc_desc *desc, struct rte_mbuf *input, 1236 struct rte_mbuf *output, uint16_t k, uint16_t e, uint16_t ncb, 1237 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset, 1238 uint8_t cbs_in_op) 1239 1240 { 1241 /* reset */ 1242 desc->done = 0; 1243 desc->crc_en = check_bit(op->turbo_enc.op_flags, 1244 RTE_BBDEV_TURBO_CRC_24B_ATTACH); 1245 desc->bypass_rm = !check_bit(op->turbo_enc.op_flags, 1246 RTE_BBDEV_TURBO_RATE_MATCH); 1247 desc->k = k; 1248 desc->e = e; 1249 desc->ncb = ncb; 1250 desc->rv = op->turbo_enc.rv_index; 1251 desc->offset = desc_offset; 1252 /* Set inbound data buffer address */ 1253 desc->in_addr_hi = (uint32_t)( 1254 rte_pktmbuf_iova_offset(input, in_offset) >> 32); 1255 desc->in_addr_lw = (uint32_t)( 1256 rte_pktmbuf_iova_offset(input, in_offset)); 1257 1258 desc->out_addr_hi = (uint32_t)( 1259 rte_pktmbuf_iova_offset(output, out_offset) >> 32); 1260 desc->out_addr_lw = (uint32_t)( 1261 rte_pktmbuf_iova_offset(output, out_offset)); 1262 1263 /* Save software context needed for dequeue */ 1264 desc->op_addr = op; 1265 1266 /* Set total number of CBs in an op */ 1267 desc->cbs_in_op = cbs_in_op; 1268 1269 return 0; 1270 } 1271 1272 /** 1273 * Set DMA descriptor for encode operation (1 Code Block) 1274 * 1275 * @param op 1276 * Pointer to a single encode operation. 1277 * @param desc 1278 * Pointer to DMA descriptor. 1279 * @param input 1280 * Pointer to pointer to input data which will be decoded. 1281 * @param in_length 1282 * Length of an input. 1283 * @param k 1284 * K value (length of an output in bits). 1285 * @param in_offset 1286 * Input offset in rte_mbuf structure. It is used for calculating the point 1287 * where data is starting. 1288 * @param out_offset 1289 * Output offset in rte_mbuf structure. It is used for calculating the point 1290 * where hard output data will be stored. 1291 * @param cbs_in_op 1292 * Number of CBs contained in one operation. 1293 */ 1294 static inline int 1295 fpga_dma_desc_td_fill(struct rte_bbdev_dec_op *op, 1296 struct fpga_dma_dec_desc *desc, struct rte_mbuf *input, 1297 struct rte_mbuf *output, uint16_t in_length, uint16_t k, 1298 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset, 1299 uint8_t cbs_in_op) 1300 { 1301 /* reset */ 1302 desc->done = 0; 1303 /* Set inbound data buffer address */ 1304 desc->in_addr_hi = (uint32_t)( 1305 rte_pktmbuf_iova_offset(input, in_offset) >> 32); 1306 desc->in_addr_lw = (uint32_t)( 1307 rte_pktmbuf_iova_offset(input, in_offset)); 1308 desc->in_len = in_length; 1309 desc->k = k; 1310 desc->crc_type = !check_bit(op->turbo_dec.op_flags, 1311 RTE_BBDEV_TURBO_CRC_TYPE_24B); 1312 if ((op->turbo_dec.code_block_mode == 0) 1313 && !check_bit(op->turbo_dec.op_flags, 1314 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP)) 1315 desc->drop_crc = 1; 1316 desc->max_iter = op->turbo_dec.iter_max * 2; 1317 desc->offset = desc_offset; 1318 desc->out_addr_hi = (uint32_t)( 1319 rte_pktmbuf_iova_offset(output, out_offset) >> 32); 1320 desc->out_addr_lw = (uint32_t)( 1321 rte_pktmbuf_iova_offset(output, out_offset)); 1322 1323 /* Save software context needed for dequeue */ 1324 desc->op_addr = op; 1325 1326 /* Set total number of CBs in an op */ 1327 desc->cbs_in_op = cbs_in_op; 1328 1329 return 0; 1330 } 1331 1332 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1333 /* Validates turbo encoder parameters */ 1334 static int 1335 validate_enc_op(struct rte_bbdev_enc_op *op) 1336 { 1337 struct rte_bbdev_op_turbo_enc *turbo_enc = &op->turbo_enc; 1338 struct rte_bbdev_op_enc_turbo_cb_params *cb = NULL; 1339 struct rte_bbdev_op_enc_turbo_tb_params *tb = NULL; 1340 uint16_t kw, kw_neg, kw_pos; 1341 1342 if (turbo_enc->input.length > 1343 RTE_BBDEV_TURBO_MAX_TB_SIZE >> 3) { 1344 rte_bbdev_log(ERR, "TB size (%u) is too big, max: %d", 1345 turbo_enc->input.length, 1346 RTE_BBDEV_TURBO_MAX_TB_SIZE); 1347 op->status = 1 << RTE_BBDEV_DATA_ERROR; 1348 return -1; 1349 } 1350 1351 if (op->mempool == NULL) { 1352 rte_bbdev_log(ERR, "Invalid mempool pointer"); 1353 return -1; 1354 } 1355 if (turbo_enc->input.data == NULL) { 1356 rte_bbdev_log(ERR, "Invalid input pointer"); 1357 return -1; 1358 } 1359 if (turbo_enc->output.data == NULL) { 1360 rte_bbdev_log(ERR, "Invalid output pointer"); 1361 return -1; 1362 } 1363 if (turbo_enc->rv_index > 3) { 1364 rte_bbdev_log(ERR, 1365 "rv_index (%u) is out of range 0 <= value <= 3", 1366 turbo_enc->rv_index); 1367 return -1; 1368 } 1369 if (turbo_enc->code_block_mode != 0 && 1370 turbo_enc->code_block_mode != 1) { 1371 rte_bbdev_log(ERR, 1372 "code_block_mode (%u) is out of range 0 <= value <= 1", 1373 turbo_enc->code_block_mode); 1374 return -1; 1375 } 1376 1377 if (turbo_enc->code_block_mode == 0) { 1378 tb = &turbo_enc->tb_params; 1379 if ((tb->k_neg < RTE_BBDEV_TURBO_MIN_CB_SIZE 1380 || tb->k_neg > RTE_BBDEV_TURBO_MAX_CB_SIZE) 1381 && tb->c_neg > 0) { 1382 rte_bbdev_log(ERR, 1383 "k_neg (%u) is out of range %u <= value <= %u", 1384 tb->k_neg, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1385 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1386 return -1; 1387 } 1388 if (tb->k_pos < RTE_BBDEV_TURBO_MIN_CB_SIZE 1389 || tb->k_pos > RTE_BBDEV_TURBO_MAX_CB_SIZE) { 1390 rte_bbdev_log(ERR, 1391 "k_pos (%u) is out of range %u <= value <= %u", 1392 tb->k_pos, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1393 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1394 return -1; 1395 } 1396 if (tb->c_neg > (RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1)) 1397 rte_bbdev_log(ERR, 1398 "c_neg (%u) is out of range 0 <= value <= %u", 1399 tb->c_neg, 1400 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1); 1401 if (tb->c < 1 || tb->c > RTE_BBDEV_TURBO_MAX_CODE_BLOCKS) { 1402 rte_bbdev_log(ERR, 1403 "c (%u) is out of range 1 <= value <= %u", 1404 tb->c, RTE_BBDEV_TURBO_MAX_CODE_BLOCKS); 1405 return -1; 1406 } 1407 if (tb->cab > tb->c) { 1408 rte_bbdev_log(ERR, 1409 "cab (%u) is greater than c (%u)", 1410 tb->cab, tb->c); 1411 return -1; 1412 } 1413 if ((tb->ea < RTE_BBDEV_TURBO_MIN_CB_SIZE || (tb->ea % 2)) 1414 && tb->r < tb->cab) { 1415 rte_bbdev_log(ERR, 1416 "ea (%u) is less than %u or it is not even", 1417 tb->ea, RTE_BBDEV_TURBO_MIN_CB_SIZE); 1418 return -1; 1419 } 1420 if ((tb->eb < RTE_BBDEV_TURBO_MIN_CB_SIZE || (tb->eb % 2)) 1421 && tb->c > tb->cab) { 1422 rte_bbdev_log(ERR, 1423 "eb (%u) is less than %u or it is not even", 1424 tb->eb, RTE_BBDEV_TURBO_MIN_CB_SIZE); 1425 return -1; 1426 } 1427 1428 kw_neg = 3 * RTE_ALIGN_CEIL(tb->k_neg + 4, 1429 RTE_BBDEV_TURBO_C_SUBBLOCK); 1430 if (tb->ncb_neg < tb->k_neg || tb->ncb_neg > kw_neg) { 1431 rte_bbdev_log(ERR, 1432 "ncb_neg (%u) is out of range (%u) k_neg <= value <= (%u) kw_neg", 1433 tb->ncb_neg, tb->k_neg, kw_neg); 1434 return -1; 1435 } 1436 1437 kw_pos = 3 * RTE_ALIGN_CEIL(tb->k_pos + 4, 1438 RTE_BBDEV_TURBO_C_SUBBLOCK); 1439 if (tb->ncb_pos < tb->k_pos || tb->ncb_pos > kw_pos) { 1440 rte_bbdev_log(ERR, 1441 "ncb_pos (%u) is out of range (%u) k_pos <= value <= (%u) kw_pos", 1442 tb->ncb_pos, tb->k_pos, kw_pos); 1443 return -1; 1444 } 1445 if (tb->r > (tb->c - 1)) { 1446 rte_bbdev_log(ERR, 1447 "r (%u) is greater than c - 1 (%u)", 1448 tb->r, tb->c - 1); 1449 return -1; 1450 } 1451 } else { 1452 cb = &turbo_enc->cb_params; 1453 if (cb->k < RTE_BBDEV_TURBO_MIN_CB_SIZE 1454 || cb->k > RTE_BBDEV_TURBO_MAX_CB_SIZE) { 1455 rte_bbdev_log(ERR, 1456 "k (%u) is out of range %u <= value <= %u", 1457 cb->k, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1458 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1459 return -1; 1460 } 1461 1462 if (cb->e < RTE_BBDEV_TURBO_MIN_CB_SIZE || (cb->e % 2)) { 1463 rte_bbdev_log(ERR, 1464 "e (%u) is less than %u or it is not even", 1465 cb->e, RTE_BBDEV_TURBO_MIN_CB_SIZE); 1466 return -1; 1467 } 1468 1469 kw = RTE_ALIGN_CEIL(cb->k + 4, RTE_BBDEV_TURBO_C_SUBBLOCK) * 3; 1470 if (cb->ncb < cb->k || cb->ncb > kw) { 1471 rte_bbdev_log(ERR, 1472 "ncb (%u) is out of range (%u) k <= value <= (%u) kw", 1473 cb->ncb, cb->k, kw); 1474 return -1; 1475 } 1476 } 1477 1478 return 0; 1479 } 1480 #endif 1481 1482 static inline char * 1483 mbuf_append(struct rte_mbuf *m_head, struct rte_mbuf *m, uint16_t len) 1484 { 1485 if (unlikely(len > rte_pktmbuf_tailroom(m))) 1486 return NULL; 1487 1488 char *tail = (char *)m->buf_addr + m->data_off + m->data_len; 1489 m->data_len = (uint16_t)(m->data_len + len); 1490 m_head->pkt_len = (m_head->pkt_len + len); 1491 return tail; 1492 } 1493 1494 static inline int 1495 enqueue_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op *op, 1496 uint16_t desc_offset) 1497 { 1498 union fpga_dma_desc *desc; 1499 struct rte_mbuf *input; 1500 struct rte_mbuf *output; 1501 int ret; 1502 uint16_t k, e, ncb, ring_offset; 1503 uint32_t total_left, in_length, out_length, in_offset, out_offset; 1504 1505 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1506 /* Validate op structure */ 1507 if (validate_enc_op(op) == -1) { 1508 rte_bbdev_log(ERR, "Turbo encoder validation failed"); 1509 return -EINVAL; 1510 } 1511 #endif 1512 1513 input = op->turbo_enc.input.data; 1514 output = op->turbo_enc.output.data; 1515 in_offset = op->turbo_enc.input.offset; 1516 out_offset = op->turbo_enc.output.offset; 1517 total_left = op->turbo_enc.input.length; 1518 k = op->turbo_enc.cb_params.k; 1519 e = op->turbo_enc.cb_params.e; 1520 ncb = op->turbo_enc.cb_params.ncb; 1521 1522 if (check_bit(op->turbo_enc.op_flags, RTE_BBDEV_TURBO_CRC_24B_ATTACH)) 1523 in_length = ((k - 24) >> 3); 1524 else 1525 in_length = k >> 3; 1526 1527 if (check_bit(op->turbo_enc.op_flags, RTE_BBDEV_TURBO_RATE_MATCH)) 1528 out_length = (e + 7) >> 3; 1529 else 1530 out_length = (k >> 3) * 3 + 2; 1531 1532 mbuf_append(output, output, out_length); 1533 1534 /* Offset into the ring */ 1535 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1536 /* Setup DMA Descriptor */ 1537 desc = q->ring_addr + ring_offset; 1538 1539 ret = fpga_dma_desc_te_fill(op, &desc->enc_req, input, output, k, e, 1540 ncb, in_offset, out_offset, ring_offset, 1); 1541 if (unlikely(ret < 0)) 1542 return ret; 1543 1544 /* Update lengths */ 1545 total_left -= in_length; 1546 op->turbo_enc.output.length += out_length; 1547 1548 if (total_left > 0) { 1549 rte_bbdev_log(ERR, 1550 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u", 1551 total_left, in_length); 1552 return -1; 1553 } 1554 1555 return 1; 1556 } 1557 1558 static inline int 1559 enqueue_enc_one_op_tb(struct fpga_queue *q, struct rte_bbdev_enc_op *op, 1560 uint16_t desc_offset, uint8_t cbs_in_op) 1561 { 1562 union fpga_dma_desc *desc; 1563 struct rte_mbuf *input, *output_head, *output; 1564 int ret; 1565 uint8_t r, c, crc24_bits = 0; 1566 uint16_t k, e, ncb, ring_offset; 1567 uint32_t mbuf_total_left, in_length, out_length, in_offset, out_offset; 1568 uint32_t seg_total_left; 1569 uint16_t current_enqueued_cbs = 0; 1570 1571 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1572 /* Validate op structure */ 1573 if (validate_enc_op(op) == -1) { 1574 rte_bbdev_log(ERR, "Turbo encoder validation failed"); 1575 return -EINVAL; 1576 } 1577 #endif 1578 1579 input = op->turbo_enc.input.data; 1580 output_head = output = op->turbo_enc.output.data; 1581 in_offset = op->turbo_enc.input.offset; 1582 out_offset = op->turbo_enc.output.offset; 1583 mbuf_total_left = op->turbo_enc.input.length; 1584 1585 c = op->turbo_enc.tb_params.c; 1586 r = op->turbo_enc.tb_params.r; 1587 1588 if (check_bit(op->turbo_enc.op_flags, RTE_BBDEV_TURBO_CRC_24B_ATTACH)) 1589 crc24_bits = 24; 1590 1591 while (mbuf_total_left > 0 && r < c && input != NULL) { 1592 seg_total_left = rte_pktmbuf_data_len(input) - in_offset; 1593 1594 e = (r < op->turbo_enc.tb_params.cab) ? 1595 op->turbo_enc.tb_params.ea : 1596 op->turbo_enc.tb_params.eb; 1597 k = (r < op->turbo_enc.tb_params.c_neg) ? 1598 op->turbo_enc.tb_params.k_neg : 1599 op->turbo_enc.tb_params.k_pos; 1600 ncb = (r < op->turbo_enc.tb_params.c_neg) ? 1601 op->turbo_enc.tb_params.ncb_neg : 1602 op->turbo_enc.tb_params.ncb_pos; 1603 1604 in_length = ((k - crc24_bits) >> 3); 1605 1606 if (check_bit(op->turbo_enc.op_flags, 1607 RTE_BBDEV_TURBO_RATE_MATCH)) 1608 out_length = (e + 7) >> 3; 1609 else 1610 out_length = (k >> 3) * 3 + 2; 1611 1612 mbuf_append(output_head, output, out_length); 1613 1614 /* Setup DMA Descriptor */ 1615 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1616 desc = q->ring_addr + ring_offset; 1617 ret = fpga_dma_desc_te_fill(op, &desc->enc_req, input, output, 1618 k, e, ncb, in_offset, out_offset, ring_offset, 1619 cbs_in_op); 1620 if (unlikely(ret < 0)) 1621 return ret; 1622 1623 rte_bbdev_log_debug("DMA request desc %p", desc); 1624 1625 /* Update lengths */ 1626 op->turbo_enc.output.length += out_length; 1627 mbuf_total_left -= in_length; 1628 1629 /* Update offsets */ 1630 if (seg_total_left == in_length) { 1631 /* Go to the next mbuf */ 1632 input = input->next; 1633 output = output->next; 1634 in_offset = 0; 1635 out_offset = 0; 1636 } else { 1637 in_offset += in_length; 1638 out_offset += out_length; 1639 } 1640 1641 r++; 1642 desc_offset++; 1643 current_enqueued_cbs++; 1644 } 1645 1646 if (mbuf_total_left > 0) { 1647 rte_bbdev_log(ERR, 1648 "Some date still left for processing: mbuf_total_left = %u", 1649 mbuf_total_left); 1650 return -1; 1651 } 1652 1653 return current_enqueued_cbs; 1654 } 1655 1656 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1657 /* Validates turbo decoder parameters */ 1658 static int 1659 validate_dec_op(struct rte_bbdev_dec_op *op) 1660 { 1661 struct rte_bbdev_op_turbo_dec *turbo_dec = &op->turbo_dec; 1662 struct rte_bbdev_op_dec_turbo_cb_params *cb = NULL; 1663 struct rte_bbdev_op_dec_turbo_tb_params *tb = NULL; 1664 1665 if (op->mempool == NULL) { 1666 rte_bbdev_log(ERR, "Invalid mempool pointer"); 1667 return -1; 1668 } 1669 if (turbo_dec->input.data == NULL) { 1670 rte_bbdev_log(ERR, "Invalid input pointer"); 1671 return -1; 1672 } 1673 if (turbo_dec->hard_output.data == NULL) { 1674 rte_bbdev_log(ERR, "Invalid hard_output pointer"); 1675 return -1; 1676 } 1677 if (turbo_dec->rv_index > 3) { 1678 rte_bbdev_log(ERR, 1679 "rv_index (%u) is out of range 0 <= value <= 3", 1680 turbo_dec->rv_index); 1681 return -1; 1682 } 1683 if (turbo_dec->iter_min < 1) { 1684 rte_bbdev_log(ERR, 1685 "iter_min (%u) is less than 1", 1686 turbo_dec->iter_min); 1687 return -1; 1688 } 1689 if (turbo_dec->iter_max <= 2) { 1690 rte_bbdev_log(ERR, 1691 "iter_max (%u) is less than or equal to 2", 1692 turbo_dec->iter_max); 1693 return -1; 1694 } 1695 if (turbo_dec->iter_min > turbo_dec->iter_max) { 1696 rte_bbdev_log(ERR, 1697 "iter_min (%u) is greater than iter_max (%u)", 1698 turbo_dec->iter_min, turbo_dec->iter_max); 1699 return -1; 1700 } 1701 if (turbo_dec->code_block_mode != 0 && 1702 turbo_dec->code_block_mode != 1) { 1703 rte_bbdev_log(ERR, 1704 "code_block_mode (%u) is out of range 0 <= value <= 1", 1705 turbo_dec->code_block_mode); 1706 return -1; 1707 } 1708 1709 if (turbo_dec->code_block_mode == 0) { 1710 1711 if ((turbo_dec->op_flags & 1712 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP) && 1713 !(turbo_dec->op_flags & RTE_BBDEV_TURBO_CRC_TYPE_24B)) { 1714 rte_bbdev_log(ERR, 1715 "RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP should accompany RTE_BBDEV_TURBO_CRC_TYPE_24B"); 1716 return -1; 1717 } 1718 1719 tb = &turbo_dec->tb_params; 1720 if ((tb->k_neg < RTE_BBDEV_TURBO_MIN_CB_SIZE 1721 || tb->k_neg > RTE_BBDEV_TURBO_MAX_CB_SIZE) 1722 && tb->c_neg > 0) { 1723 rte_bbdev_log(ERR, 1724 "k_neg (%u) is out of range %u <= value <= %u", 1725 tb->k_neg, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1726 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1727 return -1; 1728 } 1729 if ((tb->k_pos < RTE_BBDEV_TURBO_MIN_CB_SIZE 1730 || tb->k_pos > RTE_BBDEV_TURBO_MAX_CB_SIZE) 1731 && tb->c > tb->c_neg) { 1732 rte_bbdev_log(ERR, 1733 "k_pos (%u) is out of range %u <= value <= %u", 1734 tb->k_pos, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1735 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1736 return -1; 1737 } 1738 if (tb->c_neg > (RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1)) 1739 rte_bbdev_log(ERR, 1740 "c_neg (%u) is out of range 0 <= value <= %u", 1741 tb->c_neg, 1742 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1); 1743 if (tb->c < 1 || tb->c > RTE_BBDEV_TURBO_MAX_CODE_BLOCKS) { 1744 rte_bbdev_log(ERR, 1745 "c (%u) is out of range 1 <= value <= %u", 1746 tb->c, RTE_BBDEV_TURBO_MAX_CODE_BLOCKS); 1747 return -1; 1748 } 1749 if (tb->cab > tb->c) { 1750 rte_bbdev_log(ERR, 1751 "cab (%u) is greater than c (%u)", 1752 tb->cab, tb->c); 1753 return -1; 1754 } 1755 } else { 1756 1757 if (turbo_dec->op_flags & RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP) { 1758 rte_bbdev_log(ERR, 1759 "RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP is invalid in CB-mode"); 1760 return -1; 1761 } 1762 1763 cb = &turbo_dec->cb_params; 1764 if (cb->k < RTE_BBDEV_TURBO_MIN_CB_SIZE 1765 || cb->k > RTE_BBDEV_TURBO_MAX_CB_SIZE) { 1766 rte_bbdev_log(ERR, 1767 "k (%u) is out of range %u <= value <= %u", 1768 cb->k, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1769 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1770 return -1; 1771 } 1772 } 1773 1774 return 0; 1775 } 1776 #endif 1777 1778 static inline int 1779 enqueue_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op *op, 1780 uint16_t desc_offset) 1781 { 1782 union fpga_dma_desc *desc; 1783 struct rte_mbuf *input; 1784 struct rte_mbuf *output; 1785 int ret; 1786 uint16_t k, kw, ring_offset; 1787 uint32_t total_left, in_length, out_length, in_offset, out_offset; 1788 1789 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1790 /* Validate op structure */ 1791 if (validate_dec_op(op) == -1) { 1792 rte_bbdev_log(ERR, "Turbo decoder validation failed"); 1793 return -EINVAL; 1794 } 1795 #endif 1796 1797 input = op->turbo_dec.input.data; 1798 output = op->turbo_dec.hard_output.data; 1799 total_left = op->turbo_dec.input.length; 1800 in_offset = op->turbo_dec.input.offset; 1801 out_offset = op->turbo_dec.hard_output.offset; 1802 1803 k = op->turbo_dec.cb_params.k; 1804 kw = RTE_ALIGN_CEIL(k + 4, 32) * 3; 1805 in_length = kw; 1806 out_length = k >> 3; 1807 1808 mbuf_append(output, output, out_length); 1809 1810 /* Setup DMA Descriptor */ 1811 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1812 desc = q->ring_addr + ring_offset; 1813 ret = fpga_dma_desc_td_fill(op, &desc->dec_req, input, output, 1814 in_length, k, in_offset, out_offset, ring_offset, 1); 1815 if (unlikely(ret < 0)) 1816 return ret; 1817 1818 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1819 print_dma_dec_desc_debug_info(desc); 1820 #endif 1821 1822 /* Update lengths */ 1823 total_left -= in_length; 1824 op->turbo_dec.hard_output.length += out_length; 1825 1826 if (total_left > 0) { 1827 rte_bbdev_log(ERR, 1828 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u", 1829 total_left, in_length); 1830 return -1; 1831 } 1832 1833 return 1; 1834 } 1835 1836 1837 static inline int 1838 enqueue_dec_one_op_tb(struct fpga_queue *q, struct rte_bbdev_dec_op *op, 1839 uint16_t desc_offset, uint8_t cbs_in_op) 1840 { 1841 union fpga_dma_desc *desc; 1842 struct rte_mbuf *input, *output_head, *output; 1843 int ret; 1844 uint8_t r, c; 1845 uint16_t k, kw, in_length, out_length, ring_offset; 1846 uint32_t mbuf_total_left, seg_total_left, in_offset, out_offset; 1847 uint16_t current_enqueued_cbs = 0; 1848 uint16_t crc24_overlap = 0; 1849 1850 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1851 /* Validate op structure */ 1852 if (validate_dec_op(op) == -1) { 1853 rte_bbdev_log(ERR, "Turbo decoder validation failed"); 1854 return -EINVAL; 1855 } 1856 #endif 1857 1858 input = op->turbo_dec.input.data; 1859 output_head = output = op->turbo_dec.hard_output.data; 1860 mbuf_total_left = op->turbo_dec.input.length; 1861 in_offset = op->turbo_dec.input.offset; 1862 out_offset = op->turbo_dec.hard_output.offset; 1863 1864 if (!check_bit(op->turbo_dec.op_flags, 1865 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP)) 1866 crc24_overlap = 24; 1867 1868 c = op->turbo_dec.tb_params.c; 1869 r = op->turbo_dec.tb_params.r; 1870 1871 while (mbuf_total_left > 0 && r < c && input != NULL) { 1872 seg_total_left = rte_pktmbuf_data_len(input) - in_offset; 1873 k = (r < op->turbo_dec.tb_params.c_neg) ? 1874 op->turbo_dec.tb_params.k_neg : 1875 op->turbo_dec.tb_params.k_pos; 1876 kw = RTE_ALIGN_CEIL(k + 4, 32) * 3; 1877 1878 in_length = kw; 1879 out_length = (k - crc24_overlap) >> 3; 1880 1881 mbuf_append(output_head, output, out_length); 1882 1883 if (seg_total_left < in_length) { 1884 rte_bbdev_log(ERR, 1885 "Partial CB found in a TB. FPGA Driver doesn't support scatter-gather operations!"); 1886 return -1; 1887 } 1888 1889 /* Setup DMA Descriptor */ 1890 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1891 desc = q->ring_addr + ring_offset; 1892 ret = fpga_dma_desc_td_fill(op, &desc->dec_req, input, output, 1893 in_length, k, in_offset, out_offset, 1894 ring_offset, cbs_in_op); 1895 if (unlikely(ret < 0)) 1896 return ret; 1897 1898 /* Update lengths */ 1899 ret = rte_pktmbuf_trim(op->turbo_dec.hard_output.data, 1900 (crc24_overlap >> 3)); 1901 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1902 if (ret < 0) { 1903 rte_bbdev_log(ERR, 1904 "The length to remove is greater than the length of the last segment"); 1905 return -EINVAL; 1906 } 1907 #endif 1908 op->turbo_dec.hard_output.length += out_length; 1909 mbuf_total_left -= in_length; 1910 1911 /* Update offsets */ 1912 if (seg_total_left == in_length) { 1913 /* Go to the next mbuf */ 1914 input = input->next; 1915 output = output->next; 1916 in_offset = 0; 1917 out_offset = 0; 1918 } else { 1919 in_offset += in_length; 1920 out_offset += out_length; 1921 } 1922 1923 r++; 1924 desc_offset++; 1925 current_enqueued_cbs++; 1926 } 1927 1928 if (mbuf_total_left > 0) { 1929 rte_bbdev_log(ERR, 1930 "Some date still left for processing: mbuf_total_left = %u", 1931 mbuf_total_left); 1932 return -1; 1933 } 1934 1935 return current_enqueued_cbs; 1936 } 1937 1938 static uint16_t 1939 fpga_enqueue_enc(struct rte_bbdev_queue_data *q_data, 1940 struct rte_bbdev_enc_op **ops, uint16_t num) 1941 { 1942 uint8_t cbs_in_op; 1943 uint16_t i, total_enqueued_cbs = 0; 1944 int32_t avail; 1945 int enqueued_cbs; 1946 struct fpga_queue *q = q_data->queue_private; 1947 union fpga_dma_desc *desc; 1948 1949 /* Check if queue is not full */ 1950 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) == 1951 q->head_free_desc)) 1952 return 0; 1953 1954 /* Calculates available space */ 1955 avail = (q->head_free_desc > q->tail) ? 1956 q->head_free_desc - q->tail - 1 : 1957 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1; 1958 1959 for (i = 0; i < num; ++i) { 1960 if (ops[i]->turbo_enc.code_block_mode == 0) { 1961 cbs_in_op = get_num_cbs_in_op_enc(&ops[i]->turbo_enc); 1962 /* Check if there is available space for further 1963 * processing 1964 */ 1965 if (unlikely(avail - cbs_in_op < 0)) 1966 break; 1967 avail -= cbs_in_op; 1968 enqueued_cbs = enqueue_enc_one_op_tb(q, ops[i], 1969 total_enqueued_cbs, cbs_in_op); 1970 } else { 1971 /* Check if there is available space for further 1972 * processing 1973 */ 1974 if (unlikely(avail - 1 < 0)) 1975 break; 1976 avail -= 1; 1977 enqueued_cbs = enqueue_enc_one_op_cb(q, ops[i], 1978 total_enqueued_cbs); 1979 } 1980 1981 if (enqueued_cbs < 0) 1982 break; 1983 1984 total_enqueued_cbs += enqueued_cbs; 1985 1986 rte_bbdev_log_debug("enqueuing enc ops [%d/%d] | head %d | tail %d", 1987 total_enqueued_cbs, num, 1988 q->head_free_desc, q->tail); 1989 } 1990 1991 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt 1992 * only when all previous CBs were already processed. 1993 */ 1994 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1) 1995 & q->sw_ring_wrap_mask); 1996 desc->enc_req.irq_en = q->irq_enable; 1997 1998 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats); 1999 2000 /* Update stats */ 2001 q_data->queue_stats.enqueued_count += i; 2002 q_data->queue_stats.enqueue_err_count += num - i; 2003 2004 return i; 2005 } 2006 2007 static uint16_t 2008 fpga_enqueue_dec(struct rte_bbdev_queue_data *q_data, 2009 struct rte_bbdev_dec_op **ops, uint16_t num) 2010 { 2011 uint8_t cbs_in_op; 2012 uint16_t i, total_enqueued_cbs = 0; 2013 int32_t avail; 2014 int enqueued_cbs; 2015 struct fpga_queue *q = q_data->queue_private; 2016 union fpga_dma_desc *desc; 2017 2018 /* Check if queue is not full */ 2019 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) == 2020 q->head_free_desc)) 2021 return 0; 2022 2023 /* Calculates available space */ 2024 avail = (q->head_free_desc > q->tail) ? 2025 q->head_free_desc - q->tail - 1 : 2026 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1; 2027 2028 for (i = 0; i < num; ++i) { 2029 if (ops[i]->turbo_dec.code_block_mode == 0) { 2030 cbs_in_op = get_num_cbs_in_op_dec(&ops[i]->turbo_dec); 2031 /* Check if there is available space for further 2032 * processing 2033 */ 2034 if (unlikely(avail - cbs_in_op < 0)) 2035 break; 2036 avail -= cbs_in_op; 2037 enqueued_cbs = enqueue_dec_one_op_tb(q, ops[i], 2038 total_enqueued_cbs, cbs_in_op); 2039 } else { 2040 /* Check if there is available space for further 2041 * processing 2042 */ 2043 if (unlikely(avail - 1 < 0)) 2044 break; 2045 avail -= 1; 2046 enqueued_cbs = enqueue_dec_one_op_cb(q, ops[i], 2047 total_enqueued_cbs); 2048 } 2049 2050 if (enqueued_cbs < 0) 2051 break; 2052 2053 total_enqueued_cbs += enqueued_cbs; 2054 2055 rte_bbdev_log_debug("enqueuing dec ops [%d/%d] | head %d | tail %d", 2056 total_enqueued_cbs, num, 2057 q->head_free_desc, q->tail); 2058 } 2059 2060 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt 2061 * only when all previous CBs were already processed. 2062 */ 2063 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1) 2064 & q->sw_ring_wrap_mask); 2065 desc->dec_req.irq_en = q->irq_enable; 2066 2067 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats); 2068 2069 /* Update stats */ 2070 q_data->queue_stats.enqueued_count += i; 2071 q_data->queue_stats.enqueue_err_count += num - i; 2072 2073 return i; 2074 } 2075 2076 static inline int 2077 dequeue_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op **op, 2078 uint16_t desc_offset) 2079 { 2080 union fpga_dma_desc *desc; 2081 int desc_error = 0; 2082 2083 /* Set current desc */ 2084 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2085 & q->sw_ring_wrap_mask); 2086 2087 /*check if done */ 2088 if (desc->enc_req.done == 0) 2089 return -1; 2090 2091 /* make sure the response is read atomically */ 2092 rte_smp_rmb(); 2093 2094 rte_bbdev_log_debug("DMA response desc %p", desc); 2095 2096 *op = desc->enc_req.op_addr; 2097 /* Check the decriptor error field, return 1 on error */ 2098 desc_error = check_desc_error(desc->enc_req.error); 2099 (*op)->status = desc_error << RTE_BBDEV_DATA_ERROR; 2100 2101 return 1; 2102 } 2103 2104 static inline int 2105 dequeue_enc_one_op_tb(struct fpga_queue *q, struct rte_bbdev_enc_op **op, 2106 uint16_t desc_offset) 2107 { 2108 union fpga_dma_desc *desc; 2109 uint8_t cbs_in_op, cb_idx; 2110 int desc_error = 0; 2111 int status = 0; 2112 2113 /* Set descriptor */ 2114 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2115 & q->sw_ring_wrap_mask); 2116 2117 /* Verify if done bit is set */ 2118 if (desc->enc_req.done == 0) 2119 return -1; 2120 2121 /* Make sure the response is read atomically */ 2122 rte_smp_rmb(); 2123 2124 /* Verify if done bit in all CBs is set */ 2125 cbs_in_op = desc->enc_req.cbs_in_op; 2126 for (cb_idx = 1; cb_idx < cbs_in_op; ++cb_idx) { 2127 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2128 cb_idx) & q->sw_ring_wrap_mask); 2129 if (desc->enc_req.done == 0) 2130 return -1; 2131 } 2132 2133 /* Make sure the response is read atomically */ 2134 rte_smp_rmb(); 2135 2136 for (cb_idx = 0; cb_idx < cbs_in_op; ++cb_idx) { 2137 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2138 cb_idx) & q->sw_ring_wrap_mask); 2139 /* Check the decriptor error field, return 1 on error */ 2140 desc_error = check_desc_error(desc->enc_req.error); 2141 status |= desc_error << RTE_BBDEV_DATA_ERROR; 2142 rte_bbdev_log_debug("DMA response desc %p", desc); 2143 } 2144 2145 *op = desc->enc_req.op_addr; 2146 (*op)->status = status; 2147 return cbs_in_op; 2148 } 2149 2150 static inline int 2151 dequeue_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op **op, 2152 uint16_t desc_offset) 2153 { 2154 union fpga_dma_desc *desc; 2155 int desc_error = 0; 2156 /* Set descriptor */ 2157 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2158 & q->sw_ring_wrap_mask); 2159 2160 /* Verify done bit is set */ 2161 if (desc->dec_req.done == 0) 2162 return -1; 2163 2164 /* make sure the response is read atomically */ 2165 rte_smp_rmb(); 2166 2167 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2168 print_dma_dec_desc_debug_info(desc); 2169 2170 #endif 2171 2172 *op = desc->dec_req.op_addr; 2173 /* FPGA reports in half-iterations, from 0 to 31. get ceiling */ 2174 (*op)->turbo_dec.iter_count = (desc->dec_req.iter + 2) >> 1; 2175 /* crc_pass = 0 when decoder fails */ 2176 (*op)->status = !(desc->dec_req.crc_pass) << RTE_BBDEV_CRC_ERROR; 2177 /* Check the decriptor error field, return 1 on error */ 2178 desc_error = check_desc_error(desc->enc_req.error); 2179 (*op)->status |= desc_error << RTE_BBDEV_DATA_ERROR; 2180 return 1; 2181 } 2182 2183 static inline int 2184 dequeue_dec_one_op_tb(struct fpga_queue *q, struct rte_bbdev_dec_op **op, 2185 uint16_t desc_offset) 2186 { 2187 union fpga_dma_desc *desc; 2188 uint8_t cbs_in_op, cb_idx, iter_count = 0; 2189 int status = 0; 2190 int desc_error = 0; 2191 /* Set descriptor */ 2192 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2193 & q->sw_ring_wrap_mask); 2194 2195 /* Verify if done bit is set */ 2196 if (desc->dec_req.done == 0) 2197 return -1; 2198 2199 /* Make sure the response is read atomically */ 2200 rte_smp_rmb(); 2201 2202 /* Verify if done bit in all CBs is set */ 2203 cbs_in_op = desc->dec_req.cbs_in_op; 2204 for (cb_idx = 1; cb_idx < cbs_in_op; ++cb_idx) { 2205 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2206 cb_idx) & q->sw_ring_wrap_mask); 2207 if (desc->dec_req.done == 0) 2208 return -1; 2209 } 2210 2211 /* Make sure the response is read atomically */ 2212 rte_smp_rmb(); 2213 2214 for (cb_idx = 0; cb_idx < cbs_in_op; ++cb_idx) { 2215 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2216 cb_idx) & q->sw_ring_wrap_mask); 2217 /* get max iter_count for all CBs in op */ 2218 iter_count = RTE_MAX(iter_count, (uint8_t) desc->dec_req.iter); 2219 /* crc_pass = 0 when decoder fails, one fails all */ 2220 status |= !(desc->dec_req.crc_pass) << RTE_BBDEV_CRC_ERROR; 2221 /* Check the decriptor error field, return 1 on error */ 2222 desc_error = check_desc_error(desc->enc_req.error); 2223 status |= desc_error << RTE_BBDEV_DATA_ERROR; 2224 rte_bbdev_log_debug("DMA response desc %p", desc); 2225 } 2226 2227 *op = desc->dec_req.op_addr; 2228 2229 /* FPGA reports in half-iterations, get ceiling */ 2230 (*op)->turbo_dec.iter_count = (iter_count + 2) >> 1; 2231 (*op)->status = status; 2232 return cbs_in_op; 2233 } 2234 2235 static uint16_t 2236 fpga_dequeue_enc(struct rte_bbdev_queue_data *q_data, 2237 struct rte_bbdev_enc_op **ops, uint16_t num) 2238 { 2239 struct fpga_queue *q = q_data->queue_private; 2240 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask; 2241 uint16_t i; 2242 uint16_t dequeued_cbs = 0; 2243 struct rte_bbdev_enc_op *op; 2244 int ret; 2245 2246 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) { 2247 op = (q->ring_addr + ((q->head_free_desc + dequeued_cbs) 2248 & q->sw_ring_wrap_mask))->enc_req.op_addr; 2249 if (op->turbo_enc.code_block_mode == 0) 2250 ret = dequeue_enc_one_op_tb(q, &ops[i], dequeued_cbs); 2251 else 2252 ret = dequeue_enc_one_op_cb(q, &ops[i], dequeued_cbs); 2253 2254 if (ret < 0) 2255 break; 2256 2257 dequeued_cbs += ret; 2258 2259 rte_bbdev_log_debug("dequeuing enc ops [%d/%d] | head %d | tail %d", 2260 dequeued_cbs, num, q->head_free_desc, q->tail); 2261 } 2262 2263 /* Update head */ 2264 q->head_free_desc = (q->head_free_desc + dequeued_cbs) & 2265 q->sw_ring_wrap_mask; 2266 2267 /* Update stats */ 2268 q_data->queue_stats.dequeued_count += i; 2269 2270 return i; 2271 } 2272 2273 static uint16_t 2274 fpga_dequeue_dec(struct rte_bbdev_queue_data *q_data, 2275 struct rte_bbdev_dec_op **ops, uint16_t num) 2276 { 2277 struct fpga_queue *q = q_data->queue_private; 2278 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask; 2279 uint16_t i; 2280 uint16_t dequeued_cbs = 0; 2281 struct rte_bbdev_dec_op *op; 2282 int ret; 2283 2284 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) { 2285 op = (q->ring_addr + ((q->head_free_desc + dequeued_cbs) 2286 & q->sw_ring_wrap_mask))->dec_req.op_addr; 2287 if (op->turbo_dec.code_block_mode == 0) 2288 ret = dequeue_dec_one_op_tb(q, &ops[i], dequeued_cbs); 2289 else 2290 ret = dequeue_dec_one_op_cb(q, &ops[i], dequeued_cbs); 2291 2292 if (ret < 0) 2293 break; 2294 2295 dequeued_cbs += ret; 2296 2297 rte_bbdev_log_debug("dequeuing dec ops [%d/%d] | head %d | tail %d", 2298 dequeued_cbs, num, q->head_free_desc, q->tail); 2299 } 2300 2301 /* Update head */ 2302 q->head_free_desc = (q->head_free_desc + dequeued_cbs) & 2303 q->sw_ring_wrap_mask; 2304 2305 /* Update stats */ 2306 q_data->queue_stats.dequeued_count += i; 2307 2308 return i; 2309 } 2310 2311 /* Initialization Function */ 2312 static void 2313 fpga_lte_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv) 2314 { 2315 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); 2316 2317 dev->dev_ops = &fpga_ops; 2318 dev->enqueue_enc_ops = fpga_enqueue_enc; 2319 dev->enqueue_dec_ops = fpga_enqueue_dec; 2320 dev->dequeue_enc_ops = fpga_dequeue_enc; 2321 dev->dequeue_dec_ops = fpga_dequeue_dec; 2322 2323 ((struct fpga_lte_fec_device *) dev->data->dev_private)->pf_device = 2324 !strcmp(drv->driver.name, 2325 RTE_STR(FPGA_LTE_FEC_PF_DRIVER_NAME)); 2326 ((struct fpga_lte_fec_device *) dev->data->dev_private)->mmio_base = 2327 pci_dev->mem_resource[0].addr; 2328 2329 rte_bbdev_log_debug( 2330 "Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64, 2331 dev->device->driver->name, dev->data->name, 2332 (void *)pci_dev->mem_resource[0].addr, 2333 pci_dev->mem_resource[0].phys_addr); 2334 } 2335 2336 static int 2337 fpga_lte_fec_probe(struct rte_pci_driver *pci_drv, 2338 struct rte_pci_device *pci_dev) 2339 { 2340 struct rte_bbdev *bbdev = NULL; 2341 char dev_name[RTE_BBDEV_NAME_MAX_LEN]; 2342 2343 if (pci_dev == NULL) { 2344 rte_bbdev_log(ERR, "NULL PCI device"); 2345 return -EINVAL; 2346 } 2347 2348 rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name)); 2349 2350 /* Allocate memory to be used privately by drivers */ 2351 bbdev = rte_bbdev_allocate(pci_dev->device.name); 2352 if (bbdev == NULL) 2353 return -ENODEV; 2354 2355 /* allocate device private memory */ 2356 bbdev->data->dev_private = rte_zmalloc_socket(dev_name, 2357 sizeof(struct fpga_lte_fec_device), RTE_CACHE_LINE_SIZE, 2358 pci_dev->device.numa_node); 2359 2360 if (bbdev->data->dev_private == NULL) { 2361 rte_bbdev_log(CRIT, 2362 "Allocate of %zu bytes for device \"%s\" failed", 2363 sizeof(struct fpga_lte_fec_device), dev_name); 2364 rte_bbdev_release(bbdev); 2365 return -ENOMEM; 2366 } 2367 2368 /* Fill HW specific part of device structure */ 2369 bbdev->device = &pci_dev->device; 2370 bbdev->intr_handle = &pci_dev->intr_handle; 2371 bbdev->data->socket_id = pci_dev->device.numa_node; 2372 2373 /* Invoke FEC FPGA device initialization function */ 2374 fpga_lte_fec_init(bbdev, pci_drv); 2375 2376 rte_bbdev_log_debug("bbdev id = %u [%s]", 2377 bbdev->data->dev_id, dev_name); 2378 2379 struct fpga_lte_fec_device *d = bbdev->data->dev_private; 2380 uint32_t version_id = fpga_reg_read_32(d->mmio_base, 2381 FPGA_LTE_FEC_VERSION_ID); 2382 rte_bbdev_log(INFO, "FEC FPGA RTL v%u.%u", 2383 ((uint16_t)(version_id >> 16)), ((uint16_t)version_id)); 2384 2385 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2386 if (!strcmp(bbdev->device->driver->name, 2387 RTE_STR(FPGA_LTE_FEC_PF_DRIVER_NAME))) 2388 print_static_reg_debug_info(d->mmio_base); 2389 #endif 2390 return 0; 2391 } 2392 2393 static int 2394 fpga_lte_fec_remove(struct rte_pci_device *pci_dev) 2395 { 2396 struct rte_bbdev *bbdev; 2397 int ret; 2398 uint8_t dev_id; 2399 2400 if (pci_dev == NULL) 2401 return -EINVAL; 2402 2403 /* Find device */ 2404 bbdev = rte_bbdev_get_named_dev(pci_dev->device.name); 2405 if (bbdev == NULL) { 2406 rte_bbdev_log(CRIT, 2407 "Couldn't find HW dev \"%s\" to uninitialise it", 2408 pci_dev->device.name); 2409 return -ENODEV; 2410 } 2411 dev_id = bbdev->data->dev_id; 2412 2413 /* free device private memory before close */ 2414 rte_free(bbdev->data->dev_private); 2415 2416 /* Close device */ 2417 ret = rte_bbdev_close(dev_id); 2418 if (ret < 0) 2419 rte_bbdev_log(ERR, 2420 "Device %i failed to close during uninit: %i", 2421 dev_id, ret); 2422 2423 /* release bbdev from library */ 2424 ret = rte_bbdev_release(bbdev); 2425 if (ret) 2426 rte_bbdev_log(ERR, "Device %i failed to uninit: %i", dev_id, 2427 ret); 2428 2429 rte_bbdev_log_debug("Destroyed bbdev = %u", dev_id); 2430 2431 return 0; 2432 } 2433 2434 static inline void 2435 set_default_fpga_conf(struct fpga_lte_fec_conf *def_conf) 2436 { 2437 /* clear default configuration before initialization */ 2438 memset(def_conf, 0, sizeof(struct fpga_lte_fec_conf)); 2439 /* Set pf mode to true */ 2440 def_conf->pf_mode_en = true; 2441 2442 /* Set ratio between UL and DL to 1:1 (unit of weight is 3 CBs) */ 2443 def_conf->ul_bandwidth = 3; 2444 def_conf->dl_bandwidth = 3; 2445 2446 /* Set Load Balance Factor to 64 */ 2447 def_conf->dl_load_balance = 64; 2448 def_conf->ul_load_balance = 64; 2449 } 2450 2451 /* Initial configuration of FPGA LTE FEC device */ 2452 int 2453 fpga_lte_fec_configure(const char *dev_name, 2454 const struct fpga_lte_fec_conf *conf) 2455 { 2456 uint32_t payload_32, address; 2457 uint16_t payload_16; 2458 uint8_t payload_8; 2459 uint16_t q_id, vf_id, total_q_id, total_ul_q_id, total_dl_q_id; 2460 struct rte_bbdev *bbdev = rte_bbdev_get_named_dev(dev_name); 2461 struct fpga_lte_fec_conf def_conf; 2462 2463 if (bbdev == NULL) { 2464 rte_bbdev_log(ERR, 2465 "Invalid dev_name (%s), or device is not yet initialised", 2466 dev_name); 2467 return -ENODEV; 2468 } 2469 2470 struct fpga_lte_fec_device *d = bbdev->data->dev_private; 2471 2472 if (conf == NULL) { 2473 rte_bbdev_log(ERR, 2474 "FPGA Configuration was not provided. Default configuration will be loaded."); 2475 set_default_fpga_conf(&def_conf); 2476 conf = &def_conf; 2477 } 2478 2479 /* 2480 * Configure UL:DL ratio. 2481 * [7:0]: UL weight 2482 * [15:8]: DL weight 2483 */ 2484 payload_16 = (conf->dl_bandwidth << 8) | conf->ul_bandwidth; 2485 address = FPGA_LTE_FEC_CONFIGURATION; 2486 fpga_reg_write_16(d->mmio_base, address, payload_16); 2487 2488 /* Clear all queues registers */ 2489 payload_32 = FPGA_INVALID_HW_QUEUE_ID; 2490 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 2491 address = (q_id << 2) + FPGA_LTE_FEC_QUEUE_MAP; 2492 fpga_reg_write_32(d->mmio_base, address, payload_32); 2493 } 2494 2495 /* 2496 * If PF mode is enabled allocate all queues for PF only. 2497 * 2498 * For VF mode each VF can have different number of UL and DL queues. 2499 * Total number of queues to configure cannot exceed FPGA 2500 * capabilities - 64 queues - 32 queues for UL and 32 queues for DL. 2501 * Queues mapping is done according to configuration: 2502 * 2503 * UL queues: 2504 * | Q_ID | VF_ID | 2505 * | 0 | 0 | 2506 * | ... | 0 | 2507 * | conf->vf_dl_queues_number[0] - 1 | 0 | 2508 * | conf->vf_dl_queues_number[0] | 1 | 2509 * | ... | 1 | 2510 * | conf->vf_dl_queues_number[1] - 1 | 1 | 2511 * | ... | ... | 2512 * | conf->vf_dl_queues_number[7] - 1 | 7 | 2513 * 2514 * DL queues: 2515 * | Q_ID | VF_ID | 2516 * | 32 | 0 | 2517 * | ... | 0 | 2518 * | conf->vf_ul_queues_number[0] - 1 | 0 | 2519 * | conf->vf_ul_queues_number[0] | 1 | 2520 * | ... | 1 | 2521 * | conf->vf_ul_queues_number[1] - 1 | 1 | 2522 * | ... | ... | 2523 * | conf->vf_ul_queues_number[7] - 1 | 7 | 2524 * 2525 * Example of configuration: 2526 * conf->vf_ul_queues_number[0] = 4; -> 4 UL queues for VF0 2527 * conf->vf_dl_queues_number[0] = 4; -> 4 DL queues for VF0 2528 * conf->vf_ul_queues_number[1] = 2; -> 2 UL queues for VF1 2529 * conf->vf_dl_queues_number[1] = 2; -> 2 DL queues for VF1 2530 * 2531 * UL: 2532 * | Q_ID | VF_ID | 2533 * | 0 | 0 | 2534 * | 1 | 0 | 2535 * | 2 | 0 | 2536 * | 3 | 0 | 2537 * | 4 | 1 | 2538 * | 5 | 1 | 2539 * 2540 * DL: 2541 * | Q_ID | VF_ID | 2542 * | 32 | 0 | 2543 * | 33 | 0 | 2544 * | 34 | 0 | 2545 * | 35 | 0 | 2546 * | 36 | 1 | 2547 * | 37 | 1 | 2548 */ 2549 if (conf->pf_mode_en) { 2550 payload_32 = 0x1; 2551 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 2552 address = (q_id << 2) + FPGA_LTE_FEC_QUEUE_MAP; 2553 fpga_reg_write_32(d->mmio_base, address, payload_32); 2554 } 2555 } else { 2556 /* Calculate total number of UL and DL queues to configure */ 2557 total_ul_q_id = total_dl_q_id = 0; 2558 for (vf_id = 0; vf_id < FPGA_LTE_FEC_NUM_VFS; ++vf_id) { 2559 total_ul_q_id += conf->vf_ul_queues_number[vf_id]; 2560 total_dl_q_id += conf->vf_dl_queues_number[vf_id]; 2561 } 2562 total_q_id = total_dl_q_id + total_ul_q_id; 2563 /* 2564 * Check if total number of queues to configure does not exceed 2565 * FPGA capabilities (64 queues - 32 UL and 32 DL queues) 2566 */ 2567 if ((total_ul_q_id > FPGA_NUM_UL_QUEUES) || 2568 (total_dl_q_id > FPGA_NUM_DL_QUEUES) || 2569 (total_q_id > FPGA_TOTAL_NUM_QUEUES)) { 2570 rte_bbdev_log(ERR, 2571 "FPGA Configuration failed. Too many queues to configure: UL_Q %u, DL_Q %u, FPGA_Q %u", 2572 total_ul_q_id, total_dl_q_id, 2573 FPGA_TOTAL_NUM_QUEUES); 2574 return -EINVAL; 2575 } 2576 total_ul_q_id = 0; 2577 for (vf_id = 0; vf_id < FPGA_LTE_FEC_NUM_VFS; ++vf_id) { 2578 for (q_id = 0; q_id < conf->vf_ul_queues_number[vf_id]; 2579 ++q_id, ++total_ul_q_id) { 2580 address = (total_ul_q_id << 2) + 2581 FPGA_LTE_FEC_QUEUE_MAP; 2582 payload_32 = ((0x80 + vf_id) << 16) | 0x1; 2583 fpga_reg_write_32(d->mmio_base, address, 2584 payload_32); 2585 } 2586 } 2587 total_dl_q_id = 0; 2588 for (vf_id = 0; vf_id < FPGA_LTE_FEC_NUM_VFS; ++vf_id) { 2589 for (q_id = 0; q_id < conf->vf_dl_queues_number[vf_id]; 2590 ++q_id, ++total_dl_q_id) { 2591 address = ((total_dl_q_id + FPGA_NUM_UL_QUEUES) 2592 << 2) + FPGA_LTE_FEC_QUEUE_MAP; 2593 payload_32 = ((0x80 + vf_id) << 16) | 0x1; 2594 fpga_reg_write_32(d->mmio_base, address, 2595 payload_32); 2596 } 2597 } 2598 } 2599 2600 /* Setting Load Balance Factor */ 2601 payload_16 = (conf->dl_load_balance << 8) | (conf->ul_load_balance); 2602 address = FPGA_LTE_FEC_LOAD_BALANCE_FACTOR; 2603 fpga_reg_write_16(d->mmio_base, address, payload_16); 2604 2605 /* Setting length of ring descriptor entry */ 2606 payload_16 = FPGA_RING_DESC_ENTRY_LENGTH; 2607 address = FPGA_LTE_FEC_RING_DESC_LEN; 2608 fpga_reg_write_16(d->mmio_base, address, payload_16); 2609 2610 /* Setting FLR timeout value */ 2611 payload_16 = conf->flr_time_out; 2612 address = FPGA_LTE_FEC_FLR_TIME_OUT; 2613 fpga_reg_write_16(d->mmio_base, address, payload_16); 2614 2615 /* Queue PF/VF mapping table is ready */ 2616 payload_8 = 0x1; 2617 address = FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE; 2618 fpga_reg_write_8(d->mmio_base, address, payload_8); 2619 2620 rte_bbdev_log_debug("PF FPGA LTE FEC configuration complete for %s", 2621 dev_name); 2622 2623 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2624 print_static_reg_debug_info(d->mmio_base); 2625 #endif 2626 return 0; 2627 } 2628 2629 /* FPGA LTE FEC PCI PF address map */ 2630 static struct rte_pci_id pci_id_fpga_lte_fec_pf_map[] = { 2631 { 2632 RTE_PCI_DEVICE(FPGA_LTE_FEC_VENDOR_ID, 2633 FPGA_LTE_FEC_PF_DEVICE_ID) 2634 }, 2635 {.device_id = 0}, 2636 }; 2637 2638 static struct rte_pci_driver fpga_lte_fec_pci_pf_driver = { 2639 .probe = fpga_lte_fec_probe, 2640 .remove = fpga_lte_fec_remove, 2641 .id_table = pci_id_fpga_lte_fec_pf_map, 2642 .drv_flags = RTE_PCI_DRV_NEED_MAPPING 2643 }; 2644 2645 /* FPGA LTE FEC PCI VF address map */ 2646 static struct rte_pci_id pci_id_fpga_lte_fec_vf_map[] = { 2647 { 2648 RTE_PCI_DEVICE(FPGA_LTE_FEC_VENDOR_ID, 2649 FPGA_LTE_FEC_VF_DEVICE_ID) 2650 }, 2651 {.device_id = 0}, 2652 }; 2653 2654 static struct rte_pci_driver fpga_lte_fec_pci_vf_driver = { 2655 .probe = fpga_lte_fec_probe, 2656 .remove = fpga_lte_fec_remove, 2657 .id_table = pci_id_fpga_lte_fec_vf_map, 2658 .drv_flags = RTE_PCI_DRV_NEED_MAPPING 2659 }; 2660 2661 2662 RTE_PMD_REGISTER_PCI(FPGA_LTE_FEC_PF_DRIVER_NAME, fpga_lte_fec_pci_pf_driver); 2663 RTE_PMD_REGISTER_PCI_TABLE(FPGA_LTE_FEC_PF_DRIVER_NAME, 2664 pci_id_fpga_lte_fec_pf_map); 2665 RTE_PMD_REGISTER_PCI(FPGA_LTE_FEC_VF_DRIVER_NAME, fpga_lte_fec_pci_vf_driver); 2666 RTE_PMD_REGISTER_PCI_TABLE(FPGA_LTE_FEC_VF_DRIVER_NAME, 2667 pci_id_fpga_lte_fec_vf_map); 2668