1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #include <unistd.h> 6 7 #include <rte_common.h> 8 #include <rte_log.h> 9 #include <dev_driver.h> 10 #include <rte_malloc.h> 11 #include <rte_mempool.h> 12 #include <rte_errno.h> 13 #include <rte_pci.h> 14 #include <bus_pci_driver.h> 15 #include <rte_byteorder.h> 16 #include <rte_cycles.h> 17 18 #include <rte_bbdev.h> 19 #include <rte_bbdev_pmd.h> 20 21 #include "fpga_lte_fec.h" 22 23 #ifdef RTE_LIBRTE_BBDEV_DEBUG 24 RTE_LOG_REGISTER_DEFAULT(fpga_lte_fec_logtype, DEBUG); 25 #else 26 RTE_LOG_REGISTER_DEFAULT(fpga_lte_fec_logtype, NOTICE); 27 #endif 28 29 /* Helper macro for logging */ 30 #define rte_bbdev_log(level, fmt, ...) \ 31 rte_log(RTE_LOG_ ## level, fpga_lte_fec_logtype, fmt "\n", \ 32 ##__VA_ARGS__) 33 34 #ifdef RTE_LIBRTE_BBDEV_DEBUG 35 #define rte_bbdev_log_debug(fmt, ...) \ 36 rte_bbdev_log(DEBUG, "fpga_lte_fec: " fmt, \ 37 ##__VA_ARGS__) 38 #else 39 #define rte_bbdev_log_debug(fmt, ...) 40 #endif 41 42 /* FPGA LTE FEC driver names */ 43 #define FPGA_LTE_FEC_PF_DRIVER_NAME intel_fpga_lte_fec_pf 44 #define FPGA_LTE_FEC_VF_DRIVER_NAME intel_fpga_lte_fec_vf 45 46 /* FPGA LTE FEC PCI vendor & device IDs */ 47 #define FPGA_LTE_FEC_VENDOR_ID (0x1172) 48 #define FPGA_LTE_FEC_PF_DEVICE_ID (0x5052) 49 #define FPGA_LTE_FEC_VF_DEVICE_ID (0x5050) 50 51 /* Align DMA descriptors to 256 bytes - cache-aligned */ 52 #define FPGA_RING_DESC_ENTRY_LENGTH (8) 53 /* Ring size is in 256 bits (32 bytes) units */ 54 #define FPGA_RING_DESC_LEN_UNIT_BYTES (32) 55 /* Maximum size of queue */ 56 #define FPGA_RING_MAX_SIZE (1024) 57 #define FPGA_FLR_TIMEOUT_UNIT (16.384) 58 59 #define FPGA_NUM_UL_QUEUES (32) 60 #define FPGA_NUM_DL_QUEUES (32) 61 #define FPGA_TOTAL_NUM_QUEUES (FPGA_NUM_UL_QUEUES + FPGA_NUM_DL_QUEUES) 62 #define FPGA_NUM_INTR_VEC (FPGA_TOTAL_NUM_QUEUES - RTE_INTR_VEC_RXTX_OFFSET) 63 64 #define FPGA_INVALID_HW_QUEUE_ID (0xFFFFFFFF) 65 66 #define FPGA_QUEUE_FLUSH_TIMEOUT_US (1000) 67 #define FPGA_TIMEOUT_CHECK_INTERVAL (5) 68 69 /* FPGA LTE FEC Register mapping on BAR0 */ 70 enum { 71 FPGA_LTE_FEC_VERSION_ID = 0x00000000, /* len: 4B */ 72 FPGA_LTE_FEC_CONFIGURATION = 0x00000004, /* len: 2B */ 73 FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE = 0x00000008, /* len: 1B */ 74 FPGA_LTE_FEC_LOAD_BALANCE_FACTOR = 0x0000000a, /* len: 2B */ 75 FPGA_LTE_FEC_RING_DESC_LEN = 0x0000000c, /* len: 2B */ 76 FPGA_LTE_FEC_FLR_TIME_OUT = 0x0000000e, /* len: 2B */ 77 FPGA_LTE_FEC_VFQ_FLUSH_STATUS_LW = 0x00000018, /* len: 4B */ 78 FPGA_LTE_FEC_VFQ_FLUSH_STATUS_HI = 0x0000001c, /* len: 4B */ 79 FPGA_LTE_FEC_VF0_DEBUG = 0x00000020, /* len: 4B */ 80 FPGA_LTE_FEC_VF1_DEBUG = 0x00000024, /* len: 4B */ 81 FPGA_LTE_FEC_VF2_DEBUG = 0x00000028, /* len: 4B */ 82 FPGA_LTE_FEC_VF3_DEBUG = 0x0000002c, /* len: 4B */ 83 FPGA_LTE_FEC_VF4_DEBUG = 0x00000030, /* len: 4B */ 84 FPGA_LTE_FEC_VF5_DEBUG = 0x00000034, /* len: 4B */ 85 FPGA_LTE_FEC_VF6_DEBUG = 0x00000038, /* len: 4B */ 86 FPGA_LTE_FEC_VF7_DEBUG = 0x0000003c, /* len: 4B */ 87 FPGA_LTE_FEC_QUEUE_MAP = 0x00000040, /* len: 256B */ 88 FPGA_LTE_FEC_RING_CTRL_REGS = 0x00000200 /* len: 2048B */ 89 }; 90 91 /* FPGA LTE FEC Ring Control Registers */ 92 enum { 93 FPGA_LTE_FEC_RING_HEAD_ADDR = 0x00000008, 94 FPGA_LTE_FEC_RING_SIZE = 0x00000010, 95 FPGA_LTE_FEC_RING_MISC = 0x00000014, 96 FPGA_LTE_FEC_RING_ENABLE = 0x00000015, 97 FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN = 0x00000016, 98 FPGA_LTE_FEC_RING_SHADOW_TAIL = 0x00000018, 99 FPGA_LTE_FEC_RING_HEAD_POINT = 0x0000001C 100 }; 101 102 /* FPGA LTE FEC DESCRIPTOR ERROR */ 103 enum { 104 DESC_ERR_NO_ERR = 0x0, 105 DESC_ERR_K_OUT_OF_RANGE = 0x1, 106 DESC_ERR_K_NOT_NORMAL = 0x2, 107 DESC_ERR_KPAI_NOT_NORMAL = 0x3, 108 DESC_ERR_DESC_OFFSET_ERR = 0x4, 109 DESC_ERR_DESC_READ_FAIL = 0x8, 110 DESC_ERR_DESC_READ_TIMEOUT = 0x9, 111 DESC_ERR_DESC_READ_TLP_POISONED = 0xA, 112 DESC_ERR_CB_READ_FAIL = 0xC, 113 DESC_ERR_CB_READ_TIMEOUT = 0xD, 114 DESC_ERR_CB_READ_TLP_POISONED = 0xE 115 }; 116 117 /* FPGA LTE FEC DMA Encoding Request Descriptor */ 118 struct __rte_packed fpga_dma_enc_desc { 119 uint32_t done:1, 120 rsrvd0:11, 121 error:4, 122 rsrvd1:16; 123 uint32_t ncb:16, 124 rsrvd2:14, 125 rv:2; 126 uint32_t bypass_rm:1, 127 irq_en:1, 128 crc_en:1, 129 rsrvd3:13, 130 offset:10, 131 rsrvd4:6; 132 uint16_t e; 133 uint16_t k; 134 uint32_t out_addr_lw; 135 uint32_t out_addr_hi; 136 uint32_t in_addr_lw; 137 uint32_t in_addr_hi; 138 139 union { 140 struct { 141 /* Virtual addresses used to retrieve SW context info */ 142 void *op_addr; 143 /* Stores information about total number of Code Blocks 144 * in currently processed Transport Block 145 */ 146 uint64_t cbs_in_op; 147 }; 148 149 uint8_t sw_ctxt[FPGA_RING_DESC_LEN_UNIT_BYTES * 150 (FPGA_RING_DESC_ENTRY_LENGTH - 1)]; 151 }; 152 }; 153 154 /* FPGA LTE FEC DMA Decoding Request Descriptor */ 155 struct __rte_packed fpga_dma_dec_desc { 156 uint32_t done:1, 157 iter:5, 158 rsrvd0:2, 159 crc_pass:1, 160 rsrvd1:3, 161 error:4, 162 crc_type:1, 163 rsrvd2:7, 164 max_iter:5, 165 rsrvd3:3; 166 uint32_t rsrvd4; 167 uint32_t bypass_rm:1, 168 irq_en:1, 169 drop_crc:1, 170 rsrvd5:13, 171 offset:10, 172 rsrvd6:6; 173 uint16_t k; 174 uint16_t in_len; 175 uint32_t out_addr_lw; 176 uint32_t out_addr_hi; 177 uint32_t in_addr_lw; 178 uint32_t in_addr_hi; 179 180 union { 181 struct { 182 /* Virtual addresses used to retrieve SW context info */ 183 void *op_addr; 184 /* Stores information about total number of Code Blocks 185 * in currently processed Transport Block 186 */ 187 uint8_t cbs_in_op; 188 }; 189 190 uint32_t sw_ctxt[8 * (FPGA_RING_DESC_ENTRY_LENGTH - 1)]; 191 }; 192 }; 193 194 /* FPGA LTE DMA Descriptor */ 195 union fpga_dma_desc { 196 struct fpga_dma_enc_desc enc_req; 197 struct fpga_dma_dec_desc dec_req; 198 }; 199 200 /* FPGA LTE FEC Ring Control Register */ 201 struct __rte_packed fpga_ring_ctrl_reg { 202 uint64_t ring_base_addr; 203 uint64_t ring_head_addr; 204 uint16_t ring_size:11; 205 uint16_t rsrvd0; 206 union { /* Miscellaneous register */ 207 uint8_t misc; 208 uint8_t max_ul_dec:5, 209 max_ul_dec_en:1, 210 rsrvd1:2; 211 }; 212 uint8_t enable; 213 uint8_t flush_queue_en; 214 uint8_t rsrvd2; 215 uint16_t shadow_tail; 216 uint16_t rsrvd3; 217 uint16_t head_point; 218 uint16_t rsrvd4; 219 220 }; 221 222 /* Private data structure for each FPGA FEC device */ 223 struct fpga_lte_fec_device { 224 /** Base address of MMIO registers (BAR0) */ 225 void *mmio_base; 226 /** Base address of memory for sw rings */ 227 void *sw_rings; 228 /** Physical address of sw_rings */ 229 rte_iova_t sw_rings_phys; 230 /** Number of bytes available for each queue in device. */ 231 uint32_t sw_ring_size; 232 /** Max number of entries available for each queue in device */ 233 uint32_t sw_ring_max_depth; 234 /** Base address of response tail pointer buffer */ 235 uint32_t *tail_ptrs; 236 /** Physical address of tail pointers */ 237 rte_iova_t tail_ptr_phys; 238 /** Queues flush completion flag */ 239 uint64_t *flush_queue_status; 240 /* Bitmap capturing which Queues are bound to the PF/VF */ 241 uint64_t q_bound_bit_map; 242 /* Bitmap capturing which Queues have already been assigned */ 243 uint64_t q_assigned_bit_map; 244 /** True if this is a PF FPGA FEC device */ 245 bool pf_device; 246 }; 247 248 /* Structure associated with each queue. */ 249 struct __rte_cache_aligned fpga_queue { 250 struct fpga_ring_ctrl_reg ring_ctrl_reg; /* Ring Control Register */ 251 union fpga_dma_desc *ring_addr; /* Virtual address of software ring */ 252 uint64_t *ring_head_addr; /* Virtual address of completion_head */ 253 uint64_t shadow_completion_head; /* Shadow completion head value */ 254 uint16_t head_free_desc; /* Ring head */ 255 uint16_t tail; /* Ring tail */ 256 /* Mask used to wrap enqueued descriptors on the sw ring */ 257 uint32_t sw_ring_wrap_mask; 258 uint32_t irq_enable; /* Enable ops dequeue interrupts if set to 1 */ 259 uint8_t q_idx; /* Queue index */ 260 struct fpga_lte_fec_device *d; 261 /* MMIO register of shadow_tail used to enqueue descriptors */ 262 void *shadow_tail_addr; 263 }; 264 265 /* Write to 16 bit MMIO register address */ 266 static inline void 267 mmio_write_16(void *addr, uint16_t value) 268 { 269 *((volatile uint16_t *)(addr)) = rte_cpu_to_le_16(value); 270 } 271 272 /* Write to 32 bit MMIO register address */ 273 static inline void 274 mmio_write_32(void *addr, uint32_t value) 275 { 276 *((volatile uint32_t *)(addr)) = rte_cpu_to_le_32(value); 277 } 278 279 /* Write to 64 bit MMIO register address */ 280 static inline void 281 mmio_write_64(void *addr, uint64_t value) 282 { 283 *((volatile uint64_t *)(addr)) = rte_cpu_to_le_64(value); 284 } 285 286 /* Write a 8 bit register of a FPGA LTE FEC device */ 287 static inline void 288 fpga_reg_write_8(void *mmio_base, uint32_t offset, uint8_t payload) 289 { 290 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 291 *((volatile uint8_t *)(reg_addr)) = payload; 292 } 293 294 /* Write a 16 bit register of a FPGA LTE FEC device */ 295 static inline void 296 fpga_reg_write_16(void *mmio_base, uint32_t offset, uint16_t payload) 297 { 298 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 299 mmio_write_16(reg_addr, payload); 300 } 301 302 /* Write a 32 bit register of a FPGA LTE FEC device */ 303 static inline void 304 fpga_reg_write_32(void *mmio_base, uint32_t offset, uint32_t payload) 305 { 306 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 307 mmio_write_32(reg_addr, payload); 308 } 309 310 /* Write a 64 bit register of a FPGA LTE FEC device */ 311 static inline void 312 fpga_reg_write_64(void *mmio_base, uint32_t offset, uint64_t payload) 313 { 314 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 315 mmio_write_64(reg_addr, payload); 316 } 317 318 /* Write a ring control register of a FPGA LTE FEC device */ 319 static inline void 320 fpga_ring_reg_write(void *mmio_base, uint32_t offset, 321 struct fpga_ring_ctrl_reg payload) 322 { 323 fpga_reg_write_64(mmio_base, offset, payload.ring_base_addr); 324 fpga_reg_write_64(mmio_base, offset + FPGA_LTE_FEC_RING_HEAD_ADDR, 325 payload.ring_head_addr); 326 fpga_reg_write_16(mmio_base, offset + FPGA_LTE_FEC_RING_SIZE, 327 payload.ring_size); 328 fpga_reg_write_16(mmio_base, offset + FPGA_LTE_FEC_RING_HEAD_POINT, 329 payload.head_point); 330 fpga_reg_write_8(mmio_base, offset + FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN, 331 payload.flush_queue_en); 332 fpga_reg_write_16(mmio_base, offset + FPGA_LTE_FEC_RING_SHADOW_TAIL, 333 payload.shadow_tail); 334 fpga_reg_write_8(mmio_base, offset + FPGA_LTE_FEC_RING_MISC, 335 payload.misc); 336 fpga_reg_write_8(mmio_base, offset + FPGA_LTE_FEC_RING_ENABLE, 337 payload.enable); 338 } 339 340 /* Read a register of FPGA LTE FEC device */ 341 static uint32_t 342 fpga_reg_read_32(void *mmio_base, uint32_t offset) 343 { 344 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 345 uint32_t ret = *((volatile uint32_t *)(reg_addr)); 346 return rte_le_to_cpu_32(ret); 347 } 348 349 #ifdef RTE_LIBRTE_BBDEV_DEBUG 350 /* Read a register of FPGA LTE FEC device */ 351 static uint8_t 352 fpga_reg_read_8(void *mmio_base, uint32_t offset) 353 { 354 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 355 return *((volatile uint8_t *)(reg_addr)); 356 } 357 358 /* Read a register of FPGA LTE FEC device */ 359 static uint16_t 360 fpga_reg_read_16(void *mmio_base, uint32_t offset) 361 { 362 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 363 uint16_t ret = *((volatile uint16_t *)(reg_addr)); 364 return rte_le_to_cpu_16(ret); 365 } 366 367 /* Read a register of FPGA LTE FEC device */ 368 static uint64_t 369 fpga_reg_read_64(void *mmio_base, uint32_t offset) 370 { 371 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 372 uint64_t ret = *((volatile uint64_t *)(reg_addr)); 373 return rte_le_to_cpu_64(ret); 374 } 375 376 /* Read Ring Control Register of FPGA LTE FEC device */ 377 static inline void 378 print_ring_reg_debug_info(void *mmio_base, uint32_t offset) 379 { 380 rte_bbdev_log_debug( 381 "FPGA MMIO base address @ %p | Ring Control Register @ offset = 0x%08" 382 PRIx32, mmio_base, offset); 383 rte_bbdev_log_debug( 384 "RING_BASE_ADDR = 0x%016"PRIx64, 385 fpga_reg_read_64(mmio_base, offset)); 386 rte_bbdev_log_debug( 387 "RING_HEAD_ADDR = 0x%016"PRIx64, 388 fpga_reg_read_64(mmio_base, offset + 389 FPGA_LTE_FEC_RING_HEAD_ADDR)); 390 rte_bbdev_log_debug( 391 "RING_SIZE = 0x%04"PRIx16, 392 fpga_reg_read_16(mmio_base, offset + 393 FPGA_LTE_FEC_RING_SIZE)); 394 rte_bbdev_log_debug( 395 "RING_MISC = 0x%02"PRIx8, 396 fpga_reg_read_8(mmio_base, offset + 397 FPGA_LTE_FEC_RING_MISC)); 398 rte_bbdev_log_debug( 399 "RING_ENABLE = 0x%02"PRIx8, 400 fpga_reg_read_8(mmio_base, offset + 401 FPGA_LTE_FEC_RING_ENABLE)); 402 rte_bbdev_log_debug( 403 "RING_FLUSH_QUEUE_EN = 0x%02"PRIx8, 404 fpga_reg_read_8(mmio_base, offset + 405 FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN)); 406 rte_bbdev_log_debug( 407 "RING_SHADOW_TAIL = 0x%04"PRIx16, 408 fpga_reg_read_16(mmio_base, offset + 409 FPGA_LTE_FEC_RING_SHADOW_TAIL)); 410 rte_bbdev_log_debug( 411 "RING_HEAD_POINT = 0x%04"PRIx16, 412 fpga_reg_read_16(mmio_base, offset + 413 FPGA_LTE_FEC_RING_HEAD_POINT)); 414 } 415 416 /* Read Static Register of FPGA LTE FEC device */ 417 static inline void 418 print_static_reg_debug_info(void *mmio_base) 419 { 420 uint16_t config = fpga_reg_read_16(mmio_base, 421 FPGA_LTE_FEC_CONFIGURATION); 422 uint8_t qmap_done = fpga_reg_read_8(mmio_base, 423 FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE); 424 uint16_t lb_factor = fpga_reg_read_16(mmio_base, 425 FPGA_LTE_FEC_LOAD_BALANCE_FACTOR); 426 uint16_t ring_desc_len = fpga_reg_read_16(mmio_base, 427 FPGA_LTE_FEC_RING_DESC_LEN); 428 uint16_t flr_time_out = fpga_reg_read_16(mmio_base, 429 FPGA_LTE_FEC_FLR_TIME_OUT); 430 431 rte_bbdev_log_debug("UL.DL Weights = %u.%u", 432 ((uint8_t)config), ((uint8_t)(config >> 8))); 433 rte_bbdev_log_debug("UL.DL Load Balance = %u.%u", 434 ((uint8_t)lb_factor), ((uint8_t)(lb_factor >> 8))); 435 rte_bbdev_log_debug("Queue-PF/VF Mapping Table = %s", 436 (qmap_done > 0) ? "READY" : "NOT-READY"); 437 rte_bbdev_log_debug("Ring Descriptor Size = %u bytes", 438 ring_desc_len*FPGA_RING_DESC_LEN_UNIT_BYTES); 439 rte_bbdev_log_debug("FLR Timeout = %f usec", 440 (float)flr_time_out*FPGA_FLR_TIMEOUT_UNIT); 441 } 442 443 /* Print decode DMA Descriptor of FPGA LTE FEC device */ 444 static void 445 print_dma_dec_desc_debug_info(union fpga_dma_desc *desc) 446 { 447 rte_bbdev_log_debug("DMA response desc %p\n" 448 "\t-- done(%"PRIu32") | iter(%"PRIu32") | crc_pass(%"PRIu32")" 449 " | error (%"PRIu32") | crc_type(%"PRIu32")\n" 450 "\t-- max_iter(%"PRIu32") | bypass_rm(%"PRIu32") | " 451 "irq_en (%"PRIu32") | drop_crc(%"PRIu32") | offset(%"PRIu32")\n" 452 "\t-- k(%"PRIu32") | in_len (%"PRIu16") | op_add(%p)\n" 453 "\t-- cbs_in_op(%"PRIu32") | in_add (0x%08"PRIx32"%08"PRIx32") | " 454 "out_add (0x%08"PRIx32"%08"PRIx32")", 455 desc, 456 (uint32_t)desc->dec_req.done, 457 (uint32_t)desc->dec_req.iter, 458 (uint32_t)desc->dec_req.crc_pass, 459 (uint32_t)desc->dec_req.error, 460 (uint32_t)desc->dec_req.crc_type, 461 (uint32_t)desc->dec_req.max_iter, 462 (uint32_t)desc->dec_req.bypass_rm, 463 (uint32_t)desc->dec_req.irq_en, 464 (uint32_t)desc->dec_req.drop_crc, 465 (uint32_t)desc->dec_req.offset, 466 (uint32_t)desc->dec_req.k, 467 (uint16_t)desc->dec_req.in_len, 468 desc->dec_req.op_addr, 469 (uint32_t)desc->dec_req.cbs_in_op, 470 (uint32_t)desc->dec_req.in_addr_hi, 471 (uint32_t)desc->dec_req.in_addr_lw, 472 (uint32_t)desc->dec_req.out_addr_hi, 473 (uint32_t)desc->dec_req.out_addr_lw); 474 } 475 #endif 476 477 static int 478 fpga_setup_queues(struct rte_bbdev *dev, uint16_t num_queues, int socket_id) 479 { 480 /* Number of queues bound to a PF/VF */ 481 uint32_t hw_q_num = 0; 482 uint32_t ring_size, payload, address, q_id, offset; 483 rte_iova_t phys_addr; 484 struct fpga_ring_ctrl_reg ring_reg; 485 struct fpga_lte_fec_device *fpga_dev = dev->data->dev_private; 486 487 address = FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE; 488 if (!(fpga_reg_read_32(fpga_dev->mmio_base, address) & 0x1)) { 489 rte_bbdev_log(ERR, 490 "Queue-PF/VF mapping is not set! Was PF configured for device (%s) ?", 491 dev->data->name); 492 return -EPERM; 493 } 494 495 /* Clear queue registers structure */ 496 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg)); 497 498 /* Scan queue map. 499 * If a queue is valid and mapped to a calling PF/VF the read value is 500 * replaced with a queue ID and if it's not then 501 * FPGA_INVALID_HW_QUEUE_ID is returned. 502 */ 503 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 504 uint32_t hw_q_id = fpga_reg_read_32(fpga_dev->mmio_base, 505 FPGA_LTE_FEC_QUEUE_MAP + (q_id << 2)); 506 507 rte_bbdev_log_debug("%s: queue ID: %u, registry queue ID: %u", 508 dev->device->name, q_id, hw_q_id); 509 510 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) { 511 fpga_dev->q_bound_bit_map |= (1ULL << q_id); 512 /* Clear queue register of found queue */ 513 offset = FPGA_LTE_FEC_RING_CTRL_REGS + 514 (sizeof(struct fpga_ring_ctrl_reg) * q_id); 515 fpga_ring_reg_write(fpga_dev->mmio_base, 516 offset, ring_reg); 517 ++hw_q_num; 518 } 519 } 520 if (hw_q_num == 0) { 521 rte_bbdev_log(ERR, 522 "No HW queues assigned to this device. Probably this is a VF configured for PF mode. Check device configuration!"); 523 return -ENODEV; 524 } 525 526 if (num_queues > hw_q_num) { 527 rte_bbdev_log(ERR, 528 "Not enough queues for device %s! Requested: %u, available: %u", 529 dev->device->name, num_queues, hw_q_num); 530 return -EINVAL; 531 } 532 533 ring_size = FPGA_RING_MAX_SIZE * sizeof(struct fpga_dma_dec_desc); 534 535 /* Enforce 32 byte alignment */ 536 RTE_BUILD_BUG_ON((RTE_CACHE_LINE_SIZE % 32) != 0); 537 538 /* Allocate memory for SW descriptor rings */ 539 fpga_dev->sw_rings = rte_zmalloc_socket(dev->device->driver->name, 540 num_queues * ring_size, RTE_CACHE_LINE_SIZE, 541 socket_id); 542 if (fpga_dev->sw_rings == NULL) { 543 rte_bbdev_log(ERR, 544 "Failed to allocate memory for %s:%u sw_rings", 545 dev->device->driver->name, dev->data->dev_id); 546 return -ENOMEM; 547 } 548 549 fpga_dev->sw_rings_phys = rte_malloc_virt2iova(fpga_dev->sw_rings); 550 fpga_dev->sw_ring_size = ring_size; 551 fpga_dev->sw_ring_max_depth = FPGA_RING_MAX_SIZE; 552 553 /* Allocate memory for ring flush status */ 554 fpga_dev->flush_queue_status = rte_zmalloc_socket(NULL, 555 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, socket_id); 556 if (fpga_dev->flush_queue_status == NULL) { 557 rte_bbdev_log(ERR, 558 "Failed to allocate memory for %s:%u flush_queue_status", 559 dev->device->driver->name, dev->data->dev_id); 560 return -ENOMEM; 561 } 562 563 /* Set the flush status address registers */ 564 phys_addr = rte_malloc_virt2iova(fpga_dev->flush_queue_status); 565 566 address = FPGA_LTE_FEC_VFQ_FLUSH_STATUS_LW; 567 payload = (uint32_t)(phys_addr); 568 fpga_reg_write_32(fpga_dev->mmio_base, address, payload); 569 570 address = FPGA_LTE_FEC_VFQ_FLUSH_STATUS_HI; 571 payload = (uint32_t)(phys_addr >> 32); 572 fpga_reg_write_32(fpga_dev->mmio_base, address, payload); 573 574 return 0; 575 } 576 577 static int 578 fpga_dev_close(struct rte_bbdev *dev) 579 { 580 struct fpga_lte_fec_device *fpga_dev = dev->data->dev_private; 581 582 rte_free(fpga_dev->sw_rings); 583 rte_free(fpga_dev->flush_queue_status); 584 585 return 0; 586 } 587 588 static void 589 fpga_dev_info_get(struct rte_bbdev *dev, 590 struct rte_bbdev_driver_info *dev_info) 591 { 592 struct fpga_lte_fec_device *d = dev->data->dev_private; 593 uint32_t q_id = 0; 594 595 /* TODO RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN and numbers of buffers are set 596 * to temporary values as they are required by test application while 597 * validation phase. 598 */ 599 static const struct rte_bbdev_op_cap bbdev_capabilities[] = { 600 { 601 .type = RTE_BBDEV_OP_TURBO_DEC, 602 .cap.turbo_dec = { 603 .capability_flags = 604 RTE_BBDEV_TURBO_CRC_TYPE_24B | 605 RTE_BBDEV_TURBO_SUBBLOCK_DEINTERLEAVE | 606 RTE_BBDEV_TURBO_DEC_INTERRUPTS | 607 RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN | 608 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP, 609 .max_llr_modulus = INT8_MAX, 610 .num_buffers_src = 611 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS, 612 .num_buffers_hard_out = 613 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS, 614 .num_buffers_soft_out = 0 615 } 616 }, 617 { 618 .type = RTE_BBDEV_OP_TURBO_ENC, 619 .cap.turbo_enc = { 620 .capability_flags = 621 RTE_BBDEV_TURBO_CRC_24B_ATTACH | 622 RTE_BBDEV_TURBO_RATE_MATCH | 623 RTE_BBDEV_TURBO_ENC_INTERRUPTS, 624 .num_buffers_src = 625 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS, 626 .num_buffers_dst = 627 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS 628 } 629 }, 630 RTE_BBDEV_END_OF_CAPABILITIES_LIST() 631 }; 632 633 static struct rte_bbdev_queue_conf default_queue_conf; 634 default_queue_conf.socket = dev->data->socket_id; 635 default_queue_conf.queue_size = FPGA_RING_MAX_SIZE; 636 637 638 dev_info->driver_name = dev->device->driver->name; 639 dev_info->queue_size_lim = FPGA_RING_MAX_SIZE; 640 dev_info->hardware_accelerated = true; 641 dev_info->min_alignment = 64; 642 dev_info->default_queue_conf = default_queue_conf; 643 dev_info->capabilities = bbdev_capabilities; 644 dev_info->cpu_flag_reqs = NULL; 645 dev_info->data_endianness = RTE_LITTLE_ENDIAN; 646 dev_info->device_status = RTE_BBDEV_DEV_NOT_SUPPORTED; 647 648 /* Calculates number of queues assigned to device */ 649 dev_info->max_num_queues = 0; 650 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 651 uint32_t hw_q_id = fpga_reg_read_32(d->mmio_base, 652 FPGA_LTE_FEC_QUEUE_MAP + (q_id << 2)); 653 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) 654 dev_info->max_num_queues++; 655 } 656 /* Expose number of queue per operation type */ 657 dev_info->num_queues[RTE_BBDEV_OP_NONE] = 0; 658 dev_info->num_queues[RTE_BBDEV_OP_TURBO_DEC] = dev_info->max_num_queues / 2; 659 dev_info->num_queues[RTE_BBDEV_OP_TURBO_ENC] = dev_info->max_num_queues / 2; 660 dev_info->num_queues[RTE_BBDEV_OP_LDPC_DEC] = 0; 661 dev_info->num_queues[RTE_BBDEV_OP_LDPC_ENC] = 0; 662 dev_info->queue_priority[RTE_BBDEV_OP_TURBO_DEC] = 1; 663 dev_info->queue_priority[RTE_BBDEV_OP_TURBO_ENC] = 1; 664 } 665 666 /** 667 * Find index of queue bound to current PF/VF which is unassigned. Return -1 668 * when there is no available queue 669 */ 670 static int 671 fpga_find_free_queue_idx(struct rte_bbdev *dev, 672 const struct rte_bbdev_queue_conf *conf) 673 { 674 struct fpga_lte_fec_device *d = dev->data->dev_private; 675 uint64_t q_idx; 676 uint8_t i = 0; 677 uint8_t range = FPGA_TOTAL_NUM_QUEUES >> 1; 678 679 if (conf->op_type == RTE_BBDEV_OP_TURBO_ENC) { 680 i = FPGA_NUM_DL_QUEUES; 681 range = FPGA_TOTAL_NUM_QUEUES; 682 } 683 684 for (; i < range; ++i) { 685 q_idx = 1ULL << i; 686 /* Check if index of queue is bound to current PF/VF */ 687 if (d->q_bound_bit_map & q_idx) 688 /* Check if found queue was not already assigned */ 689 if (!(d->q_assigned_bit_map & q_idx)) { 690 d->q_assigned_bit_map |= q_idx; 691 return i; 692 } 693 } 694 695 rte_bbdev_log(INFO, "Failed to find free queue on %s", dev->data->name); 696 697 return -1; 698 } 699 700 static int 701 fpga_queue_setup(struct rte_bbdev *dev, uint16_t queue_id, 702 const struct rte_bbdev_queue_conf *conf) 703 { 704 uint32_t address, ring_offset; 705 struct fpga_lte_fec_device *d = dev->data->dev_private; 706 struct fpga_queue *q; 707 int8_t q_idx; 708 709 /* Check if there is a free queue to assign */ 710 q_idx = fpga_find_free_queue_idx(dev, conf); 711 if (q_idx == -1) 712 return -1; 713 714 /* Allocate the queue data structure. */ 715 q = rte_zmalloc_socket(dev->device->driver->name, sizeof(*q), 716 RTE_CACHE_LINE_SIZE, conf->socket); 717 if (q == NULL) { 718 /* Mark queue as un-assigned */ 719 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 720 rte_bbdev_log(ERR, "Failed to allocate queue memory"); 721 return -ENOMEM; 722 } 723 724 q->d = d; 725 q->q_idx = q_idx; 726 727 /* Set ring_base_addr */ 728 q->ring_addr = RTE_PTR_ADD(d->sw_rings, (d->sw_ring_size * queue_id)); 729 q->ring_ctrl_reg.ring_base_addr = d->sw_rings_phys + 730 (d->sw_ring_size * queue_id); 731 732 /* Allocate memory for Completion Head variable*/ 733 q->ring_head_addr = rte_zmalloc_socket(dev->device->driver->name, 734 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, conf->socket); 735 if (q->ring_head_addr == NULL) { 736 /* Mark queue as un-assigned */ 737 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 738 rte_free(q); 739 rte_bbdev_log(ERR, 740 "Failed to allocate memory for %s:%u completion_head", 741 dev->device->driver->name, dev->data->dev_id); 742 return -ENOMEM; 743 } 744 /* Set ring_head_addr */ 745 q->ring_ctrl_reg.ring_head_addr = 746 rte_malloc_virt2iova(q->ring_head_addr); 747 748 /* Clear shadow_completion_head */ 749 q->shadow_completion_head = 0; 750 751 /* Set ring_size */ 752 if (conf->queue_size > FPGA_RING_MAX_SIZE) { 753 /* Mark queue as un-assigned */ 754 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 755 rte_free(q->ring_head_addr); 756 rte_free(q); 757 rte_bbdev_log(ERR, 758 "Size of queue is too big %d (MAX: %d ) for %s:%u", 759 conf->queue_size, FPGA_RING_MAX_SIZE, 760 dev->device->driver->name, dev->data->dev_id); 761 return -EINVAL; 762 } 763 q->ring_ctrl_reg.ring_size = conf->queue_size; 764 765 /* Set Miscellaneous FPGA register*/ 766 /* Max iteration number for TTI mitigation - todo */ 767 q->ring_ctrl_reg.max_ul_dec = 0; 768 /* Enable max iteration number for TTI - todo */ 769 q->ring_ctrl_reg.max_ul_dec_en = 0; 770 771 /* Enable the ring */ 772 q->ring_ctrl_reg.enable = 1; 773 774 /* Set FPGA head_point and tail registers */ 775 q->ring_ctrl_reg.head_point = q->tail = 0; 776 777 /* Set FPGA shadow_tail register */ 778 q->ring_ctrl_reg.shadow_tail = q->tail; 779 780 /* Calculates the ring offset for found queue */ 781 ring_offset = FPGA_LTE_FEC_RING_CTRL_REGS + 782 (sizeof(struct fpga_ring_ctrl_reg) * q_idx); 783 784 /* Set FPGA Ring Control Registers */ 785 fpga_ring_reg_write(d->mmio_base, ring_offset, q->ring_ctrl_reg); 786 787 /* Store MMIO register of shadow_tail */ 788 address = ring_offset + FPGA_LTE_FEC_RING_SHADOW_TAIL; 789 q->shadow_tail_addr = RTE_PTR_ADD(d->mmio_base, address); 790 791 q->head_free_desc = q->tail; 792 793 /* Set wrap mask */ 794 q->sw_ring_wrap_mask = conf->queue_size - 1; 795 796 rte_bbdev_log_debug("Setup dev%u q%u: queue_idx=%u", 797 dev->data->dev_id, queue_id, q->q_idx); 798 799 dev->data->queues[queue_id].queue_private = q; 800 801 rte_bbdev_log_debug("BBDEV queue[%d] set up for FPGA queue[%d]", 802 queue_id, q_idx); 803 804 #ifdef RTE_LIBRTE_BBDEV_DEBUG 805 /* Read FPGA Ring Control Registers after configuration*/ 806 print_ring_reg_debug_info(d->mmio_base, ring_offset); 807 #endif 808 return 0; 809 } 810 811 static int 812 fpga_queue_release(struct rte_bbdev *dev, uint16_t queue_id) 813 { 814 struct fpga_lte_fec_device *d = dev->data->dev_private; 815 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 816 struct fpga_ring_ctrl_reg ring_reg; 817 uint32_t offset; 818 819 rte_bbdev_log_debug("FPGA Queue[%d] released", queue_id); 820 821 if (q != NULL) { 822 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg)); 823 offset = FPGA_LTE_FEC_RING_CTRL_REGS + 824 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 825 /* Disable queue */ 826 fpga_reg_write_8(d->mmio_base, 827 offset + FPGA_LTE_FEC_RING_ENABLE, 0x00); 828 /* Clear queue registers */ 829 fpga_ring_reg_write(d->mmio_base, offset, ring_reg); 830 831 /* Mark the Queue as un-assigned */ 832 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q->q_idx)); 833 rte_free(q->ring_head_addr); 834 rte_free(q); 835 dev->data->queues[queue_id].queue_private = NULL; 836 } 837 838 return 0; 839 } 840 841 /* Function starts a device queue. */ 842 static int 843 fpga_queue_start(struct rte_bbdev *dev, uint16_t queue_id) 844 { 845 struct fpga_lte_fec_device *d = dev->data->dev_private; 846 #ifdef RTE_LIBRTE_BBDEV_DEBUG 847 if (d == NULL) { 848 rte_bbdev_log(ERR, "Invalid device pointer"); 849 return -1; 850 } 851 #endif 852 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 853 uint32_t offset = FPGA_LTE_FEC_RING_CTRL_REGS + 854 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 855 uint8_t enable = 0x01; 856 uint16_t zero = 0x0000; 857 858 /* Clear queue head and tail variables */ 859 q->tail = q->head_free_desc = 0; 860 861 /* Clear FPGA head_point and tail registers */ 862 fpga_reg_write_16(d->mmio_base, offset + FPGA_LTE_FEC_RING_HEAD_POINT, 863 zero); 864 fpga_reg_write_16(d->mmio_base, offset + FPGA_LTE_FEC_RING_SHADOW_TAIL, 865 zero); 866 867 /* Enable queue */ 868 fpga_reg_write_8(d->mmio_base, offset + FPGA_LTE_FEC_RING_ENABLE, 869 enable); 870 871 rte_bbdev_log_debug("FPGA Queue[%d] started", queue_id); 872 return 0; 873 } 874 875 /* Function stops a device queue. */ 876 static int 877 fpga_queue_stop(struct rte_bbdev *dev, uint16_t queue_id) 878 { 879 struct fpga_lte_fec_device *d = dev->data->dev_private; 880 #ifdef RTE_LIBRTE_BBDEV_DEBUG 881 if (d == NULL) { 882 rte_bbdev_log(ERR, "Invalid device pointer"); 883 return -1; 884 } 885 #endif 886 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 887 uint32_t offset = FPGA_LTE_FEC_RING_CTRL_REGS + 888 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 889 uint8_t payload = 0x01; 890 uint8_t counter = 0; 891 uint8_t timeout = FPGA_QUEUE_FLUSH_TIMEOUT_US / 892 FPGA_TIMEOUT_CHECK_INTERVAL; 893 894 /* Set flush_queue_en bit to trigger queue flushing */ 895 fpga_reg_write_8(d->mmio_base, 896 offset + FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN, payload); 897 898 /** Check if queue flush is completed. 899 * FPGA will update the completion flag after queue flushing is 900 * completed. If completion flag is not updated within 1ms it is 901 * considered as a failure. 902 */ 903 while (!(*((volatile uint8_t *)d->flush_queue_status + q->q_idx) & payload)) { 904 if (counter > timeout) { 905 rte_bbdev_log(ERR, "FPGA Queue Flush failed for queue %d", 906 queue_id); 907 return -1; 908 } 909 usleep(FPGA_TIMEOUT_CHECK_INTERVAL); 910 counter++; 911 } 912 913 /* Disable queue */ 914 payload = 0x00; 915 fpga_reg_write_8(d->mmio_base, offset + FPGA_LTE_FEC_RING_ENABLE, 916 payload); 917 918 rte_bbdev_log_debug("FPGA Queue[%d] stopped", queue_id); 919 return 0; 920 } 921 922 static inline uint16_t 923 get_queue_id(struct rte_bbdev_data *data, uint8_t q_idx) 924 { 925 uint16_t queue_id; 926 927 for (queue_id = 0; queue_id < data->num_queues; ++queue_id) { 928 struct fpga_queue *q = data->queues[queue_id].queue_private; 929 if (q != NULL && q->q_idx == q_idx) 930 return queue_id; 931 } 932 933 return -1; 934 } 935 936 /* Interrupt handler triggered by FPGA dev for handling specific interrupt */ 937 static void 938 fpga_dev_interrupt_handler(void *cb_arg) 939 { 940 struct rte_bbdev *dev = cb_arg; 941 struct fpga_lte_fec_device *fpga_dev = dev->data->dev_private; 942 struct fpga_queue *q; 943 uint64_t ring_head; 944 uint64_t q_idx; 945 uint16_t queue_id; 946 uint8_t i; 947 948 /* Scan queue assigned to this device */ 949 for (i = 0; i < FPGA_TOTAL_NUM_QUEUES; ++i) { 950 q_idx = 1ULL << i; 951 if (fpga_dev->q_bound_bit_map & q_idx) { 952 queue_id = get_queue_id(dev->data, i); 953 if (queue_id == (uint16_t) -1) 954 continue; 955 956 /* Check if completion head was changed */ 957 q = dev->data->queues[queue_id].queue_private; 958 ring_head = *q->ring_head_addr; 959 if (q->shadow_completion_head != ring_head && 960 q->irq_enable == 1) { 961 q->shadow_completion_head = ring_head; 962 rte_bbdev_pmd_callback_process( 963 dev, 964 RTE_BBDEV_EVENT_DEQUEUE, 965 &queue_id); 966 } 967 } 968 } 969 } 970 971 static int 972 fpga_queue_intr_enable(struct rte_bbdev *dev, uint16_t queue_id) 973 { 974 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 975 976 if (!rte_intr_cap_multiple(dev->intr_handle)) 977 return -ENOTSUP; 978 979 q->irq_enable = 1; 980 981 return 0; 982 } 983 984 static int 985 fpga_queue_intr_disable(struct rte_bbdev *dev, uint16_t queue_id) 986 { 987 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 988 q->irq_enable = 0; 989 990 return 0; 991 } 992 993 static int 994 fpga_intr_enable(struct rte_bbdev *dev) 995 { 996 int ret; 997 uint8_t i; 998 999 if (!rte_intr_cap_multiple(dev->intr_handle)) { 1000 rte_bbdev_log(ERR, "Multiple intr vector is not supported by FPGA (%s)", 1001 dev->data->name); 1002 return -ENOTSUP; 1003 } 1004 1005 /* Create event file descriptors for each of 64 queue. Event fds will be 1006 * mapped to FPGA IRQs in rte_intr_enable(). This is a 1:1 mapping where 1007 * the IRQ number is a direct translation to the queue number. 1008 * 1009 * 63 (FPGA_NUM_INTR_VEC) event fds are created as rte_intr_enable() 1010 * mapped the first IRQ to already created interrupt event file 1011 * descriptor (intr_handle->fd). 1012 */ 1013 if (rte_intr_efd_enable(dev->intr_handle, FPGA_NUM_INTR_VEC)) { 1014 rte_bbdev_log(ERR, "Failed to create fds for %u queues", 1015 dev->data->num_queues); 1016 return -1; 1017 } 1018 1019 /* TODO Each event file descriptor is overwritten by interrupt event 1020 * file descriptor. That descriptor is added to epoll observed list. 1021 * It ensures that callback function assigned to that descriptor will 1022 * invoked when any FPGA queue issues interrupt. 1023 */ 1024 for (i = 0; i < FPGA_NUM_INTR_VEC; ++i) { 1025 if (rte_intr_efds_index_set(dev->intr_handle, i, 1026 rte_intr_fd_get(dev->intr_handle))) 1027 return -rte_errno; 1028 } 1029 1030 if (rte_intr_vec_list_alloc(dev->intr_handle, "intr_vec", 1031 dev->data->num_queues)) { 1032 rte_bbdev_log(ERR, "Failed to allocate %u vectors", 1033 dev->data->num_queues); 1034 return -ENOMEM; 1035 } 1036 1037 ret = rte_intr_enable(dev->intr_handle); 1038 if (ret < 0) { 1039 rte_bbdev_log(ERR, 1040 "Couldn't enable interrupts for device: %s", 1041 dev->data->name); 1042 return ret; 1043 } 1044 1045 ret = rte_intr_callback_register(dev->intr_handle, 1046 fpga_dev_interrupt_handler, dev); 1047 if (ret < 0) { 1048 rte_bbdev_log(ERR, 1049 "Couldn't register interrupt callback for device: %s", 1050 dev->data->name); 1051 return ret; 1052 } 1053 1054 return 0; 1055 } 1056 1057 static const struct rte_bbdev_ops fpga_ops = { 1058 .setup_queues = fpga_setup_queues, 1059 .intr_enable = fpga_intr_enable, 1060 .close = fpga_dev_close, 1061 .info_get = fpga_dev_info_get, 1062 .queue_setup = fpga_queue_setup, 1063 .queue_stop = fpga_queue_stop, 1064 .queue_start = fpga_queue_start, 1065 .queue_release = fpga_queue_release, 1066 .queue_intr_enable = fpga_queue_intr_enable, 1067 .queue_intr_disable = fpga_queue_intr_disable 1068 }; 1069 1070 static inline void 1071 fpga_dma_enqueue(struct fpga_queue *q, uint16_t num_desc, 1072 struct rte_bbdev_stats *queue_stats) 1073 { 1074 uint64_t start_time = 0; 1075 queue_stats->acc_offload_cycles = 0; 1076 1077 /* Update tail and shadow_tail register */ 1078 q->tail = (q->tail + num_desc) & q->sw_ring_wrap_mask; 1079 1080 rte_wmb(); 1081 1082 /* Start time measurement for enqueue function offload. */ 1083 start_time = rte_rdtsc_precise(); 1084 mmio_write_16(q->shadow_tail_addr, q->tail); 1085 1086 rte_wmb(); 1087 queue_stats->acc_offload_cycles += rte_rdtsc_precise() - start_time; 1088 } 1089 1090 /* Calculates number of CBs in processed encoder TB based on 'r' and input 1091 * length. 1092 */ 1093 static inline uint8_t 1094 get_num_cbs_in_op_enc(struct rte_bbdev_op_turbo_enc *turbo_enc) 1095 { 1096 uint8_t c, c_neg, r, crc24_bits = 0; 1097 uint16_t k, k_neg, k_pos; 1098 uint8_t cbs_in_op = 0; 1099 int32_t length; 1100 1101 length = turbo_enc->input.length; 1102 r = turbo_enc->tb_params.r; 1103 c = turbo_enc->tb_params.c; 1104 c_neg = turbo_enc->tb_params.c_neg; 1105 k_neg = turbo_enc->tb_params.k_neg; 1106 k_pos = turbo_enc->tb_params.k_pos; 1107 crc24_bits = 24; 1108 while (length > 0 && r < c) { 1109 k = (r < c_neg) ? k_neg : k_pos; 1110 length -= (k - crc24_bits) >> 3; 1111 r++; 1112 cbs_in_op++; 1113 } 1114 1115 return cbs_in_op; 1116 } 1117 1118 /* Calculates number of CBs in processed decoder TB based on 'r' and input 1119 * length. 1120 */ 1121 static inline uint16_t 1122 get_num_cbs_in_op_dec(struct rte_bbdev_op_turbo_dec *turbo_dec) 1123 { 1124 uint8_t c, c_neg, r = 0; 1125 uint16_t kw, k, k_neg, k_pos, cbs_in_op = 0; 1126 int32_t length; 1127 1128 length = turbo_dec->input.length; 1129 r = turbo_dec->tb_params.r; 1130 c = turbo_dec->tb_params.c; 1131 c_neg = turbo_dec->tb_params.c_neg; 1132 k_neg = turbo_dec->tb_params.k_neg; 1133 k_pos = turbo_dec->tb_params.k_pos; 1134 while (length > 0 && r < c) { 1135 k = (r < c_neg) ? k_neg : k_pos; 1136 kw = RTE_ALIGN_CEIL(k + 4, 32) * 3; 1137 length -= kw; 1138 r++; 1139 cbs_in_op++; 1140 } 1141 1142 return cbs_in_op; 1143 } 1144 1145 /* Read flag value 0/1/ from bitmap */ 1146 static inline bool 1147 check_bit(uint32_t bitmap, uint32_t bitmask) 1148 { 1149 return bitmap & bitmask; 1150 } 1151 1152 /* Print an error if a descriptor error has occurred. 1153 * Return 0 on success, 1 on failure 1154 */ 1155 static inline int 1156 check_desc_error(uint32_t error_code) { 1157 switch (error_code) { 1158 case DESC_ERR_NO_ERR: 1159 return 0; 1160 case DESC_ERR_K_OUT_OF_RANGE: 1161 rte_bbdev_log(ERR, "Block_size_k is out of range (k<40 or k>6144)"); 1162 break; 1163 case DESC_ERR_K_NOT_NORMAL: 1164 rte_bbdev_log(ERR, "Block_size_k is not a normal value within normal range"); 1165 break; 1166 case DESC_ERR_KPAI_NOT_NORMAL: 1167 rte_bbdev_log(ERR, "Three_kpai is not a normal value for UL only"); 1168 break; 1169 case DESC_ERR_DESC_OFFSET_ERR: 1170 rte_bbdev_log(ERR, "Queue offset does not meet the expectation in the FPGA"); 1171 break; 1172 case (DESC_ERR_K_OUT_OF_RANGE | DESC_ERR_DESC_OFFSET_ERR): 1173 rte_bbdev_log(ERR, "Block_size_k is out of range (k<40 or k>6144) and queue offset error"); 1174 break; 1175 case (DESC_ERR_K_NOT_NORMAL | DESC_ERR_DESC_OFFSET_ERR): 1176 rte_bbdev_log(ERR, "Block_size_k is not a normal value within normal range and queue offset error"); 1177 break; 1178 case (DESC_ERR_KPAI_NOT_NORMAL | DESC_ERR_DESC_OFFSET_ERR): 1179 rte_bbdev_log(ERR, "Three_kpai is not a normal value for UL only and queue offset error"); 1180 break; 1181 case DESC_ERR_DESC_READ_FAIL: 1182 rte_bbdev_log(ERR, "Unsuccessful completion for descriptor read"); 1183 break; 1184 case DESC_ERR_DESC_READ_TIMEOUT: 1185 rte_bbdev_log(ERR, "Descriptor read time-out"); 1186 break; 1187 case DESC_ERR_DESC_READ_TLP_POISONED: 1188 rte_bbdev_log(ERR, "Descriptor read TLP poisoned"); 1189 break; 1190 case DESC_ERR_CB_READ_FAIL: 1191 rte_bbdev_log(ERR, "Unsuccessful completion for code block"); 1192 break; 1193 case DESC_ERR_CB_READ_TIMEOUT: 1194 rte_bbdev_log(ERR, "Code block read time-out"); 1195 break; 1196 case DESC_ERR_CB_READ_TLP_POISONED: 1197 rte_bbdev_log(ERR, "Code block read TLP poisoned"); 1198 break; 1199 default: 1200 rte_bbdev_log(ERR, "Descriptor error unknown error code %u", 1201 error_code); 1202 break; 1203 } 1204 return 1; 1205 } 1206 1207 /** 1208 * Set DMA descriptor for encode operation (1 Code Block) 1209 * 1210 * @param op 1211 * Pointer to a single encode operation. 1212 * @param desc 1213 * Pointer to DMA descriptor. 1214 * @param input 1215 * Pointer to pointer to input data which will be decoded. 1216 * @param k 1217 * K value (length of input in bits). 1218 * @param e 1219 * E value (length of output in bits). 1220 * @param ncb 1221 * Ncb value (size of the soft buffer). 1222 * @param out_length 1223 * Length of output buffer 1224 * @param in_offset 1225 * Input offset in rte_mbuf structure. It is used for calculating the point 1226 * where data is starting. 1227 * @param out_offset 1228 * Output offset in rte_mbuf structure. It is used for calculating the point 1229 * where hard output data will be stored. 1230 * @param cbs_in_op 1231 * Number of CBs contained in one operation. 1232 */ 1233 static inline int 1234 fpga_dma_desc_te_fill(struct rte_bbdev_enc_op *op, 1235 struct fpga_dma_enc_desc *desc, struct rte_mbuf *input, 1236 struct rte_mbuf *output, uint16_t k, uint16_t e, uint16_t ncb, 1237 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset, 1238 uint8_t cbs_in_op) 1239 1240 { 1241 /* reset */ 1242 desc->done = 0; 1243 desc->crc_en = check_bit(op->turbo_enc.op_flags, 1244 RTE_BBDEV_TURBO_CRC_24B_ATTACH); 1245 desc->bypass_rm = !check_bit(op->turbo_enc.op_flags, 1246 RTE_BBDEV_TURBO_RATE_MATCH); 1247 desc->k = k; 1248 desc->e = e; 1249 desc->ncb = ncb; 1250 desc->rv = op->turbo_enc.rv_index; 1251 desc->offset = desc_offset; 1252 /* Set inbound data buffer address */ 1253 desc->in_addr_hi = (uint32_t)( 1254 rte_pktmbuf_iova_offset(input, in_offset) >> 32); 1255 desc->in_addr_lw = (uint32_t)( 1256 rte_pktmbuf_iova_offset(input, in_offset)); 1257 1258 desc->out_addr_hi = (uint32_t)( 1259 rte_pktmbuf_iova_offset(output, out_offset) >> 32); 1260 desc->out_addr_lw = (uint32_t)( 1261 rte_pktmbuf_iova_offset(output, out_offset)); 1262 1263 /* Save software context needed for dequeue */ 1264 desc->op_addr = op; 1265 1266 /* Set total number of CBs in an op */ 1267 desc->cbs_in_op = cbs_in_op; 1268 1269 return 0; 1270 } 1271 1272 /** 1273 * Set DMA descriptor for encode operation (1 Code Block) 1274 * 1275 * @param op 1276 * Pointer to a single encode operation. 1277 * @param desc 1278 * Pointer to DMA descriptor. 1279 * @param input 1280 * Pointer to pointer to input data which will be decoded. 1281 * @param in_length 1282 * Length of an input. 1283 * @param k 1284 * K value (length of an output in bits). 1285 * @param in_offset 1286 * Input offset in rte_mbuf structure. It is used for calculating the point 1287 * where data is starting. 1288 * @param out_offset 1289 * Output offset in rte_mbuf structure. It is used for calculating the point 1290 * where hard output data will be stored. 1291 * @param cbs_in_op 1292 * Number of CBs contained in one operation. 1293 */ 1294 static inline int 1295 fpga_dma_desc_td_fill(struct rte_bbdev_dec_op *op, 1296 struct fpga_dma_dec_desc *desc, struct rte_mbuf *input, 1297 struct rte_mbuf *output, uint16_t in_length, uint16_t k, 1298 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset, 1299 uint8_t cbs_in_op) 1300 { 1301 /* reset */ 1302 desc->done = 0; 1303 /* Set inbound data buffer address */ 1304 desc->in_addr_hi = (uint32_t)( 1305 rte_pktmbuf_iova_offset(input, in_offset) >> 32); 1306 desc->in_addr_lw = (uint32_t)( 1307 rte_pktmbuf_iova_offset(input, in_offset)); 1308 desc->in_len = in_length; 1309 desc->k = k; 1310 desc->crc_type = !check_bit(op->turbo_dec.op_flags, 1311 RTE_BBDEV_TURBO_CRC_TYPE_24B); 1312 if ((op->turbo_dec.code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) 1313 && !check_bit(op->turbo_dec.op_flags, 1314 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP)) 1315 desc->drop_crc = 1; 1316 desc->max_iter = op->turbo_dec.iter_max * 2; 1317 desc->offset = desc_offset; 1318 desc->out_addr_hi = (uint32_t)( 1319 rte_pktmbuf_iova_offset(output, out_offset) >> 32); 1320 desc->out_addr_lw = (uint32_t)( 1321 rte_pktmbuf_iova_offset(output, out_offset)); 1322 1323 /* Save software context needed for dequeue */ 1324 desc->op_addr = op; 1325 1326 /* Set total number of CBs in an op */ 1327 desc->cbs_in_op = cbs_in_op; 1328 1329 return 0; 1330 } 1331 1332 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1333 /* Validates turbo encoder parameters */ 1334 static int 1335 validate_enc_op(struct rte_bbdev_enc_op *op) 1336 { 1337 struct rte_bbdev_op_turbo_enc *turbo_enc = &op->turbo_enc; 1338 struct rte_bbdev_op_enc_turbo_cb_params *cb = NULL; 1339 struct rte_bbdev_op_enc_turbo_tb_params *tb = NULL; 1340 uint16_t kw, kw_neg, kw_pos; 1341 1342 if (turbo_enc->input.length > 1343 RTE_BBDEV_TURBO_MAX_TB_SIZE >> 3) { 1344 rte_bbdev_log(ERR, "TB size (%u) is too big, max: %d", 1345 turbo_enc->input.length, 1346 RTE_BBDEV_TURBO_MAX_TB_SIZE); 1347 op->status = 1 << RTE_BBDEV_DATA_ERROR; 1348 return -1; 1349 } 1350 1351 if (op->mempool == NULL) { 1352 rte_bbdev_log(ERR, "Invalid mempool pointer"); 1353 return -1; 1354 } 1355 if (turbo_enc->input.data == NULL) { 1356 rte_bbdev_log(ERR, "Invalid input pointer"); 1357 return -1; 1358 } 1359 if (turbo_enc->output.data == NULL) { 1360 rte_bbdev_log(ERR, "Invalid output pointer"); 1361 return -1; 1362 } 1363 if (turbo_enc->rv_index > 3) { 1364 rte_bbdev_log(ERR, 1365 "rv_index (%u) is out of range 0 <= value <= 3", 1366 turbo_enc->rv_index); 1367 return -1; 1368 } 1369 if (turbo_enc->code_block_mode != RTE_BBDEV_TRANSPORT_BLOCK && 1370 turbo_enc->code_block_mode != RTE_BBDEV_CODE_BLOCK) { 1371 rte_bbdev_log(ERR, 1372 "code_block_mode (%u) is out of range 0 <= value <= 1", 1373 turbo_enc->code_block_mode); 1374 return -1; 1375 } 1376 1377 if (turbo_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1378 tb = &turbo_enc->tb_params; 1379 if ((tb->k_neg < RTE_BBDEV_TURBO_MIN_CB_SIZE 1380 || tb->k_neg > RTE_BBDEV_TURBO_MAX_CB_SIZE) 1381 && tb->c_neg > 0) { 1382 rte_bbdev_log(ERR, 1383 "k_neg (%u) is out of range %u <= value <= %u", 1384 tb->k_neg, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1385 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1386 return -1; 1387 } 1388 if (tb->k_pos < RTE_BBDEV_TURBO_MIN_CB_SIZE 1389 || tb->k_pos > RTE_BBDEV_TURBO_MAX_CB_SIZE) { 1390 rte_bbdev_log(ERR, 1391 "k_pos (%u) is out of range %u <= value <= %u", 1392 tb->k_pos, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1393 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1394 return -1; 1395 } 1396 if (tb->c_neg > (RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1)) 1397 rte_bbdev_log(ERR, 1398 "c_neg (%u) is out of range 0 <= value <= %u", 1399 tb->c_neg, 1400 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1); 1401 if (tb->c < 1 || tb->c > RTE_BBDEV_TURBO_MAX_CODE_BLOCKS) { 1402 rte_bbdev_log(ERR, 1403 "c (%u) is out of range 1 <= value <= %u", 1404 tb->c, RTE_BBDEV_TURBO_MAX_CODE_BLOCKS); 1405 return -1; 1406 } 1407 if (tb->cab > tb->c) { 1408 rte_bbdev_log(ERR, 1409 "cab (%u) is greater than c (%u)", 1410 tb->cab, tb->c); 1411 return -1; 1412 } 1413 if ((tb->ea < RTE_BBDEV_TURBO_MIN_CB_SIZE || (tb->ea % 2)) 1414 && tb->r < tb->cab) { 1415 rte_bbdev_log(ERR, 1416 "ea (%u) is less than %u or it is not even", 1417 tb->ea, RTE_BBDEV_TURBO_MIN_CB_SIZE); 1418 return -1; 1419 } 1420 if ((tb->eb < RTE_BBDEV_TURBO_MIN_CB_SIZE || (tb->eb % 2)) 1421 && tb->c > tb->cab) { 1422 rte_bbdev_log(ERR, 1423 "eb (%u) is less than %u or it is not even", 1424 tb->eb, RTE_BBDEV_TURBO_MIN_CB_SIZE); 1425 return -1; 1426 } 1427 1428 kw_neg = 3 * RTE_ALIGN_CEIL(tb->k_neg + 4, 1429 RTE_BBDEV_TURBO_C_SUBBLOCK); 1430 if (tb->ncb_neg < tb->k_neg || tb->ncb_neg > kw_neg) { 1431 rte_bbdev_log(ERR, 1432 "ncb_neg (%u) is out of range (%u) k_neg <= value <= (%u) kw_neg", 1433 tb->ncb_neg, tb->k_neg, kw_neg); 1434 return -1; 1435 } 1436 1437 kw_pos = 3 * RTE_ALIGN_CEIL(tb->k_pos + 4, 1438 RTE_BBDEV_TURBO_C_SUBBLOCK); 1439 if (tb->ncb_pos < tb->k_pos || tb->ncb_pos > kw_pos) { 1440 rte_bbdev_log(ERR, 1441 "ncb_pos (%u) is out of range (%u) k_pos <= value <= (%u) kw_pos", 1442 tb->ncb_pos, tb->k_pos, kw_pos); 1443 return -1; 1444 } 1445 if (tb->r > (tb->c - 1)) { 1446 rte_bbdev_log(ERR, 1447 "r (%u) is greater than c - 1 (%u)", 1448 tb->r, tb->c - 1); 1449 return -1; 1450 } 1451 } else { 1452 cb = &turbo_enc->cb_params; 1453 if (cb->k < RTE_BBDEV_TURBO_MIN_CB_SIZE 1454 || cb->k > RTE_BBDEV_TURBO_MAX_CB_SIZE) { 1455 rte_bbdev_log(ERR, 1456 "k (%u) is out of range %u <= value <= %u", 1457 cb->k, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1458 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1459 return -1; 1460 } 1461 1462 if (cb->e < RTE_BBDEV_TURBO_MIN_CB_SIZE || (cb->e % 2)) { 1463 rte_bbdev_log(ERR, 1464 "e (%u) is less than %u or it is not even", 1465 cb->e, RTE_BBDEV_TURBO_MIN_CB_SIZE); 1466 return -1; 1467 } 1468 1469 kw = RTE_ALIGN_CEIL(cb->k + 4, RTE_BBDEV_TURBO_C_SUBBLOCK) * 3; 1470 if (cb->ncb < cb->k || cb->ncb > kw) { 1471 rte_bbdev_log(ERR, 1472 "ncb (%u) is out of range (%u) k <= value <= (%u) kw", 1473 cb->ncb, cb->k, kw); 1474 return -1; 1475 } 1476 } 1477 1478 return 0; 1479 } 1480 #endif 1481 1482 static inline char * 1483 mbuf_append(struct rte_mbuf *m_head, struct rte_mbuf *m, uint16_t len) 1484 { 1485 if (unlikely(len > rte_pktmbuf_tailroom(m))) 1486 return NULL; 1487 1488 char *tail = (char *)m->buf_addr + m->data_off + m->data_len; 1489 m->data_len = (uint16_t)(m->data_len + len); 1490 m_head->pkt_len = (m_head->pkt_len + len); 1491 return tail; 1492 } 1493 1494 static inline int 1495 enqueue_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op *op, 1496 uint16_t desc_offset) 1497 { 1498 union fpga_dma_desc *desc; 1499 struct rte_mbuf *input; 1500 struct rte_mbuf *output; 1501 int ret; 1502 uint16_t k, e, ncb, ring_offset; 1503 uint32_t total_left, in_length, out_length, in_offset, out_offset; 1504 1505 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1506 /* Validate op structure */ 1507 if (validate_enc_op(op) == -1) { 1508 rte_bbdev_log(ERR, "Turbo encoder validation failed"); 1509 return -EINVAL; 1510 } 1511 #endif 1512 1513 input = op->turbo_enc.input.data; 1514 output = op->turbo_enc.output.data; 1515 in_offset = op->turbo_enc.input.offset; 1516 out_offset = op->turbo_enc.output.offset; 1517 total_left = op->turbo_enc.input.length; 1518 k = op->turbo_enc.cb_params.k; 1519 e = op->turbo_enc.cb_params.e; 1520 ncb = op->turbo_enc.cb_params.ncb; 1521 1522 if (check_bit(op->turbo_enc.op_flags, RTE_BBDEV_TURBO_CRC_24B_ATTACH)) 1523 in_length = ((k - 24) >> 3); 1524 else 1525 in_length = k >> 3; 1526 1527 if (check_bit(op->turbo_enc.op_flags, RTE_BBDEV_TURBO_RATE_MATCH)) 1528 out_length = (e + 7) >> 3; 1529 else 1530 out_length = (k >> 3) * 3 + 2; 1531 1532 mbuf_append(output, output, out_length); 1533 1534 /* Offset into the ring */ 1535 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1536 /* Setup DMA Descriptor */ 1537 desc = q->ring_addr + ring_offset; 1538 1539 ret = fpga_dma_desc_te_fill(op, &desc->enc_req, input, output, k, e, 1540 ncb, in_offset, out_offset, ring_offset, 1); 1541 if (unlikely(ret < 0)) 1542 return ret; 1543 1544 /* Update lengths */ 1545 total_left -= in_length; 1546 op->turbo_enc.output.length += out_length; 1547 1548 if (total_left > 0) { 1549 rte_bbdev_log(ERR, 1550 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u", 1551 total_left, in_length); 1552 return -1; 1553 } 1554 1555 return 1; 1556 } 1557 1558 static inline int 1559 enqueue_enc_one_op_tb(struct fpga_queue *q, struct rte_bbdev_enc_op *op, 1560 uint16_t desc_offset, uint8_t cbs_in_op) 1561 { 1562 union fpga_dma_desc *desc; 1563 struct rte_mbuf *input, *output_head, *output; 1564 int ret; 1565 uint8_t r, c, crc24_bits = 0; 1566 uint16_t k, e, ncb, ring_offset; 1567 uint32_t mbuf_total_left, in_length, out_length, in_offset, out_offset; 1568 uint32_t seg_total_left; 1569 uint16_t current_enqueued_cbs = 0; 1570 1571 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1572 /* Validate op structure */ 1573 if (validate_enc_op(op) == -1) { 1574 rte_bbdev_log(ERR, "Turbo encoder validation failed"); 1575 return -EINVAL; 1576 } 1577 #endif 1578 1579 input = op->turbo_enc.input.data; 1580 output_head = output = op->turbo_enc.output.data; 1581 in_offset = op->turbo_enc.input.offset; 1582 out_offset = op->turbo_enc.output.offset; 1583 mbuf_total_left = op->turbo_enc.input.length; 1584 1585 c = op->turbo_enc.tb_params.c; 1586 r = op->turbo_enc.tb_params.r; 1587 1588 if (check_bit(op->turbo_enc.op_flags, RTE_BBDEV_TURBO_CRC_24B_ATTACH)) 1589 crc24_bits = 24; 1590 1591 while (mbuf_total_left > 0 && r < c && input != NULL) { 1592 seg_total_left = rte_pktmbuf_data_len(input) - in_offset; 1593 1594 e = (r < op->turbo_enc.tb_params.cab) ? 1595 op->turbo_enc.tb_params.ea : 1596 op->turbo_enc.tb_params.eb; 1597 k = (r < op->turbo_enc.tb_params.c_neg) ? 1598 op->turbo_enc.tb_params.k_neg : 1599 op->turbo_enc.tb_params.k_pos; 1600 ncb = (r < op->turbo_enc.tb_params.c_neg) ? 1601 op->turbo_enc.tb_params.ncb_neg : 1602 op->turbo_enc.tb_params.ncb_pos; 1603 1604 in_length = ((k - crc24_bits) >> 3); 1605 1606 if (check_bit(op->turbo_enc.op_flags, 1607 RTE_BBDEV_TURBO_RATE_MATCH)) 1608 out_length = (e + 7) >> 3; 1609 else 1610 out_length = (k >> 3) * 3 + 2; 1611 1612 mbuf_append(output_head, output, out_length); 1613 1614 /* Setup DMA Descriptor */ 1615 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1616 desc = q->ring_addr + ring_offset; 1617 ret = fpga_dma_desc_te_fill(op, &desc->enc_req, input, output, 1618 k, e, ncb, in_offset, out_offset, ring_offset, 1619 cbs_in_op); 1620 if (unlikely(ret < 0)) 1621 return ret; 1622 1623 rte_bbdev_log_debug("DMA request desc %p", desc); 1624 1625 /* Update lengths */ 1626 op->turbo_enc.output.length += out_length; 1627 mbuf_total_left -= in_length; 1628 1629 /* Update offsets */ 1630 if (seg_total_left == in_length) { 1631 /* Go to the next mbuf */ 1632 input = input->next; 1633 output = output->next; 1634 in_offset = 0; 1635 out_offset = 0; 1636 } else { 1637 in_offset += in_length; 1638 out_offset += out_length; 1639 } 1640 1641 r++; 1642 desc_offset++; 1643 current_enqueued_cbs++; 1644 } 1645 1646 if (mbuf_total_left > 0) { 1647 rte_bbdev_log(ERR, 1648 "Some date still left for processing: mbuf_total_left = %u", 1649 mbuf_total_left); 1650 return -1; 1651 } 1652 1653 return current_enqueued_cbs; 1654 } 1655 1656 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1657 /* Validates turbo decoder parameters */ 1658 static int 1659 validate_dec_op(struct rte_bbdev_dec_op *op) 1660 { 1661 struct rte_bbdev_op_turbo_dec *turbo_dec = &op->turbo_dec; 1662 struct rte_bbdev_op_dec_turbo_cb_params *cb = NULL; 1663 struct rte_bbdev_op_dec_turbo_tb_params *tb = NULL; 1664 1665 if (op->mempool == NULL) { 1666 rte_bbdev_log(ERR, "Invalid mempool pointer"); 1667 return -1; 1668 } 1669 if (turbo_dec->input.data == NULL) { 1670 rte_bbdev_log(ERR, "Invalid input pointer"); 1671 return -1; 1672 } 1673 if (turbo_dec->hard_output.data == NULL) { 1674 rte_bbdev_log(ERR, "Invalid hard_output pointer"); 1675 return -1; 1676 } 1677 if (turbo_dec->rv_index > 3) { 1678 rte_bbdev_log(ERR, 1679 "rv_index (%u) is out of range 0 <= value <= 3", 1680 turbo_dec->rv_index); 1681 return -1; 1682 } 1683 if (turbo_dec->iter_min < 1) { 1684 rte_bbdev_log(ERR, 1685 "iter_min (%u) is less than 1", 1686 turbo_dec->iter_min); 1687 return -1; 1688 } 1689 if (turbo_dec->iter_max <= 2) { 1690 rte_bbdev_log(ERR, 1691 "iter_max (%u) is less than or equal to 2", 1692 turbo_dec->iter_max); 1693 return -1; 1694 } 1695 if (turbo_dec->iter_min > turbo_dec->iter_max) { 1696 rte_bbdev_log(ERR, 1697 "iter_min (%u) is greater than iter_max (%u)", 1698 turbo_dec->iter_min, turbo_dec->iter_max); 1699 return -1; 1700 } 1701 if (turbo_dec->code_block_mode != RTE_BBDEV_TRANSPORT_BLOCK && 1702 turbo_dec->code_block_mode != RTE_BBDEV_CODE_BLOCK) { 1703 rte_bbdev_log(ERR, 1704 "code_block_mode (%u) is out of range 0 <= value <= 1", 1705 turbo_dec->code_block_mode); 1706 return -1; 1707 } 1708 1709 if (turbo_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1710 1711 if ((turbo_dec->op_flags & 1712 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP) && 1713 !(turbo_dec->op_flags & RTE_BBDEV_TURBO_CRC_TYPE_24B)) { 1714 rte_bbdev_log(ERR, 1715 "RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP should accompany RTE_BBDEV_TURBO_CRC_TYPE_24B"); 1716 return -1; 1717 } 1718 1719 tb = &turbo_dec->tb_params; 1720 if ((tb->k_neg < RTE_BBDEV_TURBO_MIN_CB_SIZE 1721 || tb->k_neg > RTE_BBDEV_TURBO_MAX_CB_SIZE) 1722 && tb->c_neg > 0) { 1723 rte_bbdev_log(ERR, 1724 "k_neg (%u) is out of range %u <= value <= %u", 1725 tb->k_neg, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1726 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1727 return -1; 1728 } 1729 if ((tb->k_pos < RTE_BBDEV_TURBO_MIN_CB_SIZE 1730 || tb->k_pos > RTE_BBDEV_TURBO_MAX_CB_SIZE) 1731 && tb->c > tb->c_neg) { 1732 rte_bbdev_log(ERR, 1733 "k_pos (%u) is out of range %u <= value <= %u", 1734 tb->k_pos, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1735 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1736 return -1; 1737 } 1738 if (tb->c_neg > (RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1)) 1739 rte_bbdev_log(ERR, 1740 "c_neg (%u) is out of range 0 <= value <= %u", 1741 tb->c_neg, 1742 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1); 1743 if (tb->c < 1 || tb->c > RTE_BBDEV_TURBO_MAX_CODE_BLOCKS) { 1744 rte_bbdev_log(ERR, 1745 "c (%u) is out of range 1 <= value <= %u", 1746 tb->c, RTE_BBDEV_TURBO_MAX_CODE_BLOCKS); 1747 return -1; 1748 } 1749 if (tb->cab > tb->c) { 1750 rte_bbdev_log(ERR, 1751 "cab (%u) is greater than c (%u)", 1752 tb->cab, tb->c); 1753 return -1; 1754 } 1755 } else { 1756 1757 if (turbo_dec->op_flags & RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP) { 1758 rte_bbdev_log(ERR, 1759 "RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP is invalid in CB-mode"); 1760 return -1; 1761 } 1762 1763 cb = &turbo_dec->cb_params; 1764 if (cb->k < RTE_BBDEV_TURBO_MIN_CB_SIZE 1765 || cb->k > RTE_BBDEV_TURBO_MAX_CB_SIZE) { 1766 rte_bbdev_log(ERR, 1767 "k (%u) is out of range %u <= value <= %u", 1768 cb->k, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1769 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1770 return -1; 1771 } 1772 } 1773 1774 return 0; 1775 } 1776 #endif 1777 1778 static inline int 1779 enqueue_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op *op, 1780 uint16_t desc_offset) 1781 { 1782 union fpga_dma_desc *desc; 1783 struct rte_mbuf *input; 1784 struct rte_mbuf *output; 1785 int ret; 1786 uint16_t k, kw, ring_offset; 1787 uint32_t total_left, in_length, out_length, in_offset, out_offset; 1788 1789 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1790 /* Validate op structure */ 1791 if (validate_dec_op(op) == -1) { 1792 rte_bbdev_log(ERR, "Turbo decoder validation failed"); 1793 return -EINVAL; 1794 } 1795 #endif 1796 1797 input = op->turbo_dec.input.data; 1798 output = op->turbo_dec.hard_output.data; 1799 total_left = op->turbo_dec.input.length; 1800 in_offset = op->turbo_dec.input.offset; 1801 out_offset = op->turbo_dec.hard_output.offset; 1802 1803 k = op->turbo_dec.cb_params.k; 1804 kw = RTE_ALIGN_CEIL(k + 4, 32) * 3; 1805 in_length = kw; 1806 out_length = k >> 3; 1807 1808 mbuf_append(output, output, out_length); 1809 1810 /* Setup DMA Descriptor */ 1811 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1812 desc = q->ring_addr + ring_offset; 1813 ret = fpga_dma_desc_td_fill(op, &desc->dec_req, input, output, 1814 in_length, k, in_offset, out_offset, ring_offset, 1); 1815 if (unlikely(ret < 0)) 1816 return ret; 1817 1818 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1819 print_dma_dec_desc_debug_info(desc); 1820 #endif 1821 1822 /* Update lengths */ 1823 total_left -= in_length; 1824 op->turbo_dec.hard_output.length += out_length; 1825 1826 if (total_left > 0) { 1827 rte_bbdev_log(ERR, 1828 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u", 1829 total_left, in_length); 1830 return -1; 1831 } 1832 1833 return 1; 1834 } 1835 1836 1837 static inline int 1838 enqueue_dec_one_op_tb(struct fpga_queue *q, struct rte_bbdev_dec_op *op, 1839 uint16_t desc_offset, uint8_t cbs_in_op) 1840 { 1841 union fpga_dma_desc *desc; 1842 struct rte_mbuf *input, *output_head, *output; 1843 int ret; 1844 uint8_t r, c; 1845 uint16_t k, kw, in_length, out_length, ring_offset; 1846 uint32_t mbuf_total_left, seg_total_left, in_offset, out_offset; 1847 uint16_t current_enqueued_cbs = 0; 1848 uint16_t crc24_overlap = 0; 1849 1850 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1851 /* Validate op structure */ 1852 if (validate_dec_op(op) == -1) { 1853 rte_bbdev_log(ERR, "Turbo decoder validation failed"); 1854 return -EINVAL; 1855 } 1856 #endif 1857 1858 input = op->turbo_dec.input.data; 1859 output_head = output = op->turbo_dec.hard_output.data; 1860 mbuf_total_left = op->turbo_dec.input.length; 1861 in_offset = op->turbo_dec.input.offset; 1862 out_offset = op->turbo_dec.hard_output.offset; 1863 1864 if (!check_bit(op->turbo_dec.op_flags, 1865 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP)) 1866 crc24_overlap = 24; 1867 1868 c = op->turbo_dec.tb_params.c; 1869 r = op->turbo_dec.tb_params.r; 1870 1871 while (mbuf_total_left > 0 && r < c && input != NULL) { 1872 seg_total_left = rte_pktmbuf_data_len(input) - in_offset; 1873 k = (r < op->turbo_dec.tb_params.c_neg) ? 1874 op->turbo_dec.tb_params.k_neg : 1875 op->turbo_dec.tb_params.k_pos; 1876 kw = RTE_ALIGN_CEIL(k + 4, 32) * 3; 1877 1878 in_length = kw; 1879 out_length = (k - crc24_overlap) >> 3; 1880 1881 mbuf_append(output_head, output, out_length); 1882 1883 if (seg_total_left < in_length) { 1884 rte_bbdev_log(ERR, 1885 "Partial CB found in a TB. FPGA Driver doesn't support scatter-gather operations!"); 1886 return -1; 1887 } 1888 1889 /* Setup DMA Descriptor */ 1890 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1891 desc = q->ring_addr + ring_offset; 1892 ret = fpga_dma_desc_td_fill(op, &desc->dec_req, input, output, 1893 in_length, k, in_offset, out_offset, 1894 ring_offset, cbs_in_op); 1895 if (unlikely(ret < 0)) 1896 return ret; 1897 1898 /* Update lengths */ 1899 ret = rte_pktmbuf_trim(op->turbo_dec.hard_output.data, 1900 (crc24_overlap >> 3)); 1901 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1902 if (ret < 0) { 1903 rte_bbdev_log(ERR, 1904 "The length to remove is greater than the length of the last segment"); 1905 return -EINVAL; 1906 } 1907 #endif 1908 op->turbo_dec.hard_output.length += out_length; 1909 mbuf_total_left -= in_length; 1910 1911 /* Update offsets */ 1912 if (seg_total_left == in_length) { 1913 /* Go to the next mbuf */ 1914 input = input->next; 1915 output = output->next; 1916 in_offset = 0; 1917 out_offset = 0; 1918 } else { 1919 in_offset += in_length; 1920 out_offset += out_length; 1921 } 1922 1923 r++; 1924 desc_offset++; 1925 current_enqueued_cbs++; 1926 } 1927 1928 if (mbuf_total_left > 0) { 1929 rte_bbdev_log(ERR, 1930 "Some date still left for processing: mbuf_total_left = %u", 1931 mbuf_total_left); 1932 return -1; 1933 } 1934 1935 return current_enqueued_cbs; 1936 } 1937 1938 static uint16_t 1939 fpga_enqueue_enc(struct rte_bbdev_queue_data *q_data, 1940 struct rte_bbdev_enc_op **ops, uint16_t num) 1941 { 1942 uint8_t cbs_in_op; 1943 uint16_t i, total_enqueued_cbs = 0; 1944 int32_t avail; 1945 int enqueued_cbs; 1946 struct fpga_queue *q = q_data->queue_private; 1947 union fpga_dma_desc *desc; 1948 1949 /* Check if queue is not full */ 1950 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) == 1951 q->head_free_desc)) 1952 return 0; 1953 1954 /* Calculates available space */ 1955 avail = (q->head_free_desc > q->tail) ? 1956 q->head_free_desc - q->tail - 1 : 1957 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1; 1958 1959 for (i = 0; i < num; ++i) { 1960 if (ops[i]->turbo_enc.code_block_mode == 1961 RTE_BBDEV_TRANSPORT_BLOCK) { 1962 cbs_in_op = get_num_cbs_in_op_enc(&ops[i]->turbo_enc); 1963 /* Check if there is available space for further 1964 * processing 1965 */ 1966 if (unlikely(avail - cbs_in_op < 0)) 1967 break; 1968 avail -= cbs_in_op; 1969 enqueued_cbs = enqueue_enc_one_op_tb(q, ops[i], 1970 total_enqueued_cbs, cbs_in_op); 1971 } else { 1972 /* Check if there is available space for further 1973 * processing 1974 */ 1975 if (unlikely(avail - 1 < 0)) 1976 break; 1977 avail -= 1; 1978 enqueued_cbs = enqueue_enc_one_op_cb(q, ops[i], 1979 total_enqueued_cbs); 1980 } 1981 1982 if (enqueued_cbs < 0) 1983 break; 1984 1985 total_enqueued_cbs += enqueued_cbs; 1986 1987 rte_bbdev_log_debug("enqueuing enc ops [%d/%d] | head %d | tail %d", 1988 total_enqueued_cbs, num, 1989 q->head_free_desc, q->tail); 1990 } 1991 1992 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt 1993 * only when all previous CBs were already processed. 1994 */ 1995 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1) 1996 & q->sw_ring_wrap_mask); 1997 desc->enc_req.irq_en = q->irq_enable; 1998 1999 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats); 2000 2001 /* Update stats */ 2002 q_data->queue_stats.enqueued_count += i; 2003 q_data->queue_stats.enqueue_err_count += num - i; 2004 2005 return i; 2006 } 2007 2008 static uint16_t 2009 fpga_enqueue_dec(struct rte_bbdev_queue_data *q_data, 2010 struct rte_bbdev_dec_op **ops, uint16_t num) 2011 { 2012 uint8_t cbs_in_op; 2013 uint16_t i, total_enqueued_cbs = 0; 2014 int32_t avail; 2015 int enqueued_cbs; 2016 struct fpga_queue *q = q_data->queue_private; 2017 union fpga_dma_desc *desc; 2018 2019 /* Check if queue is not full */ 2020 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) == 2021 q->head_free_desc)) 2022 return 0; 2023 2024 /* Calculates available space */ 2025 avail = (q->head_free_desc > q->tail) ? 2026 q->head_free_desc - q->tail - 1 : 2027 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1; 2028 2029 for (i = 0; i < num; ++i) { 2030 if (ops[i]->turbo_dec.code_block_mode == 2031 RTE_BBDEV_TRANSPORT_BLOCK) { 2032 cbs_in_op = get_num_cbs_in_op_dec(&ops[i]->turbo_dec); 2033 /* Check if there is available space for further 2034 * processing 2035 */ 2036 if (unlikely(avail - cbs_in_op < 0)) 2037 break; 2038 avail -= cbs_in_op; 2039 enqueued_cbs = enqueue_dec_one_op_tb(q, ops[i], 2040 total_enqueued_cbs, cbs_in_op); 2041 } else { 2042 /* Check if there is available space for further 2043 * processing 2044 */ 2045 if (unlikely(avail - 1 < 0)) 2046 break; 2047 avail -= 1; 2048 enqueued_cbs = enqueue_dec_one_op_cb(q, ops[i], 2049 total_enqueued_cbs); 2050 } 2051 2052 if (enqueued_cbs < 0) 2053 break; 2054 2055 total_enqueued_cbs += enqueued_cbs; 2056 2057 rte_bbdev_log_debug("enqueuing dec ops [%d/%d] | head %d | tail %d", 2058 total_enqueued_cbs, num, 2059 q->head_free_desc, q->tail); 2060 } 2061 2062 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt 2063 * only when all previous CBs were already processed. 2064 */ 2065 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1) 2066 & q->sw_ring_wrap_mask); 2067 desc->dec_req.irq_en = q->irq_enable; 2068 2069 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats); 2070 2071 /* Update stats */ 2072 q_data->queue_stats.enqueued_count += i; 2073 q_data->queue_stats.enqueue_err_count += num - i; 2074 2075 return i; 2076 } 2077 2078 static inline int 2079 dequeue_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op **op, 2080 uint16_t desc_offset) 2081 { 2082 union fpga_dma_desc *desc; 2083 int desc_error = 0; 2084 2085 /* Set current desc */ 2086 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2087 & q->sw_ring_wrap_mask); 2088 2089 /*check if done */ 2090 if (desc->enc_req.done == 0) 2091 return -1; 2092 2093 /* make sure the response is read atomically */ 2094 rte_smp_rmb(); 2095 2096 rte_bbdev_log_debug("DMA response desc %p", desc); 2097 2098 *op = desc->enc_req.op_addr; 2099 /* Check the descriptor error field, return 1 on error */ 2100 desc_error = check_desc_error(desc->enc_req.error); 2101 (*op)->status = desc_error << RTE_BBDEV_DATA_ERROR; 2102 2103 return 1; 2104 } 2105 2106 static inline int 2107 dequeue_enc_one_op_tb(struct fpga_queue *q, struct rte_bbdev_enc_op **op, 2108 uint16_t desc_offset) 2109 { 2110 union fpga_dma_desc *desc; 2111 uint8_t cbs_in_op, cb_idx; 2112 int desc_error = 0; 2113 int status = 0; 2114 2115 /* Set descriptor */ 2116 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2117 & q->sw_ring_wrap_mask); 2118 2119 /* Verify if done bit is set */ 2120 if (desc->enc_req.done == 0) 2121 return -1; 2122 2123 /* Make sure the response is read atomically */ 2124 rte_smp_rmb(); 2125 2126 /* Verify if done bit in all CBs is set */ 2127 cbs_in_op = desc->enc_req.cbs_in_op; 2128 for (cb_idx = 1; cb_idx < cbs_in_op; ++cb_idx) { 2129 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2130 cb_idx) & q->sw_ring_wrap_mask); 2131 if (desc->enc_req.done == 0) 2132 return -1; 2133 } 2134 2135 /* Make sure the response is read atomically */ 2136 rte_smp_rmb(); 2137 2138 for (cb_idx = 0; cb_idx < cbs_in_op; ++cb_idx) { 2139 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2140 cb_idx) & q->sw_ring_wrap_mask); 2141 /* Check the descriptor error field, return 1 on error */ 2142 desc_error = check_desc_error(desc->enc_req.error); 2143 status |= desc_error << RTE_BBDEV_DATA_ERROR; 2144 rte_bbdev_log_debug("DMA response desc %p", desc); 2145 } 2146 2147 *op = desc->enc_req.op_addr; 2148 (*op)->status = status; 2149 return cbs_in_op; 2150 } 2151 2152 static inline int 2153 dequeue_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op **op, 2154 uint16_t desc_offset) 2155 { 2156 union fpga_dma_desc *desc; 2157 int desc_error = 0; 2158 /* Set descriptor */ 2159 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2160 & q->sw_ring_wrap_mask); 2161 2162 /* Verify done bit is set */ 2163 if (desc->dec_req.done == 0) 2164 return -1; 2165 2166 /* make sure the response is read atomically */ 2167 rte_smp_rmb(); 2168 2169 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2170 print_dma_dec_desc_debug_info(desc); 2171 2172 #endif 2173 2174 *op = desc->dec_req.op_addr; 2175 /* FPGA reports in half-iterations, from 0 to 31. get ceiling */ 2176 (*op)->turbo_dec.iter_count = (desc->dec_req.iter + 2) >> 1; 2177 /* crc_pass = 0 when decoder fails */ 2178 (*op)->status = !(desc->dec_req.crc_pass) << RTE_BBDEV_CRC_ERROR; 2179 /* Check the descriptor error field, return 1 on error */ 2180 desc_error = check_desc_error(desc->enc_req.error); 2181 (*op)->status |= desc_error << RTE_BBDEV_DATA_ERROR; 2182 return 1; 2183 } 2184 2185 static inline int 2186 dequeue_dec_one_op_tb(struct fpga_queue *q, struct rte_bbdev_dec_op **op, 2187 uint16_t desc_offset) 2188 { 2189 union fpga_dma_desc *desc; 2190 uint8_t cbs_in_op, cb_idx, iter_count = 0; 2191 int status = 0; 2192 int desc_error = 0; 2193 /* Set descriptor */ 2194 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2195 & q->sw_ring_wrap_mask); 2196 2197 /* Verify if done bit is set */ 2198 if (desc->dec_req.done == 0) 2199 return -1; 2200 2201 /* Make sure the response is read atomically */ 2202 rte_smp_rmb(); 2203 2204 /* Verify if done bit in all CBs is set */ 2205 cbs_in_op = desc->dec_req.cbs_in_op; 2206 for (cb_idx = 1; cb_idx < cbs_in_op; ++cb_idx) { 2207 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2208 cb_idx) & q->sw_ring_wrap_mask); 2209 if (desc->dec_req.done == 0) 2210 return -1; 2211 } 2212 2213 /* Make sure the response is read atomically */ 2214 rte_smp_rmb(); 2215 2216 for (cb_idx = 0; cb_idx < cbs_in_op; ++cb_idx) { 2217 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2218 cb_idx) & q->sw_ring_wrap_mask); 2219 /* get max iter_count for all CBs in op */ 2220 iter_count = RTE_MAX(iter_count, (uint8_t) desc->dec_req.iter); 2221 /* crc_pass = 0 when decoder fails, one fails all */ 2222 status |= !(desc->dec_req.crc_pass) << RTE_BBDEV_CRC_ERROR; 2223 /* Check the descriptor error field, return 1 on error */ 2224 desc_error = check_desc_error(desc->enc_req.error); 2225 status |= desc_error << RTE_BBDEV_DATA_ERROR; 2226 rte_bbdev_log_debug("DMA response desc %p", desc); 2227 } 2228 2229 *op = desc->dec_req.op_addr; 2230 2231 /* FPGA reports in half-iterations, get ceiling */ 2232 (*op)->turbo_dec.iter_count = (iter_count + 2) >> 1; 2233 (*op)->status = status; 2234 return cbs_in_op; 2235 } 2236 2237 static uint16_t 2238 fpga_dequeue_enc(struct rte_bbdev_queue_data *q_data, 2239 struct rte_bbdev_enc_op **ops, uint16_t num) 2240 { 2241 struct fpga_queue *q = q_data->queue_private; 2242 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask; 2243 uint16_t i; 2244 uint16_t dequeued_cbs = 0; 2245 struct rte_bbdev_enc_op *op; 2246 int ret; 2247 2248 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) { 2249 op = (q->ring_addr + ((q->head_free_desc + dequeued_cbs) 2250 & q->sw_ring_wrap_mask))->enc_req.op_addr; 2251 if (op->turbo_enc.code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) 2252 ret = dequeue_enc_one_op_tb(q, &ops[i], dequeued_cbs); 2253 else 2254 ret = dequeue_enc_one_op_cb(q, &ops[i], dequeued_cbs); 2255 2256 if (ret < 0) 2257 break; 2258 2259 dequeued_cbs += ret; 2260 2261 rte_bbdev_log_debug("dequeuing enc ops [%d/%d] | head %d | tail %d", 2262 dequeued_cbs, num, q->head_free_desc, q->tail); 2263 } 2264 2265 /* Update head */ 2266 q->head_free_desc = (q->head_free_desc + dequeued_cbs) & 2267 q->sw_ring_wrap_mask; 2268 2269 /* Update stats */ 2270 q_data->queue_stats.dequeued_count += i; 2271 2272 return i; 2273 } 2274 2275 static uint16_t 2276 fpga_dequeue_dec(struct rte_bbdev_queue_data *q_data, 2277 struct rte_bbdev_dec_op **ops, uint16_t num) 2278 { 2279 struct fpga_queue *q = q_data->queue_private; 2280 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask; 2281 uint16_t i; 2282 uint16_t dequeued_cbs = 0; 2283 struct rte_bbdev_dec_op *op; 2284 int ret; 2285 2286 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) { 2287 op = (q->ring_addr + ((q->head_free_desc + dequeued_cbs) 2288 & q->sw_ring_wrap_mask))->dec_req.op_addr; 2289 if (op->turbo_dec.code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) 2290 ret = dequeue_dec_one_op_tb(q, &ops[i], dequeued_cbs); 2291 else 2292 ret = dequeue_dec_one_op_cb(q, &ops[i], dequeued_cbs); 2293 2294 if (ret < 0) 2295 break; 2296 2297 dequeued_cbs += ret; 2298 2299 rte_bbdev_log_debug("dequeuing dec ops [%d/%d] | head %d | tail %d", 2300 dequeued_cbs, num, q->head_free_desc, q->tail); 2301 } 2302 2303 /* Update head */ 2304 q->head_free_desc = (q->head_free_desc + dequeued_cbs) & 2305 q->sw_ring_wrap_mask; 2306 2307 /* Update stats */ 2308 q_data->queue_stats.dequeued_count += i; 2309 2310 return i; 2311 } 2312 2313 /* Initialization Function */ 2314 static void 2315 fpga_lte_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv) 2316 { 2317 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); 2318 2319 dev->dev_ops = &fpga_ops; 2320 dev->enqueue_enc_ops = fpga_enqueue_enc; 2321 dev->enqueue_dec_ops = fpga_enqueue_dec; 2322 dev->dequeue_enc_ops = fpga_dequeue_enc; 2323 dev->dequeue_dec_ops = fpga_dequeue_dec; 2324 2325 ((struct fpga_lte_fec_device *) dev->data->dev_private)->pf_device = 2326 !strcmp(drv->driver.name, 2327 RTE_STR(FPGA_LTE_FEC_PF_DRIVER_NAME)); 2328 ((struct fpga_lte_fec_device *) dev->data->dev_private)->mmio_base = 2329 pci_dev->mem_resource[0].addr; 2330 2331 rte_bbdev_log_debug( 2332 "Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64, 2333 drv->driver.name, dev->data->name, 2334 (void *)pci_dev->mem_resource[0].addr, 2335 pci_dev->mem_resource[0].phys_addr); 2336 } 2337 2338 static int 2339 fpga_lte_fec_probe(struct rte_pci_driver *pci_drv, 2340 struct rte_pci_device *pci_dev) 2341 { 2342 struct rte_bbdev *bbdev = NULL; 2343 char dev_name[RTE_BBDEV_NAME_MAX_LEN]; 2344 2345 if (pci_dev == NULL) { 2346 rte_bbdev_log(ERR, "NULL PCI device"); 2347 return -EINVAL; 2348 } 2349 2350 rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name)); 2351 2352 /* Allocate memory to be used privately by drivers */ 2353 bbdev = rte_bbdev_allocate(pci_dev->device.name); 2354 if (bbdev == NULL) 2355 return -ENODEV; 2356 2357 /* allocate device private memory */ 2358 bbdev->data->dev_private = rte_zmalloc_socket(dev_name, 2359 sizeof(struct fpga_lte_fec_device), RTE_CACHE_LINE_SIZE, 2360 pci_dev->device.numa_node); 2361 2362 if (bbdev->data->dev_private == NULL) { 2363 rte_bbdev_log(CRIT, 2364 "Allocate of %zu bytes for device \"%s\" failed", 2365 sizeof(struct fpga_lte_fec_device), dev_name); 2366 rte_bbdev_release(bbdev); 2367 return -ENOMEM; 2368 } 2369 2370 /* Fill HW specific part of device structure */ 2371 bbdev->device = &pci_dev->device; 2372 bbdev->intr_handle = pci_dev->intr_handle; 2373 bbdev->data->socket_id = pci_dev->device.numa_node; 2374 2375 /* Invoke FEC FPGA device initialization function */ 2376 fpga_lte_fec_init(bbdev, pci_drv); 2377 2378 rte_bbdev_log_debug("bbdev id = %u [%s]", 2379 bbdev->data->dev_id, dev_name); 2380 2381 struct fpga_lte_fec_device *d = bbdev->data->dev_private; 2382 uint32_t version_id = fpga_reg_read_32(d->mmio_base, 2383 FPGA_LTE_FEC_VERSION_ID); 2384 rte_bbdev_log(INFO, "FEC FPGA RTL v%u.%u", 2385 ((uint16_t)(version_id >> 16)), ((uint16_t)version_id)); 2386 2387 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2388 if (!strcmp(pci_drv->driver.name, 2389 RTE_STR(FPGA_LTE_FEC_PF_DRIVER_NAME))) 2390 print_static_reg_debug_info(d->mmio_base); 2391 #endif 2392 return 0; 2393 } 2394 2395 static int 2396 fpga_lte_fec_remove(struct rte_pci_device *pci_dev) 2397 { 2398 struct rte_bbdev *bbdev; 2399 int ret; 2400 uint8_t dev_id; 2401 2402 if (pci_dev == NULL) 2403 return -EINVAL; 2404 2405 /* Find device */ 2406 bbdev = rte_bbdev_get_named_dev(pci_dev->device.name); 2407 if (bbdev == NULL) { 2408 rte_bbdev_log(CRIT, 2409 "Couldn't find HW dev \"%s\" to uninitialise it", 2410 pci_dev->device.name); 2411 return -ENODEV; 2412 } 2413 dev_id = bbdev->data->dev_id; 2414 2415 /* free device private memory before close */ 2416 rte_free(bbdev->data->dev_private); 2417 2418 /* Close device */ 2419 ret = rte_bbdev_close(dev_id); 2420 if (ret < 0) 2421 rte_bbdev_log(ERR, 2422 "Device %i failed to close during uninit: %i", 2423 dev_id, ret); 2424 2425 /* release bbdev from library */ 2426 ret = rte_bbdev_release(bbdev); 2427 if (ret) 2428 rte_bbdev_log(ERR, "Device %i failed to uninit: %i", dev_id, 2429 ret); 2430 2431 rte_bbdev_log_debug("Destroyed bbdev = %u", dev_id); 2432 2433 return 0; 2434 } 2435 2436 static inline void 2437 set_default_fpga_conf(struct rte_fpga_lte_fec_conf *def_conf) 2438 { 2439 /* clear default configuration before initialization */ 2440 memset(def_conf, 0, sizeof(struct rte_fpga_lte_fec_conf)); 2441 /* Set pf mode to true */ 2442 def_conf->pf_mode_en = true; 2443 2444 /* Set ratio between UL and DL to 1:1 (unit of weight is 3 CBs) */ 2445 def_conf->ul_bandwidth = 3; 2446 def_conf->dl_bandwidth = 3; 2447 2448 /* Set Load Balance Factor to 64 */ 2449 def_conf->dl_load_balance = 64; 2450 def_conf->ul_load_balance = 64; 2451 } 2452 2453 /* Initial configuration of FPGA LTE FEC device */ 2454 int 2455 rte_fpga_lte_fec_configure(const char *dev_name, 2456 const struct rte_fpga_lte_fec_conf *conf) 2457 { 2458 uint32_t payload_32, address; 2459 uint16_t payload_16; 2460 uint8_t payload_8; 2461 uint16_t q_id, vf_id, total_q_id, total_ul_q_id, total_dl_q_id; 2462 struct rte_bbdev *bbdev = rte_bbdev_get_named_dev(dev_name); 2463 struct rte_fpga_lte_fec_conf def_conf; 2464 2465 if (bbdev == NULL) { 2466 rte_bbdev_log(ERR, 2467 "Invalid dev_name (%s), or device is not yet initialised", 2468 dev_name); 2469 return -ENODEV; 2470 } 2471 2472 struct fpga_lte_fec_device *d = bbdev->data->dev_private; 2473 2474 if (conf == NULL) { 2475 rte_bbdev_log(ERR, 2476 "FPGA Configuration was not provided. Default configuration will be loaded."); 2477 set_default_fpga_conf(&def_conf); 2478 conf = &def_conf; 2479 } 2480 2481 /* 2482 * Configure UL:DL ratio. 2483 * [7:0]: UL weight 2484 * [15:8]: DL weight 2485 */ 2486 payload_16 = (conf->dl_bandwidth << 8) | conf->ul_bandwidth; 2487 address = FPGA_LTE_FEC_CONFIGURATION; 2488 fpga_reg_write_16(d->mmio_base, address, payload_16); 2489 2490 /* Clear all queues registers */ 2491 payload_32 = FPGA_INVALID_HW_QUEUE_ID; 2492 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 2493 address = (q_id << 2) + FPGA_LTE_FEC_QUEUE_MAP; 2494 fpga_reg_write_32(d->mmio_base, address, payload_32); 2495 } 2496 2497 /* 2498 * If PF mode is enabled allocate all queues for PF only. 2499 * 2500 * For VF mode each VF can have different number of UL and DL queues. 2501 * Total number of queues to configure cannot exceed FPGA 2502 * capabilities - 64 queues - 32 queues for UL and 32 queues for DL. 2503 * Queues mapping is done according to configuration: 2504 * 2505 * UL queues: 2506 * | Q_ID | VF_ID | 2507 * | 0 | 0 | 2508 * | ... | 0 | 2509 * | conf->vf_dl_queues_number[0] - 1 | 0 | 2510 * | conf->vf_dl_queues_number[0] | 1 | 2511 * | ... | 1 | 2512 * | conf->vf_dl_queues_number[1] - 1 | 1 | 2513 * | ... | ... | 2514 * | conf->vf_dl_queues_number[7] - 1 | 7 | 2515 * 2516 * DL queues: 2517 * | Q_ID | VF_ID | 2518 * | 32 | 0 | 2519 * | ... | 0 | 2520 * | conf->vf_ul_queues_number[0] - 1 | 0 | 2521 * | conf->vf_ul_queues_number[0] | 1 | 2522 * | ... | 1 | 2523 * | conf->vf_ul_queues_number[1] - 1 | 1 | 2524 * | ... | ... | 2525 * | conf->vf_ul_queues_number[7] - 1 | 7 | 2526 * 2527 * Example of configuration: 2528 * conf->vf_ul_queues_number[0] = 4; -> 4 UL queues for VF0 2529 * conf->vf_dl_queues_number[0] = 4; -> 4 DL queues for VF0 2530 * conf->vf_ul_queues_number[1] = 2; -> 2 UL queues for VF1 2531 * conf->vf_dl_queues_number[1] = 2; -> 2 DL queues for VF1 2532 * 2533 * UL: 2534 * | Q_ID | VF_ID | 2535 * | 0 | 0 | 2536 * | 1 | 0 | 2537 * | 2 | 0 | 2538 * | 3 | 0 | 2539 * | 4 | 1 | 2540 * | 5 | 1 | 2541 * 2542 * DL: 2543 * | Q_ID | VF_ID | 2544 * | 32 | 0 | 2545 * | 33 | 0 | 2546 * | 34 | 0 | 2547 * | 35 | 0 | 2548 * | 36 | 1 | 2549 * | 37 | 1 | 2550 */ 2551 if (conf->pf_mode_en) { 2552 payload_32 = 0x1; 2553 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 2554 address = (q_id << 2) + FPGA_LTE_FEC_QUEUE_MAP; 2555 fpga_reg_write_32(d->mmio_base, address, payload_32); 2556 } 2557 } else { 2558 /* Calculate total number of UL and DL queues to configure */ 2559 total_ul_q_id = total_dl_q_id = 0; 2560 for (vf_id = 0; vf_id < FPGA_LTE_FEC_NUM_VFS; ++vf_id) { 2561 total_ul_q_id += conf->vf_ul_queues_number[vf_id]; 2562 total_dl_q_id += conf->vf_dl_queues_number[vf_id]; 2563 } 2564 total_q_id = total_dl_q_id + total_ul_q_id; 2565 /* 2566 * Check if total number of queues to configure does not exceed 2567 * FPGA capabilities (64 queues - 32 UL and 32 DL queues) 2568 */ 2569 if ((total_ul_q_id > FPGA_NUM_UL_QUEUES) || 2570 (total_dl_q_id > FPGA_NUM_DL_QUEUES) || 2571 (total_q_id > FPGA_TOTAL_NUM_QUEUES)) { 2572 rte_bbdev_log(ERR, 2573 "FPGA Configuration failed. Too many queues to configure: UL_Q %u, DL_Q %u, FPGA_Q %u", 2574 total_ul_q_id, total_dl_q_id, 2575 FPGA_TOTAL_NUM_QUEUES); 2576 return -EINVAL; 2577 } 2578 total_ul_q_id = 0; 2579 for (vf_id = 0; vf_id < FPGA_LTE_FEC_NUM_VFS; ++vf_id) { 2580 for (q_id = 0; q_id < conf->vf_ul_queues_number[vf_id]; 2581 ++q_id, ++total_ul_q_id) { 2582 address = (total_ul_q_id << 2) + 2583 FPGA_LTE_FEC_QUEUE_MAP; 2584 payload_32 = ((0x80 + vf_id) << 16) | 0x1; 2585 fpga_reg_write_32(d->mmio_base, address, 2586 payload_32); 2587 } 2588 } 2589 total_dl_q_id = 0; 2590 for (vf_id = 0; vf_id < FPGA_LTE_FEC_NUM_VFS; ++vf_id) { 2591 for (q_id = 0; q_id < conf->vf_dl_queues_number[vf_id]; 2592 ++q_id, ++total_dl_q_id) { 2593 address = ((total_dl_q_id + FPGA_NUM_UL_QUEUES) 2594 << 2) + FPGA_LTE_FEC_QUEUE_MAP; 2595 payload_32 = ((0x80 + vf_id) << 16) | 0x1; 2596 fpga_reg_write_32(d->mmio_base, address, 2597 payload_32); 2598 } 2599 } 2600 } 2601 2602 /* Setting Load Balance Factor */ 2603 payload_16 = (conf->dl_load_balance << 8) | (conf->ul_load_balance); 2604 address = FPGA_LTE_FEC_LOAD_BALANCE_FACTOR; 2605 fpga_reg_write_16(d->mmio_base, address, payload_16); 2606 2607 /* Setting length of ring descriptor entry */ 2608 payload_16 = FPGA_RING_DESC_ENTRY_LENGTH; 2609 address = FPGA_LTE_FEC_RING_DESC_LEN; 2610 fpga_reg_write_16(d->mmio_base, address, payload_16); 2611 2612 /* Setting FLR timeout value */ 2613 payload_16 = conf->flr_time_out; 2614 address = FPGA_LTE_FEC_FLR_TIME_OUT; 2615 fpga_reg_write_16(d->mmio_base, address, payload_16); 2616 2617 /* Queue PF/VF mapping table is ready */ 2618 payload_8 = 0x1; 2619 address = FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE; 2620 fpga_reg_write_8(d->mmio_base, address, payload_8); 2621 2622 rte_bbdev_log_debug("PF FPGA LTE FEC configuration complete for %s", 2623 dev_name); 2624 2625 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2626 print_static_reg_debug_info(d->mmio_base); 2627 #endif 2628 return 0; 2629 } 2630 2631 /* FPGA LTE FEC PCI PF address map */ 2632 static struct rte_pci_id pci_id_fpga_lte_fec_pf_map[] = { 2633 { 2634 RTE_PCI_DEVICE(FPGA_LTE_FEC_VENDOR_ID, 2635 FPGA_LTE_FEC_PF_DEVICE_ID) 2636 }, 2637 {.device_id = 0}, 2638 }; 2639 2640 static struct rte_pci_driver fpga_lte_fec_pci_pf_driver = { 2641 .probe = fpga_lte_fec_probe, 2642 .remove = fpga_lte_fec_remove, 2643 .id_table = pci_id_fpga_lte_fec_pf_map, 2644 .drv_flags = RTE_PCI_DRV_NEED_MAPPING 2645 }; 2646 2647 /* FPGA LTE FEC PCI VF address map */ 2648 static struct rte_pci_id pci_id_fpga_lte_fec_vf_map[] = { 2649 { 2650 RTE_PCI_DEVICE(FPGA_LTE_FEC_VENDOR_ID, 2651 FPGA_LTE_FEC_VF_DEVICE_ID) 2652 }, 2653 {.device_id = 0}, 2654 }; 2655 2656 static struct rte_pci_driver fpga_lte_fec_pci_vf_driver = { 2657 .probe = fpga_lte_fec_probe, 2658 .remove = fpga_lte_fec_remove, 2659 .id_table = pci_id_fpga_lte_fec_vf_map, 2660 .drv_flags = RTE_PCI_DRV_NEED_MAPPING 2661 }; 2662 2663 2664 RTE_PMD_REGISTER_PCI(FPGA_LTE_FEC_PF_DRIVER_NAME, fpga_lte_fec_pci_pf_driver); 2665 RTE_PMD_REGISTER_PCI_TABLE(FPGA_LTE_FEC_PF_DRIVER_NAME, 2666 pci_id_fpga_lte_fec_pf_map); 2667 RTE_PMD_REGISTER_PCI(FPGA_LTE_FEC_VF_DRIVER_NAME, fpga_lte_fec_pci_vf_driver); 2668 RTE_PMD_REGISTER_PCI_TABLE(FPGA_LTE_FEC_VF_DRIVER_NAME, 2669 pci_id_fpga_lte_fec_vf_map); 2670