1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #include <unistd.h> 6 7 #include <rte_common.h> 8 #include <rte_log.h> 9 #include <rte_dev.h> 10 #include <rte_malloc.h> 11 #include <rte_mempool.h> 12 #include <rte_errno.h> 13 #include <rte_pci.h> 14 #include <rte_bus_pci.h> 15 #include <rte_byteorder.h> 16 #ifdef RTE_BBDEV_OFFLOAD_COST 17 #include <rte_cycles.h> 18 #endif 19 20 #include <rte_bbdev.h> 21 #include <rte_bbdev_pmd.h> 22 23 #include "fpga_lte_fec.h" 24 25 /* Turbo SW PMD logging ID */ 26 static int fpga_lte_fec_logtype; 27 28 /* Helper macro for logging */ 29 #define rte_bbdev_log(level, fmt, ...) \ 30 rte_log(RTE_LOG_ ## level, fpga_lte_fec_logtype, fmt "\n", \ 31 ##__VA_ARGS__) 32 33 #ifdef RTE_LIBRTE_BBDEV_DEBUG 34 #define rte_bbdev_log_debug(fmt, ...) \ 35 rte_bbdev_log(DEBUG, "fpga_lte_fec: " fmt, \ 36 ##__VA_ARGS__) 37 #else 38 #define rte_bbdev_log_debug(fmt, ...) 39 #endif 40 41 /* FPGA LTE FEC driver names */ 42 #define FPGA_LTE_FEC_PF_DRIVER_NAME intel_fpga_lte_fec_pf 43 #define FPGA_LTE_FEC_VF_DRIVER_NAME intel_fpga_lte_fec_vf 44 45 /* FPGA LTE FEC PCI vendor & device IDs */ 46 #define FPGA_LTE_FEC_VENDOR_ID (0x1172) 47 #define FPGA_LTE_FEC_PF_DEVICE_ID (0x5052) 48 #define FPGA_LTE_FEC_VF_DEVICE_ID (0x5050) 49 50 /* Align DMA descriptors to 256 bytes - cache-aligned */ 51 #define FPGA_RING_DESC_ENTRY_LENGTH (8) 52 /* Ring size is in 256 bits (32 bytes) units */ 53 #define FPGA_RING_DESC_LEN_UNIT_BYTES (32) 54 /* Maximum size of queue */ 55 #define FPGA_RING_MAX_SIZE (1024) 56 #define FPGA_FLR_TIMEOUT_UNIT (16.384) 57 58 #define FPGA_NUM_UL_QUEUES (32) 59 #define FPGA_NUM_DL_QUEUES (32) 60 #define FPGA_TOTAL_NUM_QUEUES (FPGA_NUM_UL_QUEUES + FPGA_NUM_DL_QUEUES) 61 #define FPGA_NUM_INTR_VEC (FPGA_TOTAL_NUM_QUEUES - RTE_INTR_VEC_RXTX_OFFSET) 62 63 #define FPGA_INVALID_HW_QUEUE_ID (0xFFFFFFFF) 64 65 #define FPGA_QUEUE_FLUSH_TIMEOUT_US (1000) 66 #define FPGA_TIMEOUT_CHECK_INTERVAL (5) 67 68 /* FPGA LTE FEC Register mapping on BAR0 */ 69 enum { 70 FPGA_LTE_FEC_VERSION_ID = 0x00000000, /* len: 4B */ 71 FPGA_LTE_FEC_CONFIGURATION = 0x00000004, /* len: 2B */ 72 FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE = 0x00000008, /* len: 1B */ 73 FPGA_LTE_FEC_LOAD_BALANCE_FACTOR = 0x0000000a, /* len: 2B */ 74 FPGA_LTE_FEC_RING_DESC_LEN = 0x0000000c, /* len: 2B */ 75 FPGA_LTE_FEC_FLR_TIME_OUT = 0x0000000e, /* len: 2B */ 76 FPGA_LTE_FEC_VFQ_FLUSH_STATUS_LW = 0x00000018, /* len: 4B */ 77 FPGA_LTE_FEC_VFQ_FLUSH_STATUS_HI = 0x0000001c, /* len: 4B */ 78 FPGA_LTE_FEC_VF0_DEBUG = 0x00000020, /* len: 4B */ 79 FPGA_LTE_FEC_VF1_DEBUG = 0x00000024, /* len: 4B */ 80 FPGA_LTE_FEC_VF2_DEBUG = 0x00000028, /* len: 4B */ 81 FPGA_LTE_FEC_VF3_DEBUG = 0x0000002c, /* len: 4B */ 82 FPGA_LTE_FEC_VF4_DEBUG = 0x00000030, /* len: 4B */ 83 FPGA_LTE_FEC_VF5_DEBUG = 0x00000034, /* len: 4B */ 84 FPGA_LTE_FEC_VF6_DEBUG = 0x00000038, /* len: 4B */ 85 FPGA_LTE_FEC_VF7_DEBUG = 0x0000003c, /* len: 4B */ 86 FPGA_LTE_FEC_QUEUE_MAP = 0x00000040, /* len: 256B */ 87 FPGA_LTE_FEC_RING_CTRL_REGS = 0x00000200 /* len: 2048B */ 88 }; 89 90 /* FPGA LTE FEC Ring Control Registers */ 91 enum { 92 FPGA_LTE_FEC_RING_HEAD_ADDR = 0x00000008, 93 FPGA_LTE_FEC_RING_SIZE = 0x00000010, 94 FPGA_LTE_FEC_RING_MISC = 0x00000014, 95 FPGA_LTE_FEC_RING_ENABLE = 0x00000015, 96 FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN = 0x00000016, 97 FPGA_LTE_FEC_RING_SHADOW_TAIL = 0x00000018, 98 FPGA_LTE_FEC_RING_HEAD_POINT = 0x0000001C 99 }; 100 101 /* FPGA LTE FEC DESCRIPTOR ERROR */ 102 enum { 103 DESC_ERR_NO_ERR = 0x0, 104 DESC_ERR_K_OUT_OF_RANGE = 0x1, 105 DESC_ERR_K_NOT_NORMAL = 0x2, 106 DESC_ERR_KPAI_NOT_NORMAL = 0x3, 107 DESC_ERR_DESC_OFFSET_ERR = 0x4, 108 DESC_ERR_DESC_READ_FAIL = 0x8, 109 DESC_ERR_DESC_READ_TIMEOUT = 0x9, 110 DESC_ERR_DESC_READ_TLP_POISONED = 0xA, 111 DESC_ERR_CB_READ_FAIL = 0xC, 112 DESC_ERR_CB_READ_TIMEOUT = 0xD, 113 DESC_ERR_CB_READ_TLP_POISONED = 0xE 114 }; 115 116 /* FPGA LTE FEC DMA Encoding Request Descriptor */ 117 struct __attribute__((__packed__)) fpga_dma_enc_desc { 118 uint32_t done:1, 119 rsrvd0:11, 120 error:4, 121 rsrvd1:16; 122 uint32_t ncb:16, 123 rsrvd2:14, 124 rv:2; 125 uint32_t bypass_rm:1, 126 irq_en:1, 127 crc_en:1, 128 rsrvd3:13, 129 offset:10, 130 rsrvd4:6; 131 uint16_t e; 132 uint16_t k; 133 uint32_t out_addr_lw; 134 uint32_t out_addr_hi; 135 uint32_t in_addr_lw; 136 uint32_t in_addr_hi; 137 138 union { 139 struct { 140 /* Virtual addresses used to retrieve SW context info */ 141 void *op_addr; 142 /* Stores information about total number of Code Blocks 143 * in currently processed Transport Block 144 */ 145 uint64_t cbs_in_op; 146 }; 147 148 uint8_t sw_ctxt[FPGA_RING_DESC_LEN_UNIT_BYTES * 149 (FPGA_RING_DESC_ENTRY_LENGTH - 1)]; 150 }; 151 }; 152 153 /* FPGA LTE FEC DMA Decoding Request Descriptor */ 154 struct __attribute__((__packed__)) fpga_dma_dec_desc { 155 uint32_t done:1, 156 iter:5, 157 rsrvd0:2, 158 crc_pass:1, 159 rsrvd1:3, 160 error:4, 161 crc_type:1, 162 rsrvd2:7, 163 max_iter:5, 164 rsrvd3:3; 165 uint32_t rsrvd4; 166 uint32_t bypass_rm:1, 167 irq_en:1, 168 drop_crc:1, 169 rsrvd5:13, 170 offset:10, 171 rsrvd6:6; 172 uint16_t k; 173 uint16_t in_len; 174 uint32_t out_addr_lw; 175 uint32_t out_addr_hi; 176 uint32_t in_addr_lw; 177 uint32_t in_addr_hi; 178 179 union { 180 struct { 181 /* Virtual addresses used to retrieve SW context info */ 182 void *op_addr; 183 /* Stores information about total number of Code Blocks 184 * in currently processed Transport Block 185 */ 186 uint8_t cbs_in_op; 187 }; 188 189 uint32_t sw_ctxt[8 * (FPGA_RING_DESC_ENTRY_LENGTH - 1)]; 190 }; 191 }; 192 193 /* FPGA LTE DMA Descriptor */ 194 union fpga_dma_desc { 195 struct fpga_dma_enc_desc enc_req; 196 struct fpga_dma_dec_desc dec_req; 197 }; 198 199 /* FPGA LTE FEC Ring Control Register */ 200 struct __attribute__((__packed__)) fpga_ring_ctrl_reg { 201 uint64_t ring_base_addr; 202 uint64_t ring_head_addr; 203 uint16_t ring_size:11; 204 uint16_t rsrvd0; 205 union { /* Miscellaneous register */ 206 uint8_t misc; 207 uint8_t max_ul_dec:5, 208 max_ul_dec_en:1, 209 rsrvd1:2; 210 }; 211 uint8_t enable; 212 uint8_t flush_queue_en; 213 uint8_t rsrvd2; 214 uint16_t shadow_tail; 215 uint16_t rsrvd3; 216 uint16_t head_point; 217 uint16_t rsrvd4; 218 219 }; 220 221 /* Private data structure for each FPGA FEC device */ 222 struct fpga_lte_fec_device { 223 /** Base address of MMIO registers (BAR0) */ 224 void *mmio_base; 225 /** Base address of memory for sw rings */ 226 void *sw_rings; 227 /** Physical address of sw_rings */ 228 rte_iova_t sw_rings_phys; 229 /** Number of bytes available for each queue in device. */ 230 uint32_t sw_ring_size; 231 /** Max number of entries available for each queue in device */ 232 uint32_t sw_ring_max_depth; 233 /** Base address of response tail pointer buffer */ 234 uint32_t *tail_ptrs; 235 /** Physical address of tail pointers */ 236 rte_iova_t tail_ptr_phys; 237 /** Queues flush completion flag */ 238 uint64_t *flush_queue_status; 239 /* Bitmap capturing which Queues are bound to the PF/VF */ 240 uint64_t q_bound_bit_map; 241 /* Bitmap capturing which Queues have already been assigned */ 242 uint64_t q_assigned_bit_map; 243 /** True if this is a PF FPGA FEC device */ 244 bool pf_device; 245 }; 246 247 /* Structure associated with each queue. */ 248 struct __rte_cache_aligned fpga_queue { 249 struct fpga_ring_ctrl_reg ring_ctrl_reg; /* Ring Control Register */ 250 union fpga_dma_desc *ring_addr; /* Virtual address of software ring */ 251 uint64_t *ring_head_addr; /* Virtual address of completion_head */ 252 uint64_t shadow_completion_head; /* Shadow completion head value */ 253 uint16_t head_free_desc; /* Ring head */ 254 uint16_t tail; /* Ring tail */ 255 /* Mask used to wrap enqueued descriptors on the sw ring */ 256 uint32_t sw_ring_wrap_mask; 257 uint32_t irq_enable; /* Enable ops dequeue interrupts if set to 1 */ 258 uint8_t q_idx; /* Queue index */ 259 struct fpga_lte_fec_device *d; 260 /* MMIO register of shadow_tail used to enqueue descriptors */ 261 void *shadow_tail_addr; 262 }; 263 264 /* Write to 16 bit MMIO register address */ 265 static inline void 266 mmio_write_16(void *addr, uint16_t value) 267 { 268 *((volatile uint16_t *)(addr)) = rte_cpu_to_le_16(value); 269 } 270 271 /* Write to 32 bit MMIO register address */ 272 static inline void 273 mmio_write_32(void *addr, uint32_t value) 274 { 275 *((volatile uint32_t *)(addr)) = rte_cpu_to_le_32(value); 276 } 277 278 /* Write to 64 bit MMIO register address */ 279 static inline void 280 mmio_write_64(void *addr, uint64_t value) 281 { 282 *((volatile uint64_t *)(addr)) = rte_cpu_to_le_64(value); 283 } 284 285 /* Write a 8 bit register of a FPGA LTE FEC device */ 286 static inline void 287 fpga_reg_write_8(void *mmio_base, uint32_t offset, uint8_t payload) 288 { 289 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 290 *((volatile uint8_t *)(reg_addr)) = payload; 291 } 292 293 /* Write a 16 bit register of a FPGA LTE FEC device */ 294 static inline void 295 fpga_reg_write_16(void *mmio_base, uint32_t offset, uint16_t payload) 296 { 297 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 298 mmio_write_16(reg_addr, payload); 299 } 300 301 /* Write a 32 bit register of a FPGA LTE FEC device */ 302 static inline void 303 fpga_reg_write_32(void *mmio_base, uint32_t offset, uint32_t payload) 304 { 305 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 306 mmio_write_32(reg_addr, payload); 307 } 308 309 /* Write a 64 bit register of a FPGA LTE FEC device */ 310 static inline void 311 fpga_reg_write_64(void *mmio_base, uint32_t offset, uint64_t payload) 312 { 313 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 314 mmio_write_64(reg_addr, payload); 315 } 316 317 /* Write a ring control register of a FPGA LTE FEC device */ 318 static inline void 319 fpga_ring_reg_write(void *mmio_base, uint32_t offset, 320 struct fpga_ring_ctrl_reg payload) 321 { 322 fpga_reg_write_64(mmio_base, offset, payload.ring_base_addr); 323 fpga_reg_write_64(mmio_base, offset + FPGA_LTE_FEC_RING_HEAD_ADDR, 324 payload.ring_head_addr); 325 fpga_reg_write_16(mmio_base, offset + FPGA_LTE_FEC_RING_SIZE, 326 payload.ring_size); 327 fpga_reg_write_16(mmio_base, offset + FPGA_LTE_FEC_RING_HEAD_POINT, 328 payload.head_point); 329 fpga_reg_write_8(mmio_base, offset + FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN, 330 payload.flush_queue_en); 331 fpga_reg_write_16(mmio_base, offset + FPGA_LTE_FEC_RING_SHADOW_TAIL, 332 payload.shadow_tail); 333 fpga_reg_write_8(mmio_base, offset + FPGA_LTE_FEC_RING_MISC, 334 payload.misc); 335 fpga_reg_write_8(mmio_base, offset + FPGA_LTE_FEC_RING_ENABLE, 336 payload.enable); 337 } 338 339 /* Read a register of FPGA LTE FEC device */ 340 static uint32_t 341 fpga_reg_read_32(void *mmio_base, uint32_t offset) 342 { 343 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 344 uint32_t ret = *((volatile uint32_t *)(reg_addr)); 345 return rte_le_to_cpu_32(ret); 346 } 347 348 #ifdef RTE_LIBRTE_BBDEV_DEBUG 349 /* Read a register of FPGA LTE FEC device */ 350 static uint8_t 351 fpga_reg_read_8(void *mmio_base, uint32_t offset) 352 { 353 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 354 return *((volatile uint8_t *)(reg_addr)); 355 } 356 357 /* Read a register of FPGA LTE FEC device */ 358 static uint16_t 359 fpga_reg_read_16(void *mmio_base, uint32_t offset) 360 { 361 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 362 uint16_t ret = *((volatile uint16_t *)(reg_addr)); 363 return rte_le_to_cpu_16(ret); 364 } 365 366 /* Read a register of FPGA LTE FEC device */ 367 static uint64_t 368 fpga_reg_read_64(void *mmio_base, uint32_t offset) 369 { 370 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 371 uint64_t ret = *((volatile uint64_t *)(reg_addr)); 372 return rte_le_to_cpu_64(ret); 373 } 374 375 /* Read Ring Control Register of FPGA LTE FEC device */ 376 static inline void 377 print_ring_reg_debug_info(void *mmio_base, uint32_t offset) 378 { 379 rte_bbdev_log_debug( 380 "FPGA MMIO base address @ %p | Ring Control Register @ offset = 0x%08" 381 PRIx32, mmio_base, offset); 382 rte_bbdev_log_debug( 383 "RING_BASE_ADDR = 0x%016"PRIx64, 384 fpga_reg_read_64(mmio_base, offset)); 385 rte_bbdev_log_debug( 386 "RING_HEAD_ADDR = 0x%016"PRIx64, 387 fpga_reg_read_64(mmio_base, offset + 388 FPGA_LTE_FEC_RING_HEAD_ADDR)); 389 rte_bbdev_log_debug( 390 "RING_SIZE = 0x%04"PRIx16, 391 fpga_reg_read_16(mmio_base, offset + 392 FPGA_LTE_FEC_RING_SIZE)); 393 rte_bbdev_log_debug( 394 "RING_MISC = 0x%02"PRIx8, 395 fpga_reg_read_8(mmio_base, offset + 396 FPGA_LTE_FEC_RING_MISC)); 397 rte_bbdev_log_debug( 398 "RING_ENABLE = 0x%02"PRIx8, 399 fpga_reg_read_8(mmio_base, offset + 400 FPGA_LTE_FEC_RING_ENABLE)); 401 rte_bbdev_log_debug( 402 "RING_FLUSH_QUEUE_EN = 0x%02"PRIx8, 403 fpga_reg_read_8(mmio_base, offset + 404 FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN)); 405 rte_bbdev_log_debug( 406 "RING_SHADOW_TAIL = 0x%04"PRIx16, 407 fpga_reg_read_16(mmio_base, offset + 408 FPGA_LTE_FEC_RING_SHADOW_TAIL)); 409 rte_bbdev_log_debug( 410 "RING_HEAD_POINT = 0x%04"PRIx16, 411 fpga_reg_read_16(mmio_base, offset + 412 FPGA_LTE_FEC_RING_HEAD_POINT)); 413 } 414 415 /* Read Static Register of FPGA LTE FEC device */ 416 static inline void 417 print_static_reg_debug_info(void *mmio_base) 418 { 419 uint16_t config = fpga_reg_read_16(mmio_base, 420 FPGA_LTE_FEC_CONFIGURATION); 421 uint8_t qmap_done = fpga_reg_read_8(mmio_base, 422 FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE); 423 uint16_t lb_factor = fpga_reg_read_16(mmio_base, 424 FPGA_LTE_FEC_LOAD_BALANCE_FACTOR); 425 uint16_t ring_desc_len = fpga_reg_read_16(mmio_base, 426 FPGA_LTE_FEC_RING_DESC_LEN); 427 uint16_t flr_time_out = fpga_reg_read_16(mmio_base, 428 FPGA_LTE_FEC_FLR_TIME_OUT); 429 430 rte_bbdev_log_debug("UL.DL Weights = %u.%u", 431 ((uint8_t)config), ((uint8_t)(config >> 8))); 432 rte_bbdev_log_debug("UL.DL Load Balance = %u.%u", 433 ((uint8_t)lb_factor), ((uint8_t)(lb_factor >> 8))); 434 rte_bbdev_log_debug("Queue-PF/VF Mapping Table = %s", 435 (qmap_done > 0) ? "READY" : "NOT-READY"); 436 rte_bbdev_log_debug("Ring Descriptor Size = %u bytes", 437 ring_desc_len*FPGA_RING_DESC_LEN_UNIT_BYTES); 438 rte_bbdev_log_debug("FLR Timeout = %f usec", 439 (float)flr_time_out*FPGA_FLR_TIMEOUT_UNIT); 440 } 441 442 /* Print decode DMA Descriptor of FPGA LTE FEC device */ 443 static void 444 print_dma_dec_desc_debug_info(union fpga_dma_desc *desc) 445 { 446 rte_bbdev_log_debug("DMA response desc %p\n" 447 "\t-- done(%"PRIu32") | iter(%"PRIu32") | crc_pass(%"PRIu32")" 448 " | error (%"PRIu32") | crc_type(%"PRIu32")\n" 449 "\t-- max_iter(%"PRIu32") | bypass_rm(%"PRIu32") | " 450 "irq_en (%"PRIu32") | drop_crc(%"PRIu32") | offset(%"PRIu32")\n" 451 "\t-- k(%"PRIu32") | in_len (%"PRIu16") | op_add(%p)\n" 452 "\t-- cbs_in_op(%"PRIu32") | in_add (0x%08"PRIx32"%08"PRIx32") | " 453 "out_add (0x%08"PRIx32"%08"PRIx32")", 454 desc, 455 (uint32_t)desc->dec_req.done, 456 (uint32_t)desc->dec_req.iter, 457 (uint32_t)desc->dec_req.crc_pass, 458 (uint32_t)desc->dec_req.error, 459 (uint32_t)desc->dec_req.crc_type, 460 (uint32_t)desc->dec_req.max_iter, 461 (uint32_t)desc->dec_req.bypass_rm, 462 (uint32_t)desc->dec_req.irq_en, 463 (uint32_t)desc->dec_req.drop_crc, 464 (uint32_t)desc->dec_req.offset, 465 (uint32_t)desc->dec_req.k, 466 (uint16_t)desc->dec_req.in_len, 467 desc->dec_req.op_addr, 468 (uint32_t)desc->dec_req.cbs_in_op, 469 (uint32_t)desc->dec_req.in_addr_hi, 470 (uint32_t)desc->dec_req.in_addr_lw, 471 (uint32_t)desc->dec_req.out_addr_hi, 472 (uint32_t)desc->dec_req.out_addr_lw); 473 } 474 #endif 475 476 static int 477 fpga_setup_queues(struct rte_bbdev *dev, uint16_t num_queues, int socket_id) 478 { 479 /* Number of queues bound to a PF/VF */ 480 uint32_t hw_q_num = 0; 481 uint32_t ring_size, payload, address, q_id, offset; 482 rte_iova_t phys_addr; 483 struct fpga_ring_ctrl_reg ring_reg; 484 struct fpga_lte_fec_device *fpga_dev = dev->data->dev_private; 485 486 address = FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE; 487 if (!(fpga_reg_read_32(fpga_dev->mmio_base, address) & 0x1)) { 488 rte_bbdev_log(ERR, 489 "Queue-PF/VF mapping is not set! Was PF configured for device (%s) ?", 490 dev->data->name); 491 return -EPERM; 492 } 493 494 /* Clear queue registers structure */ 495 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg)); 496 497 /* Scan queue map. 498 * If a queue is valid and mapped to a calling PF/VF the read value is 499 * replaced with a queue ID and if it's not then 500 * FPGA_INVALID_HW_QUEUE_ID is returned. 501 */ 502 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 503 uint32_t hw_q_id = fpga_reg_read_32(fpga_dev->mmio_base, 504 FPGA_LTE_FEC_QUEUE_MAP + (q_id << 2)); 505 506 rte_bbdev_log_debug("%s: queue ID: %u, registry queue ID: %u", 507 dev->device->name, q_id, hw_q_id); 508 509 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) { 510 fpga_dev->q_bound_bit_map |= (1ULL << q_id); 511 /* Clear queue register of found queue */ 512 offset = FPGA_LTE_FEC_RING_CTRL_REGS + 513 (sizeof(struct fpga_ring_ctrl_reg) * q_id); 514 fpga_ring_reg_write(fpga_dev->mmio_base, 515 offset, ring_reg); 516 ++hw_q_num; 517 } 518 } 519 if (hw_q_num == 0) { 520 rte_bbdev_log(ERR, 521 "No HW queues assigned to this device. Probably this is a VF configured for PF mode. Check device configuration!"); 522 return -ENODEV; 523 } 524 525 if (num_queues > hw_q_num) { 526 rte_bbdev_log(ERR, 527 "Not enough queues for device %s! Requested: %u, available: %u", 528 dev->device->name, num_queues, hw_q_num); 529 return -EINVAL; 530 } 531 532 ring_size = FPGA_RING_MAX_SIZE * sizeof(struct fpga_dma_dec_desc); 533 534 /* Enforce 32 byte alignment */ 535 RTE_BUILD_BUG_ON((RTE_CACHE_LINE_SIZE % 32) != 0); 536 537 /* Allocate memory for SW descriptor rings */ 538 fpga_dev->sw_rings = rte_zmalloc_socket(dev->device->driver->name, 539 num_queues * ring_size, RTE_CACHE_LINE_SIZE, 540 socket_id); 541 if (fpga_dev->sw_rings == NULL) { 542 rte_bbdev_log(ERR, 543 "Failed to allocate memory for %s:%u sw_rings", 544 dev->device->driver->name, dev->data->dev_id); 545 return -ENOMEM; 546 } 547 548 fpga_dev->sw_rings_phys = rte_malloc_virt2iova(fpga_dev->sw_rings); 549 fpga_dev->sw_ring_size = ring_size; 550 fpga_dev->sw_ring_max_depth = FPGA_RING_MAX_SIZE; 551 552 /* Allocate memory for ring flush status */ 553 fpga_dev->flush_queue_status = rte_zmalloc_socket(NULL, 554 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, socket_id); 555 if (fpga_dev->flush_queue_status == NULL) { 556 rte_bbdev_log(ERR, 557 "Failed to allocate memory for %s:%u flush_queue_status", 558 dev->device->driver->name, dev->data->dev_id); 559 return -ENOMEM; 560 } 561 562 /* Set the flush status address registers */ 563 phys_addr = rte_malloc_virt2iova(fpga_dev->flush_queue_status); 564 565 address = FPGA_LTE_FEC_VFQ_FLUSH_STATUS_LW; 566 payload = (uint32_t)(phys_addr); 567 fpga_reg_write_32(fpga_dev->mmio_base, address, payload); 568 569 address = FPGA_LTE_FEC_VFQ_FLUSH_STATUS_HI; 570 payload = (uint32_t)(phys_addr >> 32); 571 fpga_reg_write_32(fpga_dev->mmio_base, address, payload); 572 573 return 0; 574 } 575 576 static int 577 fpga_dev_close(struct rte_bbdev *dev) 578 { 579 struct fpga_lte_fec_device *fpga_dev = dev->data->dev_private; 580 581 rte_free(fpga_dev->sw_rings); 582 rte_free(fpga_dev->flush_queue_status); 583 584 return 0; 585 } 586 587 static void 588 fpga_dev_info_get(struct rte_bbdev *dev, 589 struct rte_bbdev_driver_info *dev_info) 590 { 591 struct fpga_lte_fec_device *d = dev->data->dev_private; 592 uint32_t q_id = 0; 593 594 /* TODO RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN and numbers of buffers are set 595 * to temporary values as they are required by test application while 596 * validation phase. 597 */ 598 static const struct rte_bbdev_op_cap bbdev_capabilities[] = { 599 { 600 .type = RTE_BBDEV_OP_TURBO_DEC, 601 .cap.turbo_dec = { 602 .capability_flags = 603 RTE_BBDEV_TURBO_CRC_TYPE_24B | 604 RTE_BBDEV_TURBO_SUBBLOCK_DEINTERLEAVE | 605 RTE_BBDEV_TURBO_DEC_INTERRUPTS | 606 RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN | 607 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP, 608 .max_llr_modulus = INT8_MAX, 609 .num_buffers_src = 610 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS, 611 .num_buffers_hard_out = 612 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS, 613 .num_buffers_soft_out = 0 614 } 615 }, 616 { 617 .type = RTE_BBDEV_OP_TURBO_ENC, 618 .cap.turbo_enc = { 619 .capability_flags = 620 RTE_BBDEV_TURBO_CRC_24B_ATTACH | 621 RTE_BBDEV_TURBO_RATE_MATCH | 622 RTE_BBDEV_TURBO_ENC_INTERRUPTS, 623 .num_buffers_src = 624 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS, 625 .num_buffers_dst = 626 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS 627 } 628 }, 629 RTE_BBDEV_END_OF_CAPABILITIES_LIST() 630 }; 631 632 static struct rte_bbdev_queue_conf default_queue_conf; 633 default_queue_conf.socket = dev->data->socket_id; 634 default_queue_conf.queue_size = FPGA_RING_MAX_SIZE; 635 636 637 dev_info->driver_name = dev->device->driver->name; 638 dev_info->queue_size_lim = FPGA_RING_MAX_SIZE; 639 dev_info->hardware_accelerated = true; 640 dev_info->min_alignment = 64; 641 dev_info->default_queue_conf = default_queue_conf; 642 dev_info->capabilities = bbdev_capabilities; 643 dev_info->cpu_flag_reqs = NULL; 644 645 /* Calculates number of queues assigned to device */ 646 dev_info->max_num_queues = 0; 647 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 648 uint32_t hw_q_id = fpga_reg_read_32(d->mmio_base, 649 FPGA_LTE_FEC_QUEUE_MAP + (q_id << 2)); 650 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) 651 dev_info->max_num_queues++; 652 } 653 } 654 655 /** 656 * Find index of queue bound to current PF/VF which is unassigned. Return -1 657 * when there is no available queue 658 */ 659 static int 660 fpga_find_free_queue_idx(struct rte_bbdev *dev, 661 const struct rte_bbdev_queue_conf *conf) 662 { 663 struct fpga_lte_fec_device *d = dev->data->dev_private; 664 uint64_t q_idx; 665 uint8_t i = 0; 666 uint8_t range = FPGA_TOTAL_NUM_QUEUES >> 1; 667 668 if (conf->op_type == RTE_BBDEV_OP_TURBO_ENC) { 669 i = FPGA_NUM_DL_QUEUES; 670 range = FPGA_TOTAL_NUM_QUEUES; 671 } 672 673 for (; i < range; ++i) { 674 q_idx = 1ULL << i; 675 /* Check if index of queue is bound to current PF/VF */ 676 if (d->q_bound_bit_map & q_idx) 677 /* Check if found queue was not already assigned */ 678 if (!(d->q_assigned_bit_map & q_idx)) { 679 d->q_assigned_bit_map |= q_idx; 680 return i; 681 } 682 } 683 684 rte_bbdev_log(INFO, "Failed to find free queue on %s", dev->data->name); 685 686 return -1; 687 } 688 689 static int 690 fpga_queue_setup(struct rte_bbdev *dev, uint16_t queue_id, 691 const struct rte_bbdev_queue_conf *conf) 692 { 693 uint32_t address, ring_offset; 694 struct fpga_lte_fec_device *d = dev->data->dev_private; 695 struct fpga_queue *q; 696 int8_t q_idx; 697 698 /* Check if there is a free queue to assign */ 699 q_idx = fpga_find_free_queue_idx(dev, conf); 700 if (q_idx == -1) 701 return -1; 702 703 /* Allocate the queue data structure. */ 704 q = rte_zmalloc_socket(dev->device->driver->name, sizeof(*q), 705 RTE_CACHE_LINE_SIZE, conf->socket); 706 if (q == NULL) { 707 /* Mark queue as un-assigned */ 708 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 709 rte_bbdev_log(ERR, "Failed to allocate queue memory"); 710 return -ENOMEM; 711 } 712 713 q->d = d; 714 q->q_idx = q_idx; 715 716 /* Set ring_base_addr */ 717 q->ring_addr = RTE_PTR_ADD(d->sw_rings, (d->sw_ring_size * queue_id)); 718 q->ring_ctrl_reg.ring_base_addr = d->sw_rings_phys + 719 (d->sw_ring_size * queue_id); 720 721 /* Allocate memory for Completion Head variable*/ 722 q->ring_head_addr = rte_zmalloc_socket(dev->device->driver->name, 723 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, conf->socket); 724 if (q->ring_head_addr == NULL) { 725 /* Mark queue as un-assigned */ 726 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 727 rte_free(q); 728 rte_bbdev_log(ERR, 729 "Failed to allocate memory for %s:%u completion_head", 730 dev->device->driver->name, dev->data->dev_id); 731 return -ENOMEM; 732 } 733 /* Set ring_head_addr */ 734 q->ring_ctrl_reg.ring_head_addr = 735 rte_malloc_virt2iova(q->ring_head_addr); 736 737 /* Clear shadow_completion_head */ 738 q->shadow_completion_head = 0; 739 740 /* Set ring_size */ 741 if (conf->queue_size > FPGA_RING_MAX_SIZE) { 742 /* Mark queue as un-assigned */ 743 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 744 rte_free(q->ring_head_addr); 745 rte_free(q); 746 rte_bbdev_log(ERR, 747 "Size of queue is too big %d (MAX: %d ) for %s:%u", 748 conf->queue_size, FPGA_RING_MAX_SIZE, 749 dev->device->driver->name, dev->data->dev_id); 750 return -EINVAL; 751 } 752 q->ring_ctrl_reg.ring_size = conf->queue_size; 753 754 /* Set Miscellaneous FPGA register*/ 755 /* Max iteration number for TTI mitigation - todo */ 756 q->ring_ctrl_reg.max_ul_dec = 0; 757 /* Enable max iteration number for TTI - todo */ 758 q->ring_ctrl_reg.max_ul_dec_en = 0; 759 760 /* Enable the ring */ 761 q->ring_ctrl_reg.enable = 1; 762 763 /* Set FPGA head_point and tail registers */ 764 q->ring_ctrl_reg.head_point = q->tail = 0; 765 766 /* Set FPGA shadow_tail register */ 767 q->ring_ctrl_reg.shadow_tail = q->tail; 768 769 /* Calculates the ring offset for found queue */ 770 ring_offset = FPGA_LTE_FEC_RING_CTRL_REGS + 771 (sizeof(struct fpga_ring_ctrl_reg) * q_idx); 772 773 /* Set FPGA Ring Control Registers */ 774 fpga_ring_reg_write(d->mmio_base, ring_offset, q->ring_ctrl_reg); 775 776 /* Store MMIO register of shadow_tail */ 777 address = ring_offset + FPGA_LTE_FEC_RING_SHADOW_TAIL; 778 q->shadow_tail_addr = RTE_PTR_ADD(d->mmio_base, address); 779 780 q->head_free_desc = q->tail; 781 782 /* Set wrap mask */ 783 q->sw_ring_wrap_mask = conf->queue_size - 1; 784 785 rte_bbdev_log_debug("Setup dev%u q%u: queue_idx=%u", 786 dev->data->dev_id, queue_id, q->q_idx); 787 788 dev->data->queues[queue_id].queue_private = q; 789 790 rte_bbdev_log_debug("BBDEV queue[%d] set up for FPGA queue[%d]", 791 queue_id, q_idx); 792 793 #ifdef RTE_LIBRTE_BBDEV_DEBUG 794 /* Read FPGA Ring Control Registers after configuration*/ 795 print_ring_reg_debug_info(d->mmio_base, ring_offset); 796 #endif 797 return 0; 798 } 799 800 static int 801 fpga_queue_release(struct rte_bbdev *dev, uint16_t queue_id) 802 { 803 struct fpga_lte_fec_device *d = dev->data->dev_private; 804 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 805 struct fpga_ring_ctrl_reg ring_reg; 806 uint32_t offset; 807 808 rte_bbdev_log_debug("FPGA Queue[%d] released", queue_id); 809 810 if (q != NULL) { 811 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg)); 812 offset = FPGA_LTE_FEC_RING_CTRL_REGS + 813 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 814 /* Disable queue */ 815 fpga_reg_write_8(d->mmio_base, 816 offset + FPGA_LTE_FEC_RING_ENABLE, 0x00); 817 /* Clear queue registers */ 818 fpga_ring_reg_write(d->mmio_base, offset, ring_reg); 819 820 /* Mark the Queue as un-assigned */ 821 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q->q_idx)); 822 rte_free(q->ring_head_addr); 823 rte_free(q); 824 dev->data->queues[queue_id].queue_private = NULL; 825 } 826 827 return 0; 828 } 829 830 /* Function starts a device queue. */ 831 static int 832 fpga_queue_start(struct rte_bbdev *dev, uint16_t queue_id) 833 { 834 struct fpga_lte_fec_device *d = dev->data->dev_private; 835 #ifdef RTE_LIBRTE_BBDEV_DEBUG 836 if (d == NULL) { 837 rte_bbdev_log(ERR, "Invalid device pointer"); 838 return -1; 839 } 840 #endif 841 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 842 uint32_t offset = FPGA_LTE_FEC_RING_CTRL_REGS + 843 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 844 uint8_t enable = 0x01; 845 uint16_t zero = 0x0000; 846 847 /* Clear queue head and tail variables */ 848 q->tail = q->head_free_desc = 0; 849 850 /* Clear FPGA head_point and tail registers */ 851 fpga_reg_write_16(d->mmio_base, offset + FPGA_LTE_FEC_RING_HEAD_POINT, 852 zero); 853 fpga_reg_write_16(d->mmio_base, offset + FPGA_LTE_FEC_RING_SHADOW_TAIL, 854 zero); 855 856 /* Enable queue */ 857 fpga_reg_write_8(d->mmio_base, offset + FPGA_LTE_FEC_RING_ENABLE, 858 enable); 859 860 rte_bbdev_log_debug("FPGA Queue[%d] started", queue_id); 861 return 0; 862 } 863 864 /* Function stops a device queue. */ 865 static int 866 fpga_queue_stop(struct rte_bbdev *dev, uint16_t queue_id) 867 { 868 struct fpga_lte_fec_device *d = dev->data->dev_private; 869 #ifdef RTE_LIBRTE_BBDEV_DEBUG 870 if (d == NULL) { 871 rte_bbdev_log(ERR, "Invalid device pointer"); 872 return -1; 873 } 874 #endif 875 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 876 uint32_t offset = FPGA_LTE_FEC_RING_CTRL_REGS + 877 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 878 uint8_t payload = 0x01; 879 uint8_t counter = 0; 880 uint8_t timeout = FPGA_QUEUE_FLUSH_TIMEOUT_US / 881 FPGA_TIMEOUT_CHECK_INTERVAL; 882 883 /* Set flush_queue_en bit to trigger queue flushing */ 884 fpga_reg_write_8(d->mmio_base, 885 offset + FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN, payload); 886 887 /** Check if queue flush is completed. 888 * FPGA will update the completion flag after queue flushing is 889 * completed. If completion flag is not updated within 1ms it is 890 * considered as a failure. 891 */ 892 while (!(*((uint8_t *)d->flush_queue_status + q->q_idx) & payload)) { 893 if (counter > timeout) { 894 rte_bbdev_log(ERR, "FPGA Queue Flush failed for queue %d", 895 queue_id); 896 return -1; 897 } 898 usleep(FPGA_TIMEOUT_CHECK_INTERVAL); 899 counter++; 900 } 901 902 /* Disable queue */ 903 payload = 0x00; 904 fpga_reg_write_8(d->mmio_base, offset + FPGA_LTE_FEC_RING_ENABLE, 905 payload); 906 907 rte_bbdev_log_debug("FPGA Queue[%d] stopped", queue_id); 908 return 0; 909 } 910 911 static inline uint16_t 912 get_queue_id(struct rte_bbdev_data *data, uint8_t q_idx) 913 { 914 uint16_t queue_id; 915 916 for (queue_id = 0; queue_id < data->num_queues; ++queue_id) { 917 struct fpga_queue *q = data->queues[queue_id].queue_private; 918 if (q != NULL && q->q_idx == q_idx) 919 return queue_id; 920 } 921 922 return -1; 923 } 924 925 /* Interrupt handler triggered by FPGA dev for handling specific interrupt */ 926 static void 927 fpga_dev_interrupt_handler(void *cb_arg) 928 { 929 struct rte_bbdev *dev = cb_arg; 930 struct fpga_lte_fec_device *fpga_dev = dev->data->dev_private; 931 struct fpga_queue *q; 932 uint64_t ring_head; 933 uint64_t q_idx; 934 uint16_t queue_id; 935 uint8_t i; 936 937 /* Scan queue assigned to this device */ 938 for (i = 0; i < FPGA_TOTAL_NUM_QUEUES; ++i) { 939 q_idx = 1ULL << i; 940 if (fpga_dev->q_bound_bit_map & q_idx) { 941 queue_id = get_queue_id(dev->data, i); 942 if (queue_id == (uint16_t) -1) 943 continue; 944 945 /* Check if completion head was changed */ 946 q = dev->data->queues[queue_id].queue_private; 947 ring_head = *q->ring_head_addr; 948 if (q->shadow_completion_head != ring_head && 949 q->irq_enable == 1) { 950 q->shadow_completion_head = ring_head; 951 rte_bbdev_pmd_callback_process( 952 dev, 953 RTE_BBDEV_EVENT_DEQUEUE, 954 &queue_id); 955 } 956 } 957 } 958 } 959 960 static int 961 fpga_queue_intr_enable(struct rte_bbdev *dev, uint16_t queue_id) 962 { 963 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 964 965 if (!rte_intr_cap_multiple(dev->intr_handle)) 966 return -ENOTSUP; 967 968 q->irq_enable = 1; 969 970 return 0; 971 } 972 973 static int 974 fpga_queue_intr_disable(struct rte_bbdev *dev, uint16_t queue_id) 975 { 976 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 977 q->irq_enable = 0; 978 979 return 0; 980 } 981 982 static int 983 fpga_intr_enable(struct rte_bbdev *dev) 984 { 985 int ret; 986 uint8_t i; 987 988 if (!rte_intr_cap_multiple(dev->intr_handle)) { 989 rte_bbdev_log(ERR, "Multiple intr vector is not supported by FPGA (%s)", 990 dev->data->name); 991 return -ENOTSUP; 992 } 993 994 /* Create event file descriptors for each of 64 queue. Event fds will be 995 * mapped to FPGA IRQs in rte_intr_enable(). This is a 1:1 mapping where 996 * the IRQ number is a direct translation to the queue number. 997 * 998 * 63 (FPGA_NUM_INTR_VEC) event fds are created as rte_intr_enable() 999 * mapped the first IRQ to already created interrupt event file 1000 * descriptor (intr_handle->fd). 1001 */ 1002 if (rte_intr_efd_enable(dev->intr_handle, FPGA_NUM_INTR_VEC)) { 1003 rte_bbdev_log(ERR, "Failed to create fds for %u queues", 1004 dev->data->num_queues); 1005 return -1; 1006 } 1007 1008 /* TODO Each event file descriptor is overwritten by interrupt event 1009 * file descriptor. That descriptor is added to epoll observed list. 1010 * It ensures that callback function assigned to that descriptor will 1011 * invoked when any FPGA queue issues interrupt. 1012 */ 1013 for (i = 0; i < FPGA_NUM_INTR_VEC; ++i) 1014 dev->intr_handle->efds[i] = dev->intr_handle->fd; 1015 1016 if (!dev->intr_handle->intr_vec) { 1017 dev->intr_handle->intr_vec = rte_zmalloc("intr_vec", 1018 dev->data->num_queues * sizeof(int), 0); 1019 if (!dev->intr_handle->intr_vec) { 1020 rte_bbdev_log(ERR, "Failed to allocate %u vectors", 1021 dev->data->num_queues); 1022 return -ENOMEM; 1023 } 1024 } 1025 1026 ret = rte_intr_enable(dev->intr_handle); 1027 if (ret < 0) { 1028 rte_bbdev_log(ERR, 1029 "Couldn't enable interrupts for device: %s", 1030 dev->data->name); 1031 return ret; 1032 } 1033 1034 ret = rte_intr_callback_register(dev->intr_handle, 1035 fpga_dev_interrupt_handler, dev); 1036 if (ret < 0) { 1037 rte_bbdev_log(ERR, 1038 "Couldn't register interrupt callback for device: %s", 1039 dev->data->name); 1040 return ret; 1041 } 1042 1043 return 0; 1044 } 1045 1046 static const struct rte_bbdev_ops fpga_ops = { 1047 .setup_queues = fpga_setup_queues, 1048 .intr_enable = fpga_intr_enable, 1049 .close = fpga_dev_close, 1050 .info_get = fpga_dev_info_get, 1051 .queue_setup = fpga_queue_setup, 1052 .queue_stop = fpga_queue_stop, 1053 .queue_start = fpga_queue_start, 1054 .queue_release = fpga_queue_release, 1055 .queue_intr_enable = fpga_queue_intr_enable, 1056 .queue_intr_disable = fpga_queue_intr_disable 1057 }; 1058 1059 static inline void 1060 fpga_dma_enqueue(struct fpga_queue *q, uint16_t num_desc, 1061 struct rte_bbdev_stats *queue_stats) 1062 { 1063 #ifdef RTE_BBDEV_OFFLOAD_COST 1064 uint64_t start_time = 0; 1065 queue_stats->acc_offload_cycles = 0; 1066 #else 1067 RTE_SET_USED(queue_stats); 1068 #endif 1069 1070 /* Update tail and shadow_tail register */ 1071 q->tail = (q->tail + num_desc) & q->sw_ring_wrap_mask; 1072 1073 rte_wmb(); 1074 1075 #ifdef RTE_BBDEV_OFFLOAD_COST 1076 /* Start time measurement for enqueue function offload. */ 1077 start_time = rte_rdtsc_precise(); 1078 #endif 1079 mmio_write_16(q->shadow_tail_addr, q->tail); 1080 1081 #ifdef RTE_BBDEV_OFFLOAD_COST 1082 rte_wmb(); 1083 queue_stats->acc_offload_cycles += rte_rdtsc_precise() - start_time; 1084 #endif 1085 } 1086 1087 /* Calculates number of CBs in processed encoder TB based on 'r' and input 1088 * length. 1089 */ 1090 static inline uint8_t 1091 get_num_cbs_in_op_enc(struct rte_bbdev_op_turbo_enc *turbo_enc) 1092 { 1093 uint8_t c, c_neg, r, crc24_bits = 0; 1094 uint16_t k, k_neg, k_pos; 1095 uint8_t cbs_in_op = 0; 1096 int32_t length; 1097 1098 length = turbo_enc->input.length; 1099 r = turbo_enc->tb_params.r; 1100 c = turbo_enc->tb_params.c; 1101 c_neg = turbo_enc->tb_params.c_neg; 1102 k_neg = turbo_enc->tb_params.k_neg; 1103 k_pos = turbo_enc->tb_params.k_pos; 1104 crc24_bits = 24; 1105 while (length > 0 && r < c) { 1106 k = (r < c_neg) ? k_neg : k_pos; 1107 length -= (k - crc24_bits) >> 3; 1108 r++; 1109 cbs_in_op++; 1110 } 1111 1112 return cbs_in_op; 1113 } 1114 1115 /* Calculates number of CBs in processed decoder TB based on 'r' and input 1116 * length. 1117 */ 1118 static inline uint16_t 1119 get_num_cbs_in_op_dec(struct rte_bbdev_op_turbo_dec *turbo_dec) 1120 { 1121 uint8_t c, c_neg, r = 0; 1122 uint16_t kw, k, k_neg, k_pos, cbs_in_op = 0; 1123 int32_t length; 1124 1125 length = turbo_dec->input.length; 1126 r = turbo_dec->tb_params.r; 1127 c = turbo_dec->tb_params.c; 1128 c_neg = turbo_dec->tb_params.c_neg; 1129 k_neg = turbo_dec->tb_params.k_neg; 1130 k_pos = turbo_dec->tb_params.k_pos; 1131 while (length > 0 && r < c) { 1132 k = (r < c_neg) ? k_neg : k_pos; 1133 kw = RTE_ALIGN_CEIL(k + 4, 32) * 3; 1134 length -= kw; 1135 r++; 1136 cbs_in_op++; 1137 } 1138 1139 return cbs_in_op; 1140 } 1141 1142 /* Read flag value 0/1/ from bitmap */ 1143 static inline bool 1144 check_bit(uint32_t bitmap, uint32_t bitmask) 1145 { 1146 return bitmap & bitmask; 1147 } 1148 1149 /* Print an error if a descriptor error has occurred. 1150 * Return 0 on success, 1 on failure 1151 */ 1152 static inline int 1153 check_desc_error(uint32_t error_code) { 1154 switch (error_code) { 1155 case DESC_ERR_NO_ERR: 1156 return 0; 1157 case DESC_ERR_K_OUT_OF_RANGE: 1158 rte_bbdev_log(ERR, "Block_size_k is out of range (k<40 or k>6144)"); 1159 break; 1160 case DESC_ERR_K_NOT_NORMAL: 1161 rte_bbdev_log(ERR, "Block_size_k is not a normal value within normal range"); 1162 break; 1163 case DESC_ERR_KPAI_NOT_NORMAL: 1164 rte_bbdev_log(ERR, "Three_kpai is not a normal value for UL only"); 1165 break; 1166 case DESC_ERR_DESC_OFFSET_ERR: 1167 rte_bbdev_log(ERR, "Queue offset does not meet the expectation in the FPGA"); 1168 break; 1169 case (DESC_ERR_K_OUT_OF_RANGE | DESC_ERR_DESC_OFFSET_ERR): 1170 rte_bbdev_log(ERR, "Block_size_k is out of range (k<40 or k>6144) and queue offset error"); 1171 break; 1172 case (DESC_ERR_K_NOT_NORMAL | DESC_ERR_DESC_OFFSET_ERR): 1173 rte_bbdev_log(ERR, "Block_size_k is not a normal value within normal range and queue offset error"); 1174 break; 1175 case (DESC_ERR_KPAI_NOT_NORMAL | DESC_ERR_DESC_OFFSET_ERR): 1176 rte_bbdev_log(ERR, "Three_kpai is not a normal value for UL only and queue offset error"); 1177 break; 1178 case DESC_ERR_DESC_READ_FAIL: 1179 rte_bbdev_log(ERR, "Unsuccessful completion for descriptor read"); 1180 break; 1181 case DESC_ERR_DESC_READ_TIMEOUT: 1182 rte_bbdev_log(ERR, "Descriptor read time-out"); 1183 break; 1184 case DESC_ERR_DESC_READ_TLP_POISONED: 1185 rte_bbdev_log(ERR, "Descriptor read TLP poisoned"); 1186 break; 1187 case DESC_ERR_CB_READ_FAIL: 1188 rte_bbdev_log(ERR, "Unsuccessful completion for code block"); 1189 break; 1190 case DESC_ERR_CB_READ_TIMEOUT: 1191 rte_bbdev_log(ERR, "Code block read time-out"); 1192 break; 1193 case DESC_ERR_CB_READ_TLP_POISONED: 1194 rte_bbdev_log(ERR, "Code block read TLP poisoned"); 1195 break; 1196 default: 1197 rte_bbdev_log(ERR, "Descriptor error unknown error code %u", 1198 error_code); 1199 break; 1200 } 1201 return 1; 1202 } 1203 1204 /** 1205 * Set DMA descriptor for encode operation (1 Code Block) 1206 * 1207 * @param op 1208 * Pointer to a single encode operation. 1209 * @param desc 1210 * Pointer to DMA descriptor. 1211 * @param input 1212 * Pointer to pointer to input data which will be decoded. 1213 * @param k 1214 * K value (length of input in bits). 1215 * @param e 1216 * E value (length of output in bits). 1217 * @param ncb 1218 * Ncb value (size of the soft buffer). 1219 * @param out_length 1220 * Length of output buffer 1221 * @param in_offset 1222 * Input offset in rte_mbuf structure. It is used for calculating the point 1223 * where data is starting. 1224 * @param out_offset 1225 * Output offset in rte_mbuf structure. It is used for calculating the point 1226 * where hard output data will be stored. 1227 * @param cbs_in_op 1228 * Number of CBs contained in one operation. 1229 */ 1230 static inline int 1231 fpga_dma_desc_te_fill(struct rte_bbdev_enc_op *op, 1232 struct fpga_dma_enc_desc *desc, struct rte_mbuf *input, 1233 struct rte_mbuf *output, uint16_t k, uint16_t e, uint16_t ncb, 1234 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset, 1235 uint8_t cbs_in_op) 1236 1237 { 1238 /* reset */ 1239 desc->done = 0; 1240 desc->crc_en = check_bit(op->turbo_enc.op_flags, 1241 RTE_BBDEV_TURBO_CRC_24B_ATTACH); 1242 desc->bypass_rm = !check_bit(op->turbo_enc.op_flags, 1243 RTE_BBDEV_TURBO_RATE_MATCH); 1244 desc->k = k; 1245 desc->e = e; 1246 desc->ncb = ncb; 1247 desc->rv = op->turbo_enc.rv_index; 1248 desc->offset = desc_offset; 1249 /* Set inbound data buffer address */ 1250 desc->in_addr_hi = (uint32_t)( 1251 rte_pktmbuf_mtophys_offset(input, in_offset) >> 32); 1252 desc->in_addr_lw = (uint32_t)( 1253 rte_pktmbuf_mtophys_offset(input, in_offset)); 1254 1255 desc->out_addr_hi = (uint32_t)( 1256 rte_pktmbuf_mtophys_offset(output, out_offset) >> 32); 1257 desc->out_addr_lw = (uint32_t)( 1258 rte_pktmbuf_mtophys_offset(output, out_offset)); 1259 1260 /* Save software context needed for dequeue */ 1261 desc->op_addr = op; 1262 1263 /* Set total number of CBs in an op */ 1264 desc->cbs_in_op = cbs_in_op; 1265 1266 return 0; 1267 } 1268 1269 /** 1270 * Set DMA descriptor for encode operation (1 Code Block) 1271 * 1272 * @param op 1273 * Pointer to a single encode operation. 1274 * @param desc 1275 * Pointer to DMA descriptor. 1276 * @param input 1277 * Pointer to pointer to input data which will be decoded. 1278 * @param in_length 1279 * Length of an input. 1280 * @param k 1281 * K value (length of an output in bits). 1282 * @param in_offset 1283 * Input offset in rte_mbuf structure. It is used for calculating the point 1284 * where data is starting. 1285 * @param out_offset 1286 * Output offset in rte_mbuf structure. It is used for calculating the point 1287 * where hard output data will be stored. 1288 * @param cbs_in_op 1289 * Number of CBs contained in one operation. 1290 */ 1291 static inline int 1292 fpga_dma_desc_td_fill(struct rte_bbdev_dec_op *op, 1293 struct fpga_dma_dec_desc *desc, struct rte_mbuf *input, 1294 struct rte_mbuf *output, uint16_t in_length, uint16_t k, 1295 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset, 1296 uint8_t cbs_in_op) 1297 { 1298 /* reset */ 1299 desc->done = 0; 1300 /* Set inbound data buffer address */ 1301 desc->in_addr_hi = (uint32_t)( 1302 rte_pktmbuf_mtophys_offset(input, in_offset) >> 32); 1303 desc->in_addr_lw = (uint32_t)( 1304 rte_pktmbuf_mtophys_offset(input, in_offset)); 1305 desc->in_len = in_length; 1306 desc->k = k; 1307 desc->crc_type = !check_bit(op->turbo_dec.op_flags, 1308 RTE_BBDEV_TURBO_CRC_TYPE_24B); 1309 if ((op->turbo_dec.code_block_mode == 0) 1310 && !check_bit(op->turbo_dec.op_flags, 1311 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP)) 1312 desc->drop_crc = 1; 1313 desc->max_iter = op->turbo_dec.iter_max * 2; 1314 desc->offset = desc_offset; 1315 desc->out_addr_hi = (uint32_t)( 1316 rte_pktmbuf_mtophys_offset(output, out_offset) >> 32); 1317 desc->out_addr_lw = (uint32_t)( 1318 rte_pktmbuf_mtophys_offset(output, out_offset)); 1319 1320 /* Save software context needed for dequeue */ 1321 desc->op_addr = op; 1322 1323 /* Set total number of CBs in an op */ 1324 desc->cbs_in_op = cbs_in_op; 1325 1326 return 0; 1327 } 1328 1329 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1330 /* Validates turbo encoder parameters */ 1331 static int 1332 validate_enc_op(struct rte_bbdev_enc_op *op) 1333 { 1334 struct rte_bbdev_op_turbo_enc *turbo_enc = &op->turbo_enc; 1335 struct rte_bbdev_op_enc_turbo_cb_params *cb = NULL; 1336 struct rte_bbdev_op_enc_turbo_tb_params *tb = NULL; 1337 uint16_t kw, kw_neg, kw_pos; 1338 1339 if (turbo_enc->input.length > 1340 RTE_BBDEV_TURBO_MAX_TB_SIZE >> 3) { 1341 rte_bbdev_log(ERR, "TB size (%u) is too big, max: %d", 1342 turbo_enc->input.length, 1343 RTE_BBDEV_TURBO_MAX_TB_SIZE); 1344 op->status = 1 << RTE_BBDEV_DATA_ERROR; 1345 return -1; 1346 } 1347 1348 if (op->mempool == NULL) { 1349 rte_bbdev_log(ERR, "Invalid mempool pointer"); 1350 return -1; 1351 } 1352 if (turbo_enc->input.data == NULL) { 1353 rte_bbdev_log(ERR, "Invalid input pointer"); 1354 return -1; 1355 } 1356 if (turbo_enc->output.data == NULL) { 1357 rte_bbdev_log(ERR, "Invalid output pointer"); 1358 return -1; 1359 } 1360 if (turbo_enc->rv_index > 3) { 1361 rte_bbdev_log(ERR, 1362 "rv_index (%u) is out of range 0 <= value <= 3", 1363 turbo_enc->rv_index); 1364 return -1; 1365 } 1366 if (turbo_enc->code_block_mode != 0 && 1367 turbo_enc->code_block_mode != 1) { 1368 rte_bbdev_log(ERR, 1369 "code_block_mode (%u) is out of range 0 <= value <= 1", 1370 turbo_enc->code_block_mode); 1371 return -1; 1372 } 1373 1374 if (turbo_enc->code_block_mode == 0) { 1375 tb = &turbo_enc->tb_params; 1376 if ((tb->k_neg < RTE_BBDEV_TURBO_MIN_CB_SIZE 1377 || tb->k_neg > RTE_BBDEV_TURBO_MAX_CB_SIZE) 1378 && tb->c_neg > 0) { 1379 rte_bbdev_log(ERR, 1380 "k_neg (%u) is out of range %u <= value <= %u", 1381 tb->k_neg, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1382 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1383 return -1; 1384 } 1385 if (tb->k_pos < RTE_BBDEV_TURBO_MIN_CB_SIZE 1386 || tb->k_pos > RTE_BBDEV_TURBO_MAX_CB_SIZE) { 1387 rte_bbdev_log(ERR, 1388 "k_pos (%u) is out of range %u <= value <= %u", 1389 tb->k_pos, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1390 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1391 return -1; 1392 } 1393 if (tb->c_neg > (RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1)) 1394 rte_bbdev_log(ERR, 1395 "c_neg (%u) is out of range 0 <= value <= %u", 1396 tb->c_neg, 1397 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1); 1398 if (tb->c < 1 || tb->c > RTE_BBDEV_TURBO_MAX_CODE_BLOCKS) { 1399 rte_bbdev_log(ERR, 1400 "c (%u) is out of range 1 <= value <= %u", 1401 tb->c, RTE_BBDEV_TURBO_MAX_CODE_BLOCKS); 1402 return -1; 1403 } 1404 if (tb->cab > tb->c) { 1405 rte_bbdev_log(ERR, 1406 "cab (%u) is greater than c (%u)", 1407 tb->cab, tb->c); 1408 return -1; 1409 } 1410 if ((tb->ea < RTE_BBDEV_TURBO_MIN_CB_SIZE || (tb->ea % 2)) 1411 && tb->r < tb->cab) { 1412 rte_bbdev_log(ERR, 1413 "ea (%u) is less than %u or it is not even", 1414 tb->ea, RTE_BBDEV_TURBO_MIN_CB_SIZE); 1415 return -1; 1416 } 1417 if ((tb->eb < RTE_BBDEV_TURBO_MIN_CB_SIZE || (tb->eb % 2)) 1418 && tb->c > tb->cab) { 1419 rte_bbdev_log(ERR, 1420 "eb (%u) is less than %u or it is not even", 1421 tb->eb, RTE_BBDEV_TURBO_MIN_CB_SIZE); 1422 return -1; 1423 } 1424 1425 kw_neg = 3 * RTE_ALIGN_CEIL(tb->k_neg + 4, 1426 RTE_BBDEV_TURBO_C_SUBBLOCK); 1427 if (tb->ncb_neg < tb->k_neg || tb->ncb_neg > kw_neg) { 1428 rte_bbdev_log(ERR, 1429 "ncb_neg (%u) is out of range (%u) k_neg <= value <= (%u) kw_neg", 1430 tb->ncb_neg, tb->k_neg, kw_neg); 1431 return -1; 1432 } 1433 1434 kw_pos = 3 * RTE_ALIGN_CEIL(tb->k_pos + 4, 1435 RTE_BBDEV_TURBO_C_SUBBLOCK); 1436 if (tb->ncb_pos < tb->k_pos || tb->ncb_pos > kw_pos) { 1437 rte_bbdev_log(ERR, 1438 "ncb_pos (%u) is out of range (%u) k_pos <= value <= (%u) kw_pos", 1439 tb->ncb_pos, tb->k_pos, kw_pos); 1440 return -1; 1441 } 1442 if (tb->r > (tb->c - 1)) { 1443 rte_bbdev_log(ERR, 1444 "r (%u) is greater than c - 1 (%u)", 1445 tb->r, tb->c - 1); 1446 return -1; 1447 } 1448 } else { 1449 cb = &turbo_enc->cb_params; 1450 if (cb->k < RTE_BBDEV_TURBO_MIN_CB_SIZE 1451 || cb->k > RTE_BBDEV_TURBO_MAX_CB_SIZE) { 1452 rte_bbdev_log(ERR, 1453 "k (%u) is out of range %u <= value <= %u", 1454 cb->k, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1455 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1456 return -1; 1457 } 1458 1459 if (cb->e < RTE_BBDEV_TURBO_MIN_CB_SIZE || (cb->e % 2)) { 1460 rte_bbdev_log(ERR, 1461 "e (%u) is less than %u or it is not even", 1462 cb->e, RTE_BBDEV_TURBO_MIN_CB_SIZE); 1463 return -1; 1464 } 1465 1466 kw = RTE_ALIGN_CEIL(cb->k + 4, RTE_BBDEV_TURBO_C_SUBBLOCK) * 3; 1467 if (cb->ncb < cb->k || cb->ncb > kw) { 1468 rte_bbdev_log(ERR, 1469 "ncb (%u) is out of range (%u) k <= value <= (%u) kw", 1470 cb->ncb, cb->k, kw); 1471 return -1; 1472 } 1473 } 1474 1475 return 0; 1476 } 1477 #endif 1478 1479 static inline char * 1480 mbuf_append(struct rte_mbuf *m_head, struct rte_mbuf *m, uint16_t len) 1481 { 1482 if (unlikely(len > rte_pktmbuf_tailroom(m))) 1483 return NULL; 1484 1485 char *tail = (char *)m->buf_addr + m->data_off + m->data_len; 1486 m->data_len = (uint16_t)(m->data_len + len); 1487 m_head->pkt_len = (m_head->pkt_len + len); 1488 return tail; 1489 } 1490 1491 static inline int 1492 enqueue_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op *op, 1493 uint16_t desc_offset) 1494 { 1495 union fpga_dma_desc *desc; 1496 struct rte_mbuf *input; 1497 struct rte_mbuf *output; 1498 int ret; 1499 uint16_t k, e, ncb, ring_offset; 1500 uint32_t total_left, in_length, out_length, in_offset, out_offset; 1501 1502 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1503 /* Validate op structure */ 1504 if (validate_enc_op(op) == -1) { 1505 rte_bbdev_log(ERR, "Turbo encoder validation failed"); 1506 return -EINVAL; 1507 } 1508 #endif 1509 1510 input = op->turbo_enc.input.data; 1511 output = op->turbo_enc.output.data; 1512 in_offset = op->turbo_enc.input.offset; 1513 out_offset = op->turbo_enc.output.offset; 1514 total_left = op->turbo_enc.input.length; 1515 k = op->turbo_enc.cb_params.k; 1516 e = op->turbo_enc.cb_params.e; 1517 ncb = op->turbo_enc.cb_params.ncb; 1518 1519 if (check_bit(op->turbo_enc.op_flags, RTE_BBDEV_TURBO_CRC_24B_ATTACH)) 1520 in_length = ((k - 24) >> 3); 1521 else 1522 in_length = k >> 3; 1523 1524 if (check_bit(op->turbo_enc.op_flags, RTE_BBDEV_TURBO_RATE_MATCH)) 1525 out_length = (e + 7) >> 3; 1526 else 1527 out_length = (k >> 3) * 3 + 2; 1528 1529 mbuf_append(output, output, out_length); 1530 1531 /* Offset into the ring */ 1532 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1533 /* Setup DMA Descriptor */ 1534 desc = q->ring_addr + ring_offset; 1535 1536 ret = fpga_dma_desc_te_fill(op, &desc->enc_req, input, output, k, e, 1537 ncb, in_offset, out_offset, ring_offset, 1); 1538 if (unlikely(ret < 0)) 1539 return ret; 1540 1541 /* Update lengths */ 1542 total_left -= in_length; 1543 op->turbo_enc.output.length += out_length; 1544 1545 if (total_left > 0) { 1546 rte_bbdev_log(ERR, 1547 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u", 1548 total_left, in_length); 1549 return -1; 1550 } 1551 1552 return 1; 1553 } 1554 1555 static inline int 1556 enqueue_enc_one_op_tb(struct fpga_queue *q, struct rte_bbdev_enc_op *op, 1557 uint16_t desc_offset, uint8_t cbs_in_op) 1558 { 1559 union fpga_dma_desc *desc; 1560 struct rte_mbuf *input, *output_head, *output; 1561 int ret; 1562 uint8_t r, c, crc24_bits = 0; 1563 uint16_t k, e, ncb, ring_offset; 1564 uint32_t mbuf_total_left, in_length, out_length, in_offset, out_offset; 1565 uint32_t seg_total_left; 1566 uint16_t current_enqueued_cbs = 0; 1567 1568 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1569 /* Validate op structure */ 1570 if (validate_enc_op(op) == -1) { 1571 rte_bbdev_log(ERR, "Turbo encoder validation failed"); 1572 return -EINVAL; 1573 } 1574 #endif 1575 1576 input = op->turbo_enc.input.data; 1577 output_head = output = op->turbo_enc.output.data; 1578 in_offset = op->turbo_enc.input.offset; 1579 out_offset = op->turbo_enc.output.offset; 1580 mbuf_total_left = op->turbo_enc.input.length; 1581 1582 c = op->turbo_enc.tb_params.c; 1583 r = op->turbo_enc.tb_params.r; 1584 1585 if (check_bit(op->turbo_enc.op_flags, RTE_BBDEV_TURBO_CRC_24B_ATTACH)) 1586 crc24_bits = 24; 1587 1588 while (mbuf_total_left > 0 && r < c && input != NULL) { 1589 seg_total_left = rte_pktmbuf_data_len(input) - in_offset; 1590 1591 e = (r < op->turbo_enc.tb_params.cab) ? 1592 op->turbo_enc.tb_params.ea : 1593 op->turbo_enc.tb_params.eb; 1594 k = (r < op->turbo_enc.tb_params.c_neg) ? 1595 op->turbo_enc.tb_params.k_neg : 1596 op->turbo_enc.tb_params.k_pos; 1597 ncb = (r < op->turbo_enc.tb_params.c_neg) ? 1598 op->turbo_enc.tb_params.ncb_neg : 1599 op->turbo_enc.tb_params.ncb_pos; 1600 1601 in_length = ((k - crc24_bits) >> 3); 1602 1603 if (check_bit(op->turbo_enc.op_flags, 1604 RTE_BBDEV_TURBO_RATE_MATCH)) 1605 out_length = (e + 7) >> 3; 1606 else 1607 out_length = (k >> 3) * 3 + 2; 1608 1609 mbuf_append(output_head, output, out_length); 1610 1611 /* Setup DMA Descriptor */ 1612 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1613 desc = q->ring_addr + ring_offset; 1614 ret = fpga_dma_desc_te_fill(op, &desc->enc_req, input, output, 1615 k, e, ncb, in_offset, out_offset, ring_offset, 1616 cbs_in_op); 1617 if (unlikely(ret < 0)) 1618 return ret; 1619 1620 rte_bbdev_log_debug("DMA request desc %p", desc); 1621 1622 /* Update lengths */ 1623 op->turbo_enc.output.length += out_length; 1624 mbuf_total_left -= in_length; 1625 1626 /* Update offsets */ 1627 if (seg_total_left == in_length) { 1628 /* Go to the next mbuf */ 1629 input = input->next; 1630 output = output->next; 1631 in_offset = 0; 1632 out_offset = 0; 1633 } else { 1634 in_offset += in_length; 1635 out_offset += out_length; 1636 } 1637 1638 r++; 1639 desc_offset++; 1640 current_enqueued_cbs++; 1641 } 1642 1643 if (mbuf_total_left > 0) { 1644 rte_bbdev_log(ERR, 1645 "Some date still left for processing: mbuf_total_left = %u", 1646 mbuf_total_left); 1647 return -1; 1648 } 1649 1650 return current_enqueued_cbs; 1651 } 1652 1653 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1654 /* Validates turbo decoder parameters */ 1655 static int 1656 validate_dec_op(struct rte_bbdev_dec_op *op) 1657 { 1658 struct rte_bbdev_op_turbo_dec *turbo_dec = &op->turbo_dec; 1659 struct rte_bbdev_op_dec_turbo_cb_params *cb = NULL; 1660 struct rte_bbdev_op_dec_turbo_tb_params *tb = NULL; 1661 1662 if (op->mempool == NULL) { 1663 rte_bbdev_log(ERR, "Invalid mempool pointer"); 1664 return -1; 1665 } 1666 if (turbo_dec->input.data == NULL) { 1667 rte_bbdev_log(ERR, "Invalid input pointer"); 1668 return -1; 1669 } 1670 if (turbo_dec->hard_output.data == NULL) { 1671 rte_bbdev_log(ERR, "Invalid hard_output pointer"); 1672 return -1; 1673 } 1674 if (turbo_dec->rv_index > 3) { 1675 rte_bbdev_log(ERR, 1676 "rv_index (%u) is out of range 0 <= value <= 3", 1677 turbo_dec->rv_index); 1678 return -1; 1679 } 1680 if (turbo_dec->iter_min < 1) { 1681 rte_bbdev_log(ERR, 1682 "iter_min (%u) is less than 1", 1683 turbo_dec->iter_min); 1684 return -1; 1685 } 1686 if (turbo_dec->iter_max <= 2) { 1687 rte_bbdev_log(ERR, 1688 "iter_max (%u) is less than or equal to 2", 1689 turbo_dec->iter_max); 1690 return -1; 1691 } 1692 if (turbo_dec->iter_min > turbo_dec->iter_max) { 1693 rte_bbdev_log(ERR, 1694 "iter_min (%u) is greater than iter_max (%u)", 1695 turbo_dec->iter_min, turbo_dec->iter_max); 1696 return -1; 1697 } 1698 if (turbo_dec->code_block_mode != 0 && 1699 turbo_dec->code_block_mode != 1) { 1700 rte_bbdev_log(ERR, 1701 "code_block_mode (%u) is out of range 0 <= value <= 1", 1702 turbo_dec->code_block_mode); 1703 return -1; 1704 } 1705 1706 if (turbo_dec->code_block_mode == 0) { 1707 1708 if ((turbo_dec->op_flags & 1709 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP) && 1710 !(turbo_dec->op_flags & RTE_BBDEV_TURBO_CRC_TYPE_24B)) { 1711 rte_bbdev_log(ERR, 1712 "RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP should accompany RTE_BBDEV_TURBO_CRC_TYPE_24B"); 1713 return -1; 1714 } 1715 1716 tb = &turbo_dec->tb_params; 1717 if ((tb->k_neg < RTE_BBDEV_TURBO_MIN_CB_SIZE 1718 || tb->k_neg > RTE_BBDEV_TURBO_MAX_CB_SIZE) 1719 && tb->c_neg > 0) { 1720 rte_bbdev_log(ERR, 1721 "k_neg (%u) is out of range %u <= value <= %u", 1722 tb->k_neg, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1723 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1724 return -1; 1725 } 1726 if ((tb->k_pos < RTE_BBDEV_TURBO_MIN_CB_SIZE 1727 || tb->k_pos > RTE_BBDEV_TURBO_MAX_CB_SIZE) 1728 && tb->c > tb->c_neg) { 1729 rte_bbdev_log(ERR, 1730 "k_pos (%u) is out of range %u <= value <= %u", 1731 tb->k_pos, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1732 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1733 return -1; 1734 } 1735 if (tb->c_neg > (RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1)) 1736 rte_bbdev_log(ERR, 1737 "c_neg (%u) is out of range 0 <= value <= %u", 1738 tb->c_neg, 1739 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1); 1740 if (tb->c < 1 || tb->c > RTE_BBDEV_TURBO_MAX_CODE_BLOCKS) { 1741 rte_bbdev_log(ERR, 1742 "c (%u) is out of range 1 <= value <= %u", 1743 tb->c, RTE_BBDEV_TURBO_MAX_CODE_BLOCKS); 1744 return -1; 1745 } 1746 if (tb->cab > tb->c) { 1747 rte_bbdev_log(ERR, 1748 "cab (%u) is greater than c (%u)", 1749 tb->cab, tb->c); 1750 return -1; 1751 } 1752 } else { 1753 1754 if (turbo_dec->op_flags & RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP) { 1755 rte_bbdev_log(ERR, 1756 "RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP is invalid in CB-mode"); 1757 return -1; 1758 } 1759 1760 cb = &turbo_dec->cb_params; 1761 if (cb->k < RTE_BBDEV_TURBO_MIN_CB_SIZE 1762 || cb->k > RTE_BBDEV_TURBO_MAX_CB_SIZE) { 1763 rte_bbdev_log(ERR, 1764 "k (%u) is out of range %u <= value <= %u", 1765 cb->k, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1766 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1767 return -1; 1768 } 1769 } 1770 1771 return 0; 1772 } 1773 #endif 1774 1775 static inline int 1776 enqueue_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op *op, 1777 uint16_t desc_offset) 1778 { 1779 union fpga_dma_desc *desc; 1780 struct rte_mbuf *input; 1781 struct rte_mbuf *output; 1782 int ret; 1783 uint16_t k, kw, ring_offset; 1784 uint32_t total_left, in_length, out_length, in_offset, out_offset; 1785 1786 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1787 /* Validate op structure */ 1788 if (validate_dec_op(op) == -1) { 1789 rte_bbdev_log(ERR, "Turbo decoder validation failed"); 1790 return -EINVAL; 1791 } 1792 #endif 1793 1794 input = op->turbo_dec.input.data; 1795 output = op->turbo_dec.hard_output.data; 1796 total_left = op->turbo_dec.input.length; 1797 in_offset = op->turbo_dec.input.offset; 1798 out_offset = op->turbo_dec.hard_output.offset; 1799 1800 k = op->turbo_dec.cb_params.k; 1801 kw = RTE_ALIGN_CEIL(k + 4, 32) * 3; 1802 in_length = kw; 1803 out_length = k >> 3; 1804 1805 mbuf_append(output, output, out_length); 1806 1807 /* Setup DMA Descriptor */ 1808 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1809 desc = q->ring_addr + ring_offset; 1810 ret = fpga_dma_desc_td_fill(op, &desc->dec_req, input, output, 1811 in_length, k, in_offset, out_offset, ring_offset, 1); 1812 if (unlikely(ret < 0)) 1813 return ret; 1814 1815 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1816 print_dma_dec_desc_debug_info(desc); 1817 #endif 1818 1819 /* Update lengths */ 1820 total_left -= in_length; 1821 op->turbo_dec.hard_output.length += out_length; 1822 1823 if (total_left > 0) { 1824 rte_bbdev_log(ERR, 1825 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u", 1826 total_left, in_length); 1827 return -1; 1828 } 1829 1830 return 1; 1831 } 1832 1833 1834 static inline int 1835 enqueue_dec_one_op_tb(struct fpga_queue *q, struct rte_bbdev_dec_op *op, 1836 uint16_t desc_offset, uint8_t cbs_in_op) 1837 { 1838 union fpga_dma_desc *desc; 1839 struct rte_mbuf *input, *output_head, *output; 1840 int ret; 1841 uint8_t r, c; 1842 uint16_t k, kw, in_length, out_length, ring_offset; 1843 uint32_t mbuf_total_left, seg_total_left, in_offset, out_offset; 1844 uint16_t current_enqueued_cbs = 0; 1845 uint16_t crc24_overlap = 0; 1846 1847 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1848 /* Validate op structure */ 1849 if (validate_dec_op(op) == -1) { 1850 rte_bbdev_log(ERR, "Turbo decoder validation failed"); 1851 return -EINVAL; 1852 } 1853 #endif 1854 1855 input = op->turbo_dec.input.data; 1856 output_head = output = op->turbo_dec.hard_output.data; 1857 mbuf_total_left = op->turbo_dec.input.length; 1858 in_offset = op->turbo_dec.input.offset; 1859 out_offset = op->turbo_dec.hard_output.offset; 1860 1861 if (!check_bit(op->turbo_dec.op_flags, 1862 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP)) 1863 crc24_overlap = 24; 1864 1865 c = op->turbo_dec.tb_params.c; 1866 r = op->turbo_dec.tb_params.r; 1867 1868 while (mbuf_total_left > 0 && r < c && input != NULL) { 1869 seg_total_left = rte_pktmbuf_data_len(input) - in_offset; 1870 k = (r < op->turbo_dec.tb_params.c_neg) ? 1871 op->turbo_dec.tb_params.k_neg : 1872 op->turbo_dec.tb_params.k_pos; 1873 kw = RTE_ALIGN_CEIL(k + 4, 32) * 3; 1874 1875 in_length = kw; 1876 out_length = (k - crc24_overlap) >> 3; 1877 1878 mbuf_append(output_head, output, out_length); 1879 1880 if (seg_total_left < in_length) { 1881 rte_bbdev_log(ERR, 1882 "Partial CB found in a TB. FPGA Driver doesn't support scatter-gather operations!"); 1883 return -1; 1884 } 1885 1886 /* Setup DMA Descriptor */ 1887 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1888 desc = q->ring_addr + ring_offset; 1889 ret = fpga_dma_desc_td_fill(op, &desc->dec_req, input, output, 1890 in_length, k, in_offset, out_offset, 1891 ring_offset, cbs_in_op); 1892 if (unlikely(ret < 0)) 1893 return ret; 1894 1895 /* Update lengths */ 1896 ret = rte_pktmbuf_trim(op->turbo_dec.hard_output.data, 1897 (crc24_overlap >> 3)); 1898 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1899 if (ret < 0) { 1900 rte_bbdev_log(ERR, 1901 "The length to remove is greater than the length of the last segment"); 1902 return -EINVAL; 1903 } 1904 #endif 1905 op->turbo_dec.hard_output.length += out_length; 1906 mbuf_total_left -= in_length; 1907 1908 /* Update offsets */ 1909 if (seg_total_left == in_length) { 1910 /* Go to the next mbuf */ 1911 input = input->next; 1912 output = output->next; 1913 in_offset = 0; 1914 out_offset = 0; 1915 } else { 1916 in_offset += in_length; 1917 out_offset += out_length; 1918 } 1919 1920 r++; 1921 desc_offset++; 1922 current_enqueued_cbs++; 1923 } 1924 1925 if (mbuf_total_left > 0) { 1926 rte_bbdev_log(ERR, 1927 "Some date still left for processing: mbuf_total_left = %u", 1928 mbuf_total_left); 1929 return -1; 1930 } 1931 1932 return current_enqueued_cbs; 1933 } 1934 1935 static uint16_t 1936 fpga_enqueue_enc(struct rte_bbdev_queue_data *q_data, 1937 struct rte_bbdev_enc_op **ops, uint16_t num) 1938 { 1939 uint8_t cbs_in_op; 1940 uint16_t i, total_enqueued_cbs = 0; 1941 int32_t avail; 1942 int enqueued_cbs; 1943 struct fpga_queue *q = q_data->queue_private; 1944 union fpga_dma_desc *desc; 1945 1946 /* Check if queue is not full */ 1947 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) == 1948 q->head_free_desc)) 1949 return 0; 1950 1951 /* Calculates available space */ 1952 avail = (q->head_free_desc > q->tail) ? 1953 q->head_free_desc - q->tail - 1 : 1954 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1; 1955 1956 for (i = 0; i < num; ++i) { 1957 if (ops[i]->turbo_enc.code_block_mode == 0) { 1958 cbs_in_op = get_num_cbs_in_op_enc(&ops[i]->turbo_enc); 1959 /* Check if there is available space for further 1960 * processing 1961 */ 1962 if (unlikely(avail - cbs_in_op < 0)) 1963 break; 1964 avail -= cbs_in_op; 1965 enqueued_cbs = enqueue_enc_one_op_tb(q, ops[i], 1966 total_enqueued_cbs, cbs_in_op); 1967 } else { 1968 /* Check if there is available space for further 1969 * processing 1970 */ 1971 if (unlikely(avail - 1 < 0)) 1972 break; 1973 avail -= 1; 1974 enqueued_cbs = enqueue_enc_one_op_cb(q, ops[i], 1975 total_enqueued_cbs); 1976 } 1977 1978 if (enqueued_cbs < 0) 1979 break; 1980 1981 total_enqueued_cbs += enqueued_cbs; 1982 1983 rte_bbdev_log_debug("enqueuing enc ops [%d/%d] | head %d | tail %d", 1984 total_enqueued_cbs, num, 1985 q->head_free_desc, q->tail); 1986 } 1987 1988 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt 1989 * only when all previous CBs were already processed. 1990 */ 1991 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1) 1992 & q->sw_ring_wrap_mask); 1993 desc->enc_req.irq_en = q->irq_enable; 1994 1995 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats); 1996 1997 /* Update stats */ 1998 q_data->queue_stats.enqueued_count += i; 1999 q_data->queue_stats.enqueue_err_count += num - i; 2000 2001 return i; 2002 } 2003 2004 static uint16_t 2005 fpga_enqueue_dec(struct rte_bbdev_queue_data *q_data, 2006 struct rte_bbdev_dec_op **ops, uint16_t num) 2007 { 2008 uint8_t cbs_in_op; 2009 uint16_t i, total_enqueued_cbs = 0; 2010 int32_t avail; 2011 int enqueued_cbs; 2012 struct fpga_queue *q = q_data->queue_private; 2013 union fpga_dma_desc *desc; 2014 2015 /* Check if queue is not full */ 2016 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) == 2017 q->head_free_desc)) 2018 return 0; 2019 2020 /* Calculates available space */ 2021 avail = (q->head_free_desc > q->tail) ? 2022 q->head_free_desc - q->tail - 1 : 2023 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1; 2024 2025 for (i = 0; i < num; ++i) { 2026 if (ops[i]->turbo_dec.code_block_mode == 0) { 2027 cbs_in_op = get_num_cbs_in_op_dec(&ops[i]->turbo_dec); 2028 /* Check if there is available space for further 2029 * processing 2030 */ 2031 if (unlikely(avail - cbs_in_op < 0)) 2032 break; 2033 avail -= cbs_in_op; 2034 enqueued_cbs = enqueue_dec_one_op_tb(q, ops[i], 2035 total_enqueued_cbs, cbs_in_op); 2036 } else { 2037 /* Check if there is available space for further 2038 * processing 2039 */ 2040 if (unlikely(avail - 1 < 0)) 2041 break; 2042 avail -= 1; 2043 enqueued_cbs = enqueue_dec_one_op_cb(q, ops[i], 2044 total_enqueued_cbs); 2045 } 2046 2047 if (enqueued_cbs < 0) 2048 break; 2049 2050 total_enqueued_cbs += enqueued_cbs; 2051 2052 rte_bbdev_log_debug("enqueuing dec ops [%d/%d] | head %d | tail %d", 2053 total_enqueued_cbs, num, 2054 q->head_free_desc, q->tail); 2055 } 2056 2057 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt 2058 * only when all previous CBs were already processed. 2059 */ 2060 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1) 2061 & q->sw_ring_wrap_mask); 2062 desc->dec_req.irq_en = q->irq_enable; 2063 2064 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats); 2065 2066 /* Update stats */ 2067 q_data->queue_stats.enqueued_count += i; 2068 q_data->queue_stats.enqueue_err_count += num - i; 2069 2070 return i; 2071 } 2072 2073 static inline int 2074 dequeue_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op **op, 2075 uint16_t desc_offset) 2076 { 2077 union fpga_dma_desc *desc; 2078 int desc_error = 0; 2079 2080 /* Set current desc */ 2081 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2082 & q->sw_ring_wrap_mask); 2083 2084 /*check if done */ 2085 if (desc->enc_req.done == 0) 2086 return -1; 2087 2088 /* make sure the response is read atomically */ 2089 rte_smp_rmb(); 2090 2091 rte_bbdev_log_debug("DMA response desc %p", desc); 2092 2093 *op = desc->enc_req.op_addr; 2094 /* Check the decriptor error field, return 1 on error */ 2095 desc_error = check_desc_error(desc->enc_req.error); 2096 (*op)->status = desc_error << RTE_BBDEV_DATA_ERROR; 2097 2098 return 1; 2099 } 2100 2101 static inline int 2102 dequeue_enc_one_op_tb(struct fpga_queue *q, struct rte_bbdev_enc_op **op, 2103 uint16_t desc_offset) 2104 { 2105 union fpga_dma_desc *desc; 2106 uint8_t cbs_in_op, cb_idx; 2107 int desc_error = 0; 2108 int status = 0; 2109 2110 /* Set descriptor */ 2111 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2112 & q->sw_ring_wrap_mask); 2113 2114 /* Verify if done bit is set */ 2115 if (desc->enc_req.done == 0) 2116 return -1; 2117 2118 /* Make sure the response is read atomically */ 2119 rte_smp_rmb(); 2120 2121 /* Verify if done bit in all CBs is set */ 2122 cbs_in_op = desc->enc_req.cbs_in_op; 2123 for (cb_idx = 1; cb_idx < cbs_in_op; ++cb_idx) { 2124 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2125 cb_idx) & q->sw_ring_wrap_mask); 2126 if (desc->enc_req.done == 0) 2127 return -1; 2128 } 2129 2130 /* Make sure the response is read atomically */ 2131 rte_smp_rmb(); 2132 2133 for (cb_idx = 0; cb_idx < cbs_in_op; ++cb_idx) { 2134 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2135 cb_idx) & q->sw_ring_wrap_mask); 2136 /* Check the decriptor error field, return 1 on error */ 2137 desc_error = check_desc_error(desc->enc_req.error); 2138 status |= desc_error << RTE_BBDEV_DATA_ERROR; 2139 rte_bbdev_log_debug("DMA response desc %p", desc); 2140 } 2141 2142 *op = desc->enc_req.op_addr; 2143 (*op)->status = status; 2144 return cbs_in_op; 2145 } 2146 2147 static inline int 2148 dequeue_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op **op, 2149 uint16_t desc_offset) 2150 { 2151 union fpga_dma_desc *desc; 2152 int desc_error = 0; 2153 /* Set descriptor */ 2154 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2155 & q->sw_ring_wrap_mask); 2156 2157 /* Verify done bit is set */ 2158 if (desc->dec_req.done == 0) 2159 return -1; 2160 2161 /* make sure the response is read atomically */ 2162 rte_smp_rmb(); 2163 2164 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2165 print_dma_dec_desc_debug_info(desc); 2166 2167 #endif 2168 2169 *op = desc->dec_req.op_addr; 2170 /* FPGA reports in half-iterations, from 0 to 31. get ceiling */ 2171 (*op)->turbo_dec.iter_count = (desc->dec_req.iter + 2) >> 1; 2172 /* crc_pass = 0 when decoder fails */ 2173 (*op)->status = !(desc->dec_req.crc_pass) << RTE_BBDEV_CRC_ERROR; 2174 /* Check the decriptor error field, return 1 on error */ 2175 desc_error = check_desc_error(desc->enc_req.error); 2176 (*op)->status |= desc_error << RTE_BBDEV_DATA_ERROR; 2177 return 1; 2178 } 2179 2180 static inline int 2181 dequeue_dec_one_op_tb(struct fpga_queue *q, struct rte_bbdev_dec_op **op, 2182 uint16_t desc_offset) 2183 { 2184 union fpga_dma_desc *desc; 2185 uint8_t cbs_in_op, cb_idx, iter_count = 0; 2186 int status = 0; 2187 int desc_error = 0; 2188 /* Set descriptor */ 2189 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2190 & q->sw_ring_wrap_mask); 2191 2192 /* Verify if done bit is set */ 2193 if (desc->dec_req.done == 0) 2194 return -1; 2195 2196 /* Make sure the response is read atomically */ 2197 rte_smp_rmb(); 2198 2199 /* Verify if done bit in all CBs is set */ 2200 cbs_in_op = desc->dec_req.cbs_in_op; 2201 for (cb_idx = 1; cb_idx < cbs_in_op; ++cb_idx) { 2202 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2203 cb_idx) & q->sw_ring_wrap_mask); 2204 if (desc->dec_req.done == 0) 2205 return -1; 2206 } 2207 2208 /* Make sure the response is read atomically */ 2209 rte_smp_rmb(); 2210 2211 for (cb_idx = 0; cb_idx < cbs_in_op; ++cb_idx) { 2212 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2213 cb_idx) & q->sw_ring_wrap_mask); 2214 /* get max iter_count for all CBs in op */ 2215 iter_count = RTE_MAX(iter_count, (uint8_t) desc->dec_req.iter); 2216 /* crc_pass = 0 when decoder fails, one fails all */ 2217 status |= !(desc->dec_req.crc_pass) << RTE_BBDEV_CRC_ERROR; 2218 /* Check the decriptor error field, return 1 on error */ 2219 desc_error = check_desc_error(desc->enc_req.error); 2220 status |= desc_error << RTE_BBDEV_DATA_ERROR; 2221 rte_bbdev_log_debug("DMA response desc %p", desc); 2222 } 2223 2224 *op = desc->dec_req.op_addr; 2225 2226 /* FPGA reports in half-iterations, get ceiling */ 2227 (*op)->turbo_dec.iter_count = (iter_count + 2) >> 1; 2228 (*op)->status = status; 2229 return cbs_in_op; 2230 } 2231 2232 static uint16_t 2233 fpga_dequeue_enc(struct rte_bbdev_queue_data *q_data, 2234 struct rte_bbdev_enc_op **ops, uint16_t num) 2235 { 2236 struct fpga_queue *q = q_data->queue_private; 2237 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask; 2238 uint16_t i; 2239 uint16_t dequeued_cbs = 0; 2240 struct rte_bbdev_enc_op *op; 2241 int ret; 2242 2243 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) { 2244 op = (q->ring_addr + ((q->head_free_desc + dequeued_cbs) 2245 & q->sw_ring_wrap_mask))->enc_req.op_addr; 2246 if (op->turbo_enc.code_block_mode == 0) 2247 ret = dequeue_enc_one_op_tb(q, &ops[i], dequeued_cbs); 2248 else 2249 ret = dequeue_enc_one_op_cb(q, &ops[i], dequeued_cbs); 2250 2251 if (ret < 0) 2252 break; 2253 2254 dequeued_cbs += ret; 2255 2256 rte_bbdev_log_debug("dequeuing enc ops [%d/%d] | head %d | tail %d", 2257 dequeued_cbs, num, q->head_free_desc, q->tail); 2258 } 2259 2260 /* Update head */ 2261 q->head_free_desc = (q->head_free_desc + dequeued_cbs) & 2262 q->sw_ring_wrap_mask; 2263 2264 /* Update stats */ 2265 q_data->queue_stats.dequeued_count += i; 2266 2267 return i; 2268 } 2269 2270 static uint16_t 2271 fpga_dequeue_dec(struct rte_bbdev_queue_data *q_data, 2272 struct rte_bbdev_dec_op **ops, uint16_t num) 2273 { 2274 struct fpga_queue *q = q_data->queue_private; 2275 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask; 2276 uint16_t i; 2277 uint16_t dequeued_cbs = 0; 2278 struct rte_bbdev_dec_op *op; 2279 int ret; 2280 2281 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) { 2282 op = (q->ring_addr + ((q->head_free_desc + dequeued_cbs) 2283 & q->sw_ring_wrap_mask))->dec_req.op_addr; 2284 if (op->turbo_dec.code_block_mode == 0) 2285 ret = dequeue_dec_one_op_tb(q, &ops[i], dequeued_cbs); 2286 else 2287 ret = dequeue_dec_one_op_cb(q, &ops[i], dequeued_cbs); 2288 2289 if (ret < 0) 2290 break; 2291 2292 dequeued_cbs += ret; 2293 2294 rte_bbdev_log_debug("dequeuing dec ops [%d/%d] | head %d | tail %d", 2295 dequeued_cbs, num, q->head_free_desc, q->tail); 2296 } 2297 2298 /* Update head */ 2299 q->head_free_desc = (q->head_free_desc + dequeued_cbs) & 2300 q->sw_ring_wrap_mask; 2301 2302 /* Update stats */ 2303 q_data->queue_stats.dequeued_count += i; 2304 2305 return i; 2306 } 2307 2308 /* Initialization Function */ 2309 static void 2310 fpga_lte_fec_init(struct rte_bbdev *dev) 2311 { 2312 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); 2313 2314 dev->dev_ops = &fpga_ops; 2315 dev->enqueue_enc_ops = fpga_enqueue_enc; 2316 dev->enqueue_dec_ops = fpga_enqueue_dec; 2317 dev->dequeue_enc_ops = fpga_dequeue_enc; 2318 dev->dequeue_dec_ops = fpga_dequeue_dec; 2319 2320 ((struct fpga_lte_fec_device *) dev->data->dev_private)->pf_device = 2321 !strcmp(dev->device->driver->name, 2322 RTE_STR(FPGA_LTE_FEC_PF_DRIVER_NAME)); 2323 ((struct fpga_lte_fec_device *) dev->data->dev_private)->mmio_base = 2324 pci_dev->mem_resource[0].addr; 2325 2326 rte_bbdev_log_debug( 2327 "Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64, 2328 dev->device->driver->name, dev->data->name, 2329 (void *)pci_dev->mem_resource[0].addr, 2330 pci_dev->mem_resource[0].phys_addr); 2331 } 2332 2333 static int 2334 fpga_lte_fec_probe(struct rte_pci_driver *pci_drv __rte_unused, 2335 struct rte_pci_device *pci_dev) 2336 { 2337 struct rte_bbdev *bbdev = NULL; 2338 char dev_name[RTE_BBDEV_NAME_MAX_LEN]; 2339 2340 if (pci_dev == NULL) { 2341 rte_bbdev_log(ERR, "NULL PCI device"); 2342 return -EINVAL; 2343 } 2344 2345 rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name)); 2346 2347 /* Allocate memory to be used privately by drivers */ 2348 bbdev = rte_bbdev_allocate(pci_dev->device.name); 2349 if (bbdev == NULL) 2350 return -ENODEV; 2351 2352 /* allocate device private memory */ 2353 bbdev->data->dev_private = rte_zmalloc_socket(dev_name, 2354 sizeof(struct fpga_lte_fec_device), RTE_CACHE_LINE_SIZE, 2355 pci_dev->device.numa_node); 2356 2357 if (bbdev->data->dev_private == NULL) { 2358 rte_bbdev_log(CRIT, 2359 "Allocate of %zu bytes for device \"%s\" failed", 2360 sizeof(struct fpga_lte_fec_device), dev_name); 2361 rte_bbdev_release(bbdev); 2362 return -ENOMEM; 2363 } 2364 2365 /* Fill HW specific part of device structure */ 2366 bbdev->device = &pci_dev->device; 2367 bbdev->intr_handle = &pci_dev->intr_handle; 2368 bbdev->data->socket_id = pci_dev->device.numa_node; 2369 2370 /* Invoke FEC FPGA device initialization function */ 2371 fpga_lte_fec_init(bbdev); 2372 2373 rte_bbdev_log_debug("bbdev id = %u [%s]", 2374 bbdev->data->dev_id, dev_name); 2375 2376 struct fpga_lte_fec_device *d = bbdev->data->dev_private; 2377 uint32_t version_id = fpga_reg_read_32(d->mmio_base, 2378 FPGA_LTE_FEC_VERSION_ID); 2379 rte_bbdev_log(INFO, "FEC FPGA RTL v%u.%u", 2380 ((uint16_t)(version_id >> 16)), ((uint16_t)version_id)); 2381 2382 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2383 if (!strcmp(bbdev->device->driver->name, 2384 RTE_STR(FPGA_LTE_FEC_PF_DRIVER_NAME))) 2385 print_static_reg_debug_info(d->mmio_base); 2386 #endif 2387 return 0; 2388 } 2389 2390 static int 2391 fpga_lte_fec_remove(struct rte_pci_device *pci_dev) 2392 { 2393 struct rte_bbdev *bbdev; 2394 int ret; 2395 uint8_t dev_id; 2396 2397 if (pci_dev == NULL) 2398 return -EINVAL; 2399 2400 /* Find device */ 2401 bbdev = rte_bbdev_get_named_dev(pci_dev->device.name); 2402 if (bbdev == NULL) { 2403 rte_bbdev_log(CRIT, 2404 "Couldn't find HW dev \"%s\" to uninitialise it", 2405 pci_dev->device.name); 2406 return -ENODEV; 2407 } 2408 dev_id = bbdev->data->dev_id; 2409 2410 /* free device private memory before close */ 2411 rte_free(bbdev->data->dev_private); 2412 2413 /* Close device */ 2414 ret = rte_bbdev_close(dev_id); 2415 if (ret < 0) 2416 rte_bbdev_log(ERR, 2417 "Device %i failed to close during uninit: %i", 2418 dev_id, ret); 2419 2420 /* release bbdev from library */ 2421 ret = rte_bbdev_release(bbdev); 2422 if (ret) 2423 rte_bbdev_log(ERR, "Device %i failed to uninit: %i", dev_id, 2424 ret); 2425 2426 rte_bbdev_log_debug("Destroyed bbdev = %u", dev_id); 2427 2428 return 0; 2429 } 2430 2431 static inline void 2432 set_default_fpga_conf(struct fpga_lte_fec_conf *def_conf) 2433 { 2434 /* clear default configuration before initialization */ 2435 memset(def_conf, 0, sizeof(struct fpga_lte_fec_conf)); 2436 /* Set pf mode to true */ 2437 def_conf->pf_mode_en = true; 2438 2439 /* Set ratio between UL and DL to 1:1 (unit of weight is 3 CBs) */ 2440 def_conf->ul_bandwidth = 3; 2441 def_conf->dl_bandwidth = 3; 2442 2443 /* Set Load Balance Factor to 64 */ 2444 def_conf->dl_load_balance = 64; 2445 def_conf->ul_load_balance = 64; 2446 } 2447 2448 /* Initial configuration of FPGA LTE FEC device */ 2449 int 2450 fpga_lte_fec_configure(const char *dev_name, 2451 const struct fpga_lte_fec_conf *conf) 2452 { 2453 uint32_t payload_32, address; 2454 uint16_t payload_16; 2455 uint8_t payload_8; 2456 uint16_t q_id, vf_id, total_q_id, total_ul_q_id, total_dl_q_id; 2457 struct rte_bbdev *bbdev = rte_bbdev_get_named_dev(dev_name); 2458 struct fpga_lte_fec_conf def_conf; 2459 2460 if (bbdev == NULL) { 2461 rte_bbdev_log(ERR, 2462 "Invalid dev_name (%s), or device is not yet initialised", 2463 dev_name); 2464 return -ENODEV; 2465 } 2466 2467 struct fpga_lte_fec_device *d = bbdev->data->dev_private; 2468 2469 if (conf == NULL) { 2470 rte_bbdev_log(ERR, 2471 "FPGA Configuration was not provided. Default configuration will be loaded."); 2472 set_default_fpga_conf(&def_conf); 2473 conf = &def_conf; 2474 } 2475 2476 /* 2477 * Configure UL:DL ratio. 2478 * [7:0]: UL weight 2479 * [15:8]: DL weight 2480 */ 2481 payload_16 = (conf->dl_bandwidth << 8) | conf->ul_bandwidth; 2482 address = FPGA_LTE_FEC_CONFIGURATION; 2483 fpga_reg_write_16(d->mmio_base, address, payload_16); 2484 2485 /* Clear all queues registers */ 2486 payload_32 = FPGA_INVALID_HW_QUEUE_ID; 2487 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 2488 address = (q_id << 2) + FPGA_LTE_FEC_QUEUE_MAP; 2489 fpga_reg_write_32(d->mmio_base, address, payload_32); 2490 } 2491 2492 /* 2493 * If PF mode is enabled allocate all queues for PF only. 2494 * 2495 * For VF mode each VF can have different number of UL and DL queues. 2496 * Total number of queues to configure cannot exceed FPGA 2497 * capabilities - 64 queues - 32 queues for UL and 32 queues for DL. 2498 * Queues mapping is done according to configuration: 2499 * 2500 * UL queues: 2501 * | Q_ID | VF_ID | 2502 * | 0 | 0 | 2503 * | ... | 0 | 2504 * | conf->vf_dl_queues_number[0] - 1 | 0 | 2505 * | conf->vf_dl_queues_number[0] | 1 | 2506 * | ... | 1 | 2507 * | conf->vf_dl_queues_number[1] - 1 | 1 | 2508 * | ... | ... | 2509 * | conf->vf_dl_queues_number[7] - 1 | 7 | 2510 * 2511 * DL queues: 2512 * | Q_ID | VF_ID | 2513 * | 32 | 0 | 2514 * | ... | 0 | 2515 * | conf->vf_ul_queues_number[0] - 1 | 0 | 2516 * | conf->vf_ul_queues_number[0] | 1 | 2517 * | ... | 1 | 2518 * | conf->vf_ul_queues_number[1] - 1 | 1 | 2519 * | ... | ... | 2520 * | conf->vf_ul_queues_number[7] - 1 | 7 | 2521 * 2522 * Example of configuration: 2523 * conf->vf_ul_queues_number[0] = 4; -> 4 UL queues for VF0 2524 * conf->vf_dl_queues_number[0] = 4; -> 4 DL queues for VF0 2525 * conf->vf_ul_queues_number[1] = 2; -> 2 UL queues for VF1 2526 * conf->vf_dl_queues_number[1] = 2; -> 2 DL queues for VF1 2527 * 2528 * UL: 2529 * | Q_ID | VF_ID | 2530 * | 0 | 0 | 2531 * | 1 | 0 | 2532 * | 2 | 0 | 2533 * | 3 | 0 | 2534 * | 4 | 1 | 2535 * | 5 | 1 | 2536 * 2537 * DL: 2538 * | Q_ID | VF_ID | 2539 * | 32 | 0 | 2540 * | 33 | 0 | 2541 * | 34 | 0 | 2542 * | 35 | 0 | 2543 * | 36 | 1 | 2544 * | 37 | 1 | 2545 */ 2546 if (conf->pf_mode_en) { 2547 payload_32 = 0x1; 2548 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 2549 address = (q_id << 2) + FPGA_LTE_FEC_QUEUE_MAP; 2550 fpga_reg_write_32(d->mmio_base, address, payload_32); 2551 } 2552 } else { 2553 /* Calculate total number of UL and DL queues to configure */ 2554 total_ul_q_id = total_dl_q_id = 0; 2555 for (vf_id = 0; vf_id < FPGA_LTE_FEC_NUM_VFS; ++vf_id) { 2556 total_ul_q_id += conf->vf_ul_queues_number[vf_id]; 2557 total_dl_q_id += conf->vf_dl_queues_number[vf_id]; 2558 } 2559 total_q_id = total_dl_q_id + total_ul_q_id; 2560 /* 2561 * Check if total number of queues to configure does not exceed 2562 * FPGA capabilities (64 queues - 32 UL and 32 DL queues) 2563 */ 2564 if ((total_ul_q_id > FPGA_NUM_UL_QUEUES) || 2565 (total_dl_q_id > FPGA_NUM_DL_QUEUES) || 2566 (total_q_id > FPGA_TOTAL_NUM_QUEUES)) { 2567 rte_bbdev_log(ERR, 2568 "FPGA Configuration failed. Too many queues to configure: UL_Q %u, DL_Q %u, FPGA_Q %u", 2569 total_ul_q_id, total_dl_q_id, 2570 FPGA_TOTAL_NUM_QUEUES); 2571 return -EINVAL; 2572 } 2573 total_ul_q_id = 0; 2574 for (vf_id = 0; vf_id < FPGA_LTE_FEC_NUM_VFS; ++vf_id) { 2575 for (q_id = 0; q_id < conf->vf_ul_queues_number[vf_id]; 2576 ++q_id, ++total_ul_q_id) { 2577 address = (total_ul_q_id << 2) + 2578 FPGA_LTE_FEC_QUEUE_MAP; 2579 payload_32 = ((0x80 + vf_id) << 16) | 0x1; 2580 fpga_reg_write_32(d->mmio_base, address, 2581 payload_32); 2582 } 2583 } 2584 total_dl_q_id = 0; 2585 for (vf_id = 0; vf_id < FPGA_LTE_FEC_NUM_VFS; ++vf_id) { 2586 for (q_id = 0; q_id < conf->vf_dl_queues_number[vf_id]; 2587 ++q_id, ++total_dl_q_id) { 2588 address = ((total_dl_q_id + FPGA_NUM_UL_QUEUES) 2589 << 2) + FPGA_LTE_FEC_QUEUE_MAP; 2590 payload_32 = ((0x80 + vf_id) << 16) | 0x1; 2591 fpga_reg_write_32(d->mmio_base, address, 2592 payload_32); 2593 } 2594 } 2595 } 2596 2597 /* Setting Load Balance Factor */ 2598 payload_16 = (conf->dl_load_balance << 8) | (conf->ul_load_balance); 2599 address = FPGA_LTE_FEC_LOAD_BALANCE_FACTOR; 2600 fpga_reg_write_16(d->mmio_base, address, payload_16); 2601 2602 /* Setting length of ring descriptor entry */ 2603 payload_16 = FPGA_RING_DESC_ENTRY_LENGTH; 2604 address = FPGA_LTE_FEC_RING_DESC_LEN; 2605 fpga_reg_write_16(d->mmio_base, address, payload_16); 2606 2607 /* Setting FLR timeout value */ 2608 payload_16 = conf->flr_time_out; 2609 address = FPGA_LTE_FEC_FLR_TIME_OUT; 2610 fpga_reg_write_16(d->mmio_base, address, payload_16); 2611 2612 /* Queue PF/VF mapping table is ready */ 2613 payload_8 = 0x1; 2614 address = FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE; 2615 fpga_reg_write_8(d->mmio_base, address, payload_8); 2616 2617 rte_bbdev_log_debug("PF FPGA LTE FEC configuration complete for %s", 2618 dev_name); 2619 2620 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2621 print_static_reg_debug_info(d->mmio_base); 2622 #endif 2623 return 0; 2624 } 2625 2626 /* FPGA LTE FEC PCI PF address map */ 2627 static struct rte_pci_id pci_id_fpga_lte_fec_pf_map[] = { 2628 { 2629 RTE_PCI_DEVICE(FPGA_LTE_FEC_VENDOR_ID, 2630 FPGA_LTE_FEC_PF_DEVICE_ID) 2631 }, 2632 {.device_id = 0}, 2633 }; 2634 2635 static struct rte_pci_driver fpga_lte_fec_pci_pf_driver = { 2636 .probe = fpga_lte_fec_probe, 2637 .remove = fpga_lte_fec_remove, 2638 .id_table = pci_id_fpga_lte_fec_pf_map, 2639 .drv_flags = RTE_PCI_DRV_NEED_MAPPING 2640 }; 2641 2642 /* FPGA LTE FEC PCI VF address map */ 2643 static struct rte_pci_id pci_id_fpga_lte_fec_vf_map[] = { 2644 { 2645 RTE_PCI_DEVICE(FPGA_LTE_FEC_VENDOR_ID, 2646 FPGA_LTE_FEC_VF_DEVICE_ID) 2647 }, 2648 {.device_id = 0}, 2649 }; 2650 2651 static struct rte_pci_driver fpga_lte_fec_pci_vf_driver = { 2652 .probe = fpga_lte_fec_probe, 2653 .remove = fpga_lte_fec_remove, 2654 .id_table = pci_id_fpga_lte_fec_vf_map, 2655 .drv_flags = RTE_PCI_DRV_NEED_MAPPING 2656 }; 2657 2658 2659 RTE_PMD_REGISTER_PCI(FPGA_LTE_FEC_PF_DRIVER_NAME, fpga_lte_fec_pci_pf_driver); 2660 RTE_PMD_REGISTER_PCI_TABLE(FPGA_LTE_FEC_PF_DRIVER_NAME, 2661 pci_id_fpga_lte_fec_pf_map); 2662 RTE_PMD_REGISTER_PCI(FPGA_LTE_FEC_VF_DRIVER_NAME, fpga_lte_fec_pci_vf_driver); 2663 RTE_PMD_REGISTER_PCI_TABLE(FPGA_LTE_FEC_VF_DRIVER_NAME, 2664 pci_id_fpga_lte_fec_vf_map); 2665 2666 RTE_INIT(fpga_lte_fec_init_log) 2667 { 2668 fpga_lte_fec_logtype = rte_log_register("pmd.bb.fpga_lte_fec"); 2669 if (fpga_lte_fec_logtype >= 0) 2670 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2671 rte_log_set_level(fpga_lte_fec_logtype, RTE_LOG_DEBUG); 2672 #else 2673 rte_log_set_level(fpga_lte_fec_logtype, RTE_LOG_NOTICE); 2674 #endif 2675 } 2676