1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(c) 2019 Intel Corporation 3 */ 4 5 #include <unistd.h> 6 7 #include <rte_common.h> 8 #include <rte_log.h> 9 #include <dev_driver.h> 10 #include <rte_malloc.h> 11 #include <rte_mempool.h> 12 #include <rte_errno.h> 13 #include <rte_pci.h> 14 #include <bus_pci_driver.h> 15 #include <rte_byteorder.h> 16 #include <rte_cycles.h> 17 18 #include <rte_bbdev.h> 19 #include <rte_bbdev_pmd.h> 20 21 #include "fpga_lte_fec.h" 22 23 #ifdef RTE_LIBRTE_BBDEV_DEBUG 24 RTE_LOG_REGISTER_DEFAULT(fpga_lte_fec_logtype, DEBUG); 25 #else 26 RTE_LOG_REGISTER_DEFAULT(fpga_lte_fec_logtype, NOTICE); 27 #endif 28 #define RTE_LOGTYPE_FPGA_LTE_FEC fpga_lte_fec_logtype 29 30 /* Helper macro for logging */ 31 #define rte_bbdev_log(level, ...) \ 32 RTE_LOG_LINE(level, FPGA_LTE_FEC, __VA_ARGS__) 33 34 #ifdef RTE_LIBRTE_BBDEV_DEBUG 35 #define rte_bbdev_log_debug(...) \ 36 rte_bbdev_log(DEBUG, __VA_ARGS__) 37 #else 38 #define rte_bbdev_log_debug(...) 39 #endif 40 41 /* FPGA LTE FEC driver names */ 42 #define FPGA_LTE_FEC_PF_DRIVER_NAME intel_fpga_lte_fec_pf 43 #define FPGA_LTE_FEC_VF_DRIVER_NAME intel_fpga_lte_fec_vf 44 45 /* FPGA LTE FEC PCI vendor & device IDs */ 46 #define FPGA_LTE_FEC_VENDOR_ID (0x1172) 47 #define FPGA_LTE_FEC_PF_DEVICE_ID (0x5052) 48 #define FPGA_LTE_FEC_VF_DEVICE_ID (0x5050) 49 50 /* Align DMA descriptors to 256 bytes - cache-aligned */ 51 #define FPGA_RING_DESC_ENTRY_LENGTH (8) 52 /* Ring size is in 256 bits (32 bytes) units */ 53 #define FPGA_RING_DESC_LEN_UNIT_BYTES (32) 54 /* Maximum size of queue */ 55 #define FPGA_RING_MAX_SIZE (1024) 56 #define FPGA_FLR_TIMEOUT_UNIT (16.384) 57 58 #define FPGA_NUM_UL_QUEUES (32) 59 #define FPGA_NUM_DL_QUEUES (32) 60 #define FPGA_TOTAL_NUM_QUEUES (FPGA_NUM_UL_QUEUES + FPGA_NUM_DL_QUEUES) 61 #define FPGA_NUM_INTR_VEC (FPGA_TOTAL_NUM_QUEUES - RTE_INTR_VEC_RXTX_OFFSET) 62 63 #define FPGA_INVALID_HW_QUEUE_ID (0xFFFFFFFF) 64 65 #define FPGA_QUEUE_FLUSH_TIMEOUT_US (1000) 66 #define FPGA_TIMEOUT_CHECK_INTERVAL (5) 67 68 /* FPGA LTE FEC Register mapping on BAR0 */ 69 enum { 70 FPGA_LTE_FEC_VERSION_ID = 0x00000000, /* len: 4B */ 71 FPGA_LTE_FEC_CONFIGURATION = 0x00000004, /* len: 2B */ 72 FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE = 0x00000008, /* len: 1B */ 73 FPGA_LTE_FEC_LOAD_BALANCE_FACTOR = 0x0000000a, /* len: 2B */ 74 FPGA_LTE_FEC_RING_DESC_LEN = 0x0000000c, /* len: 2B */ 75 FPGA_LTE_FEC_FLR_TIME_OUT = 0x0000000e, /* len: 2B */ 76 FPGA_LTE_FEC_VFQ_FLUSH_STATUS_LW = 0x00000018, /* len: 4B */ 77 FPGA_LTE_FEC_VFQ_FLUSH_STATUS_HI = 0x0000001c, /* len: 4B */ 78 FPGA_LTE_FEC_VF0_DEBUG = 0x00000020, /* len: 4B */ 79 FPGA_LTE_FEC_VF1_DEBUG = 0x00000024, /* len: 4B */ 80 FPGA_LTE_FEC_VF2_DEBUG = 0x00000028, /* len: 4B */ 81 FPGA_LTE_FEC_VF3_DEBUG = 0x0000002c, /* len: 4B */ 82 FPGA_LTE_FEC_VF4_DEBUG = 0x00000030, /* len: 4B */ 83 FPGA_LTE_FEC_VF5_DEBUG = 0x00000034, /* len: 4B */ 84 FPGA_LTE_FEC_VF6_DEBUG = 0x00000038, /* len: 4B */ 85 FPGA_LTE_FEC_VF7_DEBUG = 0x0000003c, /* len: 4B */ 86 FPGA_LTE_FEC_QUEUE_MAP = 0x00000040, /* len: 256B */ 87 FPGA_LTE_FEC_RING_CTRL_REGS = 0x00000200 /* len: 2048B */ 88 }; 89 90 /* FPGA LTE FEC Ring Control Registers */ 91 enum { 92 FPGA_LTE_FEC_RING_HEAD_ADDR = 0x00000008, 93 FPGA_LTE_FEC_RING_SIZE = 0x00000010, 94 FPGA_LTE_FEC_RING_MISC = 0x00000014, 95 FPGA_LTE_FEC_RING_ENABLE = 0x00000015, 96 FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN = 0x00000016, 97 FPGA_LTE_FEC_RING_SHADOW_TAIL = 0x00000018, 98 FPGA_LTE_FEC_RING_HEAD_POINT = 0x0000001C 99 }; 100 101 /* FPGA LTE FEC DESCRIPTOR ERROR */ 102 enum { 103 DESC_ERR_NO_ERR = 0x0, 104 DESC_ERR_K_OUT_OF_RANGE = 0x1, 105 DESC_ERR_K_NOT_NORMAL = 0x2, 106 DESC_ERR_KPAI_NOT_NORMAL = 0x3, 107 DESC_ERR_DESC_OFFSET_ERR = 0x4, 108 DESC_ERR_DESC_READ_FAIL = 0x8, 109 DESC_ERR_DESC_READ_TIMEOUT = 0x9, 110 DESC_ERR_DESC_READ_TLP_POISONED = 0xA, 111 DESC_ERR_CB_READ_FAIL = 0xC, 112 DESC_ERR_CB_READ_TIMEOUT = 0xD, 113 DESC_ERR_CB_READ_TLP_POISONED = 0xE 114 }; 115 116 /* FPGA LTE FEC DMA Encoding Request Descriptor */ 117 struct __rte_packed_begin fpga_dma_enc_desc { 118 uint32_t done:1, 119 rsrvd0:11, 120 error:4, 121 rsrvd1:16; 122 uint32_t ncb:16, 123 rsrvd2:14, 124 rv:2; 125 uint32_t bypass_rm:1, 126 irq_en:1, 127 crc_en:1, 128 rsrvd3:13, 129 offset:10, 130 rsrvd4:6; 131 uint16_t e; 132 uint16_t k; 133 uint32_t out_addr_lw; 134 uint32_t out_addr_hi; 135 uint32_t in_addr_lw; 136 uint32_t in_addr_hi; 137 138 union { 139 struct { 140 /* Virtual addresses used to retrieve SW context info */ 141 void *op_addr; 142 /* Stores information about total number of Code Blocks 143 * in currently processed Transport Block 144 */ 145 uint64_t cbs_in_op; 146 }; 147 148 uint8_t sw_ctxt[FPGA_RING_DESC_LEN_UNIT_BYTES * 149 (FPGA_RING_DESC_ENTRY_LENGTH - 1)]; 150 }; 151 } __rte_packed_end; 152 153 /* FPGA LTE FEC DMA Decoding Request Descriptor */ 154 struct __rte_packed_begin fpga_dma_dec_desc { 155 uint32_t done:1, 156 iter:5, 157 rsrvd0:2, 158 crc_pass:1, 159 rsrvd1:3, 160 error:4, 161 crc_type:1, 162 rsrvd2:7, 163 max_iter:5, 164 rsrvd3:3; 165 uint32_t rsrvd4; 166 uint32_t bypass_rm:1, 167 irq_en:1, 168 drop_crc:1, 169 rsrvd5:13, 170 offset:10, 171 rsrvd6:6; 172 uint16_t k; 173 uint16_t in_len; 174 uint32_t out_addr_lw; 175 uint32_t out_addr_hi; 176 uint32_t in_addr_lw; 177 uint32_t in_addr_hi; 178 179 union { 180 struct { 181 /* Virtual addresses used to retrieve SW context info */ 182 void *op_addr; 183 /* Stores information about total number of Code Blocks 184 * in currently processed Transport Block 185 */ 186 uint8_t cbs_in_op; 187 }; 188 189 uint32_t sw_ctxt[8 * (FPGA_RING_DESC_ENTRY_LENGTH - 1)]; 190 }; 191 } __rte_packed_end; 192 193 /* FPGA LTE DMA Descriptor */ 194 union fpga_dma_desc { 195 struct fpga_dma_enc_desc enc_req; 196 struct fpga_dma_dec_desc dec_req; 197 }; 198 199 /* FPGA LTE FEC Ring Control Register */ 200 struct __rte_packed_begin fpga_ring_ctrl_reg { 201 uint64_t ring_base_addr; 202 uint64_t ring_head_addr; 203 uint16_t ring_size:11; 204 uint16_t rsrvd0; 205 union { /* Miscellaneous register */ 206 uint8_t misc; 207 uint8_t max_ul_dec:5, 208 max_ul_dec_en:1, 209 rsrvd1:2; 210 }; 211 uint8_t enable; 212 uint8_t flush_queue_en; 213 uint8_t rsrvd2; 214 uint16_t shadow_tail; 215 uint16_t rsrvd3; 216 uint16_t head_point; 217 uint16_t rsrvd4; 218 219 } __rte_packed_end; 220 221 /* Private data structure for each FPGA FEC device */ 222 struct fpga_lte_fec_device { 223 /** Base address of MMIO registers (BAR0) */ 224 void *mmio_base; 225 /** Base address of memory for sw rings */ 226 void *sw_rings; 227 /** Physical address of sw_rings */ 228 rte_iova_t sw_rings_phys; 229 /** Number of bytes available for each queue in device. */ 230 uint32_t sw_ring_size; 231 /** Max number of entries available for each queue in device */ 232 uint32_t sw_ring_max_depth; 233 /** Base address of response tail pointer buffer */ 234 uint32_t *tail_ptrs; 235 /** Physical address of tail pointers */ 236 rte_iova_t tail_ptr_phys; 237 /** Queues flush completion flag */ 238 uint64_t *flush_queue_status; 239 /* Bitmap capturing which Queues are bound to the PF/VF */ 240 uint64_t q_bound_bit_map; 241 /* Bitmap capturing which Queues have already been assigned */ 242 uint64_t q_assigned_bit_map; 243 /** True if this is a PF FPGA FEC device */ 244 bool pf_device; 245 }; 246 247 /* Structure associated with each queue. */ 248 struct __rte_cache_aligned fpga_queue { 249 struct fpga_ring_ctrl_reg ring_ctrl_reg; /* Ring Control Register */ 250 union fpga_dma_desc *ring_addr; /* Virtual address of software ring */ 251 uint64_t *ring_head_addr; /* Virtual address of completion_head */ 252 uint64_t shadow_completion_head; /* Shadow completion head value */ 253 uint16_t head_free_desc; /* Ring head */ 254 uint16_t tail; /* Ring tail */ 255 /* Mask used to wrap enqueued descriptors on the sw ring */ 256 uint32_t sw_ring_wrap_mask; 257 uint32_t irq_enable; /* Enable ops dequeue interrupts if set to 1 */ 258 uint8_t q_idx; /* Queue index */ 259 struct fpga_lte_fec_device *d; 260 /* MMIO register of shadow_tail used to enqueue descriptors */ 261 void *shadow_tail_addr; 262 }; 263 264 /* Write to 16 bit MMIO register address */ 265 static inline void 266 mmio_write_16(void *addr, uint16_t value) 267 { 268 *((volatile uint16_t *)(addr)) = rte_cpu_to_le_16(value); 269 } 270 271 /* Write to 32 bit MMIO register address */ 272 static inline void 273 mmio_write_32(void *addr, uint32_t value) 274 { 275 *((volatile uint32_t *)(addr)) = rte_cpu_to_le_32(value); 276 } 277 278 /* Write to 64 bit MMIO register address */ 279 static inline void 280 mmio_write_64(void *addr, uint64_t value) 281 { 282 *((volatile uint64_t *)(addr)) = rte_cpu_to_le_64(value); 283 } 284 285 /* Write a 8 bit register of a FPGA LTE FEC device */ 286 static inline void 287 fpga_reg_write_8(void *mmio_base, uint32_t offset, uint8_t payload) 288 { 289 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 290 *((volatile uint8_t *)(reg_addr)) = payload; 291 } 292 293 /* Write a 16 bit register of a FPGA LTE FEC device */ 294 static inline void 295 fpga_reg_write_16(void *mmio_base, uint32_t offset, uint16_t payload) 296 { 297 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 298 mmio_write_16(reg_addr, payload); 299 } 300 301 /* Write a 32 bit register of a FPGA LTE FEC device */ 302 static inline void 303 fpga_reg_write_32(void *mmio_base, uint32_t offset, uint32_t payload) 304 { 305 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 306 mmio_write_32(reg_addr, payload); 307 } 308 309 /* Write a 64 bit register of a FPGA LTE FEC device */ 310 static inline void 311 fpga_reg_write_64(void *mmio_base, uint32_t offset, uint64_t payload) 312 { 313 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 314 mmio_write_64(reg_addr, payload); 315 } 316 317 /* Write a ring control register of a FPGA LTE FEC device */ 318 static inline void 319 fpga_ring_reg_write(void *mmio_base, uint32_t offset, 320 struct fpga_ring_ctrl_reg payload) 321 { 322 fpga_reg_write_64(mmio_base, offset, payload.ring_base_addr); 323 fpga_reg_write_64(mmio_base, offset + FPGA_LTE_FEC_RING_HEAD_ADDR, 324 payload.ring_head_addr); 325 fpga_reg_write_16(mmio_base, offset + FPGA_LTE_FEC_RING_SIZE, 326 payload.ring_size); 327 fpga_reg_write_16(mmio_base, offset + FPGA_LTE_FEC_RING_HEAD_POINT, 328 payload.head_point); 329 fpga_reg_write_8(mmio_base, offset + FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN, 330 payload.flush_queue_en); 331 fpga_reg_write_16(mmio_base, offset + FPGA_LTE_FEC_RING_SHADOW_TAIL, 332 payload.shadow_tail); 333 fpga_reg_write_8(mmio_base, offset + FPGA_LTE_FEC_RING_MISC, 334 payload.misc); 335 fpga_reg_write_8(mmio_base, offset + FPGA_LTE_FEC_RING_ENABLE, 336 payload.enable); 337 } 338 339 /* Read a register of FPGA LTE FEC device */ 340 static uint32_t 341 fpga_reg_read_32(void *mmio_base, uint32_t offset) 342 { 343 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 344 uint32_t ret = *((volatile uint32_t *)(reg_addr)); 345 return rte_le_to_cpu_32(ret); 346 } 347 348 #ifdef RTE_LIBRTE_BBDEV_DEBUG 349 /* Read a register of FPGA LTE FEC device */ 350 static uint8_t 351 fpga_reg_read_8(void *mmio_base, uint32_t offset) 352 { 353 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 354 return *((volatile uint8_t *)(reg_addr)); 355 } 356 357 /* Read a register of FPGA LTE FEC device */ 358 static uint16_t 359 fpga_reg_read_16(void *mmio_base, uint32_t offset) 360 { 361 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 362 uint16_t ret = *((volatile uint16_t *)(reg_addr)); 363 return rte_le_to_cpu_16(ret); 364 } 365 366 /* Read a register of FPGA LTE FEC device */ 367 static uint64_t 368 fpga_reg_read_64(void *mmio_base, uint32_t offset) 369 { 370 void *reg_addr = RTE_PTR_ADD(mmio_base, offset); 371 uint64_t ret = *((volatile uint64_t *)(reg_addr)); 372 return rte_le_to_cpu_64(ret); 373 } 374 375 /* Read Ring Control Register of FPGA LTE FEC device */ 376 static inline void 377 print_ring_reg_debug_info(void *mmio_base, uint32_t offset) 378 { 379 rte_bbdev_log_debug( 380 "FPGA MMIO base address @ %p | Ring Control Register @ offset = 0x%08" 381 PRIx32, mmio_base, offset); 382 rte_bbdev_log_debug( 383 "RING_BASE_ADDR = 0x%016"PRIx64, 384 fpga_reg_read_64(mmio_base, offset)); 385 rte_bbdev_log_debug( 386 "RING_HEAD_ADDR = 0x%016"PRIx64, 387 fpga_reg_read_64(mmio_base, offset + 388 FPGA_LTE_FEC_RING_HEAD_ADDR)); 389 rte_bbdev_log_debug( 390 "RING_SIZE = 0x%04"PRIx16, 391 fpga_reg_read_16(mmio_base, offset + 392 FPGA_LTE_FEC_RING_SIZE)); 393 rte_bbdev_log_debug( 394 "RING_MISC = 0x%02"PRIx8, 395 fpga_reg_read_8(mmio_base, offset + 396 FPGA_LTE_FEC_RING_MISC)); 397 rte_bbdev_log_debug( 398 "RING_ENABLE = 0x%02"PRIx8, 399 fpga_reg_read_8(mmio_base, offset + 400 FPGA_LTE_FEC_RING_ENABLE)); 401 rte_bbdev_log_debug( 402 "RING_FLUSH_QUEUE_EN = 0x%02"PRIx8, 403 fpga_reg_read_8(mmio_base, offset + 404 FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN)); 405 rte_bbdev_log_debug( 406 "RING_SHADOW_TAIL = 0x%04"PRIx16, 407 fpga_reg_read_16(mmio_base, offset + 408 FPGA_LTE_FEC_RING_SHADOW_TAIL)); 409 rte_bbdev_log_debug( 410 "RING_HEAD_POINT = 0x%04"PRIx16, 411 fpga_reg_read_16(mmio_base, offset + 412 FPGA_LTE_FEC_RING_HEAD_POINT)); 413 } 414 415 /* Read Static Register of FPGA LTE FEC device */ 416 static inline void 417 print_static_reg_debug_info(void *mmio_base) 418 { 419 uint16_t config = fpga_reg_read_16(mmio_base, 420 FPGA_LTE_FEC_CONFIGURATION); 421 uint8_t qmap_done = fpga_reg_read_8(mmio_base, 422 FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE); 423 uint16_t lb_factor = fpga_reg_read_16(mmio_base, 424 FPGA_LTE_FEC_LOAD_BALANCE_FACTOR); 425 uint16_t ring_desc_len = fpga_reg_read_16(mmio_base, 426 FPGA_LTE_FEC_RING_DESC_LEN); 427 uint16_t flr_time_out = fpga_reg_read_16(mmio_base, 428 FPGA_LTE_FEC_FLR_TIME_OUT); 429 430 rte_bbdev_log_debug("UL.DL Weights = %u.%u", 431 ((uint8_t)config), ((uint8_t)(config >> 8))); 432 rte_bbdev_log_debug("UL.DL Load Balance = %u.%u", 433 ((uint8_t)lb_factor), ((uint8_t)(lb_factor >> 8))); 434 rte_bbdev_log_debug("Queue-PF/VF Mapping Table = %s", 435 (qmap_done > 0) ? "READY" : "NOT-READY"); 436 rte_bbdev_log_debug("Ring Descriptor Size = %u bytes", 437 ring_desc_len*FPGA_RING_DESC_LEN_UNIT_BYTES); 438 rte_bbdev_log_debug("FLR Timeout = %f usec", 439 (float)flr_time_out*FPGA_FLR_TIMEOUT_UNIT); 440 } 441 442 /* Print decode DMA Descriptor of FPGA LTE FEC device */ 443 static void 444 print_dma_dec_desc_debug_info(union fpga_dma_desc *desc) 445 { 446 rte_bbdev_log_debug("DMA response desc %p", 447 desc); 448 rte_bbdev_log_debug("\t-- done(%"PRIu32") | iter(%"PRIu32") | crc_pass(%"PRIu32")" 449 " | error (%"PRIu32") | crc_type(%"PRIu32")", 450 (uint32_t)desc->dec_req.done, 451 (uint32_t)desc->dec_req.iter, 452 (uint32_t)desc->dec_req.crc_pass, 453 (uint32_t)desc->dec_req.error, 454 (uint32_t)desc->dec_req.crc_type); 455 rte_bbdev_log_debug("\t-- max_iter(%"PRIu32") | bypass_rm(%"PRIu32") | " 456 "irq_en (%"PRIu32") | drop_crc(%"PRIu32") | offset(%"PRIu32")", 457 (uint32_t)desc->dec_req.max_iter, 458 (uint32_t)desc->dec_req.bypass_rm, 459 (uint32_t)desc->dec_req.irq_en, 460 (uint32_t)desc->dec_req.drop_crc, 461 (uint32_t)desc->dec_req.offset); 462 rte_bbdev_log_debug("\t-- k(%"PRIu32") | in_len (%"PRIu16") | op_add(%p)", 463 (uint32_t)desc->dec_req.k, 464 (uint16_t)desc->dec_req.in_len, 465 desc->dec_req.op_addr); 466 rte_bbdev_log_debug("\t-- cbs_in_op(%"PRIu32") | in_add (0x%08"PRIx32"%08"PRIx32") | " 467 "out_add (0x%08"PRIx32"%08"PRIx32")", 468 (uint32_t)desc->dec_req.cbs_in_op, 469 (uint32_t)desc->dec_req.in_addr_hi, 470 (uint32_t)desc->dec_req.in_addr_lw, 471 (uint32_t)desc->dec_req.out_addr_hi, 472 (uint32_t)desc->dec_req.out_addr_lw); 473 } 474 #endif 475 476 static int 477 fpga_setup_queues(struct rte_bbdev *dev, uint16_t num_queues, int socket_id) 478 { 479 /* Number of queues bound to a PF/VF */ 480 uint32_t hw_q_num = 0; 481 uint32_t ring_size, payload, address, q_id, offset; 482 rte_iova_t phys_addr; 483 struct fpga_ring_ctrl_reg ring_reg; 484 struct fpga_lte_fec_device *fpga_dev = dev->data->dev_private; 485 486 address = FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE; 487 if (!(fpga_reg_read_32(fpga_dev->mmio_base, address) & 0x1)) { 488 rte_bbdev_log(ERR, 489 "Queue-PF/VF mapping is not set! Was PF configured for device (%s) ?", 490 dev->data->name); 491 return -EPERM; 492 } 493 494 /* Clear queue registers structure */ 495 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg)); 496 497 /* Scan queue map. 498 * If a queue is valid and mapped to a calling PF/VF the read value is 499 * replaced with a queue ID and if it's not then 500 * FPGA_INVALID_HW_QUEUE_ID is returned. 501 */ 502 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 503 uint32_t hw_q_id = fpga_reg_read_32(fpga_dev->mmio_base, 504 FPGA_LTE_FEC_QUEUE_MAP + (q_id << 2)); 505 506 rte_bbdev_log_debug("%s: queue ID: %u, registry queue ID: %u", 507 dev->device->name, q_id, hw_q_id); 508 509 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) { 510 fpga_dev->q_bound_bit_map |= (1ULL << q_id); 511 /* Clear queue register of found queue */ 512 offset = FPGA_LTE_FEC_RING_CTRL_REGS + 513 (sizeof(struct fpga_ring_ctrl_reg) * q_id); 514 fpga_ring_reg_write(fpga_dev->mmio_base, 515 offset, ring_reg); 516 ++hw_q_num; 517 } 518 } 519 if (hw_q_num == 0) { 520 rte_bbdev_log(ERR, 521 "No HW queues assigned to this device. Probably this is a VF configured for PF mode. Check device configuration!"); 522 return -ENODEV; 523 } 524 525 if (num_queues > hw_q_num) { 526 rte_bbdev_log(ERR, 527 "Not enough queues for device %s! Requested: %u, available: %u", 528 dev->device->name, num_queues, hw_q_num); 529 return -EINVAL; 530 } 531 532 ring_size = FPGA_RING_MAX_SIZE * sizeof(struct fpga_dma_dec_desc); 533 534 /* Enforce 32 byte alignment */ 535 RTE_BUILD_BUG_ON((RTE_CACHE_LINE_SIZE % 32) != 0); 536 537 /* Allocate memory for SW descriptor rings */ 538 fpga_dev->sw_rings = rte_zmalloc_socket(dev->device->driver->name, 539 num_queues * ring_size, RTE_CACHE_LINE_SIZE, 540 socket_id); 541 if (fpga_dev->sw_rings == NULL) { 542 rte_bbdev_log(ERR, 543 "Failed to allocate memory for %s:%u sw_rings", 544 dev->device->driver->name, dev->data->dev_id); 545 return -ENOMEM; 546 } 547 548 fpga_dev->sw_rings_phys = rte_malloc_virt2iova(fpga_dev->sw_rings); 549 fpga_dev->sw_ring_size = ring_size; 550 fpga_dev->sw_ring_max_depth = FPGA_RING_MAX_SIZE; 551 552 /* Allocate memory for ring flush status */ 553 fpga_dev->flush_queue_status = rte_zmalloc_socket(NULL, 554 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, socket_id); 555 if (fpga_dev->flush_queue_status == NULL) { 556 rte_bbdev_log(ERR, 557 "Failed to allocate memory for %s:%u flush_queue_status", 558 dev->device->driver->name, dev->data->dev_id); 559 return -ENOMEM; 560 } 561 562 /* Set the flush status address registers */ 563 phys_addr = rte_malloc_virt2iova(fpga_dev->flush_queue_status); 564 565 address = FPGA_LTE_FEC_VFQ_FLUSH_STATUS_LW; 566 payload = (uint32_t)(phys_addr); 567 fpga_reg_write_32(fpga_dev->mmio_base, address, payload); 568 569 address = FPGA_LTE_FEC_VFQ_FLUSH_STATUS_HI; 570 payload = (uint32_t)(phys_addr >> 32); 571 fpga_reg_write_32(fpga_dev->mmio_base, address, payload); 572 573 return 0; 574 } 575 576 static int 577 fpga_dev_close(struct rte_bbdev *dev) 578 { 579 struct fpga_lte_fec_device *fpga_dev = dev->data->dev_private; 580 581 rte_free(fpga_dev->sw_rings); 582 rte_free(fpga_dev->flush_queue_status); 583 584 return 0; 585 } 586 587 static void 588 fpga_dev_info_get(struct rte_bbdev *dev, 589 struct rte_bbdev_driver_info *dev_info) 590 { 591 struct fpga_lte_fec_device *d = dev->data->dev_private; 592 uint32_t q_id = 0; 593 594 /* TODO RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN and numbers of buffers are set 595 * to temporary values as they are required by test application while 596 * validation phase. 597 */ 598 static const struct rte_bbdev_op_cap bbdev_capabilities[] = { 599 { 600 .type = RTE_BBDEV_OP_TURBO_DEC, 601 .cap.turbo_dec = { 602 .capability_flags = 603 RTE_BBDEV_TURBO_CRC_TYPE_24B | 604 RTE_BBDEV_TURBO_SUBBLOCK_DEINTERLEAVE | 605 RTE_BBDEV_TURBO_DEC_INTERRUPTS | 606 RTE_BBDEV_TURBO_NEG_LLR_1_BIT_IN | 607 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP, 608 .max_llr_modulus = INT8_MAX, 609 .num_buffers_src = 610 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS, 611 .num_buffers_hard_out = 612 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS, 613 .num_buffers_soft_out = 0 614 } 615 }, 616 { 617 .type = RTE_BBDEV_OP_TURBO_ENC, 618 .cap.turbo_enc = { 619 .capability_flags = 620 RTE_BBDEV_TURBO_CRC_24B_ATTACH | 621 RTE_BBDEV_TURBO_RATE_MATCH | 622 RTE_BBDEV_TURBO_ENC_INTERRUPTS, 623 .num_buffers_src = 624 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS, 625 .num_buffers_dst = 626 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS 627 } 628 }, 629 RTE_BBDEV_END_OF_CAPABILITIES_LIST() 630 }; 631 632 static struct rte_bbdev_queue_conf default_queue_conf; 633 default_queue_conf.socket = dev->data->socket_id; 634 default_queue_conf.queue_size = FPGA_RING_MAX_SIZE; 635 636 637 dev_info->driver_name = dev->device->driver->name; 638 dev_info->queue_size_lim = FPGA_RING_MAX_SIZE; 639 dev_info->hardware_accelerated = true; 640 dev_info->min_alignment = 64; 641 dev_info->default_queue_conf = default_queue_conf; 642 dev_info->capabilities = bbdev_capabilities; 643 dev_info->cpu_flag_reqs = NULL; 644 dev_info->data_endianness = RTE_LITTLE_ENDIAN; 645 dev_info->device_status = RTE_BBDEV_DEV_NOT_SUPPORTED; 646 647 /* Calculates number of queues assigned to device */ 648 dev_info->max_num_queues = 0; 649 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 650 uint32_t hw_q_id = fpga_reg_read_32(d->mmio_base, 651 FPGA_LTE_FEC_QUEUE_MAP + (q_id << 2)); 652 if (hw_q_id != FPGA_INVALID_HW_QUEUE_ID) 653 dev_info->max_num_queues++; 654 } 655 /* Expose number of queue per operation type */ 656 dev_info->num_queues[RTE_BBDEV_OP_NONE] = 0; 657 dev_info->num_queues[RTE_BBDEV_OP_TURBO_DEC] = dev_info->max_num_queues / 2; 658 dev_info->num_queues[RTE_BBDEV_OP_TURBO_ENC] = dev_info->max_num_queues / 2; 659 dev_info->num_queues[RTE_BBDEV_OP_LDPC_DEC] = 0; 660 dev_info->num_queues[RTE_BBDEV_OP_LDPC_ENC] = 0; 661 dev_info->num_queues[RTE_BBDEV_OP_FFT] = 0; 662 dev_info->num_queues[RTE_BBDEV_OP_MLDTS] = 0; 663 dev_info->queue_priority[RTE_BBDEV_OP_TURBO_DEC] = 1; 664 dev_info->queue_priority[RTE_BBDEV_OP_TURBO_ENC] = 1; 665 } 666 667 /** 668 * Find index of queue bound to current PF/VF which is unassigned. Return -1 669 * when there is no available queue 670 */ 671 static int 672 fpga_find_free_queue_idx(struct rte_bbdev *dev, 673 const struct rte_bbdev_queue_conf *conf) 674 { 675 struct fpga_lte_fec_device *d = dev->data->dev_private; 676 uint64_t q_idx; 677 uint8_t i = 0; 678 uint8_t range = FPGA_TOTAL_NUM_QUEUES >> 1; 679 680 if (conf->op_type == RTE_BBDEV_OP_TURBO_ENC) { 681 i = FPGA_NUM_DL_QUEUES; 682 range = FPGA_TOTAL_NUM_QUEUES; 683 } 684 685 for (; i < range; ++i) { 686 q_idx = 1ULL << i; 687 /* Check if index of queue is bound to current PF/VF */ 688 if (d->q_bound_bit_map & q_idx) 689 /* Check if found queue was not already assigned */ 690 if (!(d->q_assigned_bit_map & q_idx)) { 691 d->q_assigned_bit_map |= q_idx; 692 return i; 693 } 694 } 695 696 rte_bbdev_log(INFO, "Failed to find free queue on %s", dev->data->name); 697 698 return -1; 699 } 700 701 static int 702 fpga_queue_setup(struct rte_bbdev *dev, uint16_t queue_id, 703 const struct rte_bbdev_queue_conf *conf) 704 { 705 uint32_t address, ring_offset; 706 struct fpga_lte_fec_device *d = dev->data->dev_private; 707 struct fpga_queue *q; 708 int8_t q_idx; 709 710 /* Check if there is a free queue to assign */ 711 q_idx = fpga_find_free_queue_idx(dev, conf); 712 if (q_idx == -1) 713 return -1; 714 715 /* Allocate the queue data structure. */ 716 q = rte_zmalloc_socket(dev->device->driver->name, sizeof(*q), 717 RTE_CACHE_LINE_SIZE, conf->socket); 718 if (q == NULL) { 719 /* Mark queue as un-assigned */ 720 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 721 rte_bbdev_log(ERR, "Failed to allocate queue memory"); 722 return -ENOMEM; 723 } 724 725 q->d = d; 726 q->q_idx = q_idx; 727 728 /* Set ring_base_addr */ 729 q->ring_addr = RTE_PTR_ADD(d->sw_rings, (d->sw_ring_size * queue_id)); 730 q->ring_ctrl_reg.ring_base_addr = d->sw_rings_phys + 731 (d->sw_ring_size * queue_id); 732 733 /* Allocate memory for Completion Head variable*/ 734 q->ring_head_addr = rte_zmalloc_socket(dev->device->driver->name, 735 sizeof(uint64_t), RTE_CACHE_LINE_SIZE, conf->socket); 736 if (q->ring_head_addr == NULL) { 737 /* Mark queue as un-assigned */ 738 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 739 rte_free(q); 740 rte_bbdev_log(ERR, 741 "Failed to allocate memory for %s:%u completion_head", 742 dev->device->driver->name, dev->data->dev_id); 743 return -ENOMEM; 744 } 745 /* Set ring_head_addr */ 746 q->ring_ctrl_reg.ring_head_addr = 747 rte_malloc_virt2iova(q->ring_head_addr); 748 749 /* Clear shadow_completion_head */ 750 q->shadow_completion_head = 0; 751 752 /* Set ring_size */ 753 if (conf->queue_size > FPGA_RING_MAX_SIZE) { 754 /* Mark queue as un-assigned */ 755 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q_idx)); 756 rte_free(q->ring_head_addr); 757 rte_free(q); 758 rte_bbdev_log(ERR, 759 "Size of queue is too big %d (MAX: %d ) for %s:%u", 760 conf->queue_size, FPGA_RING_MAX_SIZE, 761 dev->device->driver->name, dev->data->dev_id); 762 return -EINVAL; 763 } 764 q->ring_ctrl_reg.ring_size = conf->queue_size; 765 766 /* Set Miscellaneous FPGA register*/ 767 /* Max iteration number for TTI mitigation - todo */ 768 q->ring_ctrl_reg.max_ul_dec = 0; 769 /* Enable max iteration number for TTI - todo */ 770 q->ring_ctrl_reg.max_ul_dec_en = 0; 771 772 /* Enable the ring */ 773 q->ring_ctrl_reg.enable = 1; 774 775 /* Set FPGA head_point and tail registers */ 776 q->ring_ctrl_reg.head_point = q->tail = 0; 777 778 /* Set FPGA shadow_tail register */ 779 q->ring_ctrl_reg.shadow_tail = q->tail; 780 781 /* Calculates the ring offset for found queue */ 782 ring_offset = FPGA_LTE_FEC_RING_CTRL_REGS + 783 (sizeof(struct fpga_ring_ctrl_reg) * q_idx); 784 785 /* Set FPGA Ring Control Registers */ 786 fpga_ring_reg_write(d->mmio_base, ring_offset, q->ring_ctrl_reg); 787 788 /* Store MMIO register of shadow_tail */ 789 address = ring_offset + FPGA_LTE_FEC_RING_SHADOW_TAIL; 790 q->shadow_tail_addr = RTE_PTR_ADD(d->mmio_base, address); 791 792 q->head_free_desc = q->tail; 793 794 /* Set wrap mask */ 795 q->sw_ring_wrap_mask = conf->queue_size - 1; 796 797 rte_bbdev_log_debug("Setup dev%u q%u: queue_idx=%u", 798 dev->data->dev_id, queue_id, q->q_idx); 799 800 dev->data->queues[queue_id].queue_private = q; 801 802 rte_bbdev_log_debug("BBDEV queue[%d] set up for FPGA queue[%d]", 803 queue_id, q_idx); 804 805 #ifdef RTE_LIBRTE_BBDEV_DEBUG 806 /* Read FPGA Ring Control Registers after configuration*/ 807 print_ring_reg_debug_info(d->mmio_base, ring_offset); 808 #endif 809 return 0; 810 } 811 812 static int 813 fpga_queue_release(struct rte_bbdev *dev, uint16_t queue_id) 814 { 815 struct fpga_lte_fec_device *d = dev->data->dev_private; 816 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 817 struct fpga_ring_ctrl_reg ring_reg; 818 uint32_t offset; 819 820 rte_bbdev_log_debug("FPGA Queue[%d] released", queue_id); 821 822 if (q != NULL) { 823 memset(&ring_reg, 0, sizeof(struct fpga_ring_ctrl_reg)); 824 offset = FPGA_LTE_FEC_RING_CTRL_REGS + 825 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 826 /* Disable queue */ 827 fpga_reg_write_8(d->mmio_base, 828 offset + FPGA_LTE_FEC_RING_ENABLE, 0x00); 829 /* Clear queue registers */ 830 fpga_ring_reg_write(d->mmio_base, offset, ring_reg); 831 832 /* Mark the Queue as un-assigned */ 833 d->q_assigned_bit_map &= (0xFFFFFFFF - (1ULL << q->q_idx)); 834 rte_free(q->ring_head_addr); 835 rte_free(q); 836 dev->data->queues[queue_id].queue_private = NULL; 837 } 838 839 return 0; 840 } 841 842 /* Function starts a device queue. */ 843 static int 844 fpga_queue_start(struct rte_bbdev *dev, uint16_t queue_id) 845 { 846 struct fpga_lte_fec_device *d = dev->data->dev_private; 847 #ifdef RTE_LIBRTE_BBDEV_DEBUG 848 if (d == NULL) { 849 rte_bbdev_log(ERR, "Invalid device pointer"); 850 return -1; 851 } 852 #endif 853 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 854 uint32_t offset = FPGA_LTE_FEC_RING_CTRL_REGS + 855 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 856 uint8_t enable = 0x01; 857 uint16_t zero = 0x0000; 858 859 /* Clear queue head and tail variables */ 860 q->tail = q->head_free_desc = 0; 861 862 /* Clear FPGA head_point and tail registers */ 863 fpga_reg_write_16(d->mmio_base, offset + FPGA_LTE_FEC_RING_HEAD_POINT, 864 zero); 865 fpga_reg_write_16(d->mmio_base, offset + FPGA_LTE_FEC_RING_SHADOW_TAIL, 866 zero); 867 868 /* Enable queue */ 869 fpga_reg_write_8(d->mmio_base, offset + FPGA_LTE_FEC_RING_ENABLE, 870 enable); 871 872 rte_bbdev_log_debug("FPGA Queue[%d] started", queue_id); 873 return 0; 874 } 875 876 /* Function stops a device queue. */ 877 static int 878 fpga_queue_stop(struct rte_bbdev *dev, uint16_t queue_id) 879 { 880 struct fpga_lte_fec_device *d = dev->data->dev_private; 881 #ifdef RTE_LIBRTE_BBDEV_DEBUG 882 if (d == NULL) { 883 rte_bbdev_log(ERR, "Invalid device pointer"); 884 return -1; 885 } 886 #endif 887 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 888 uint32_t offset = FPGA_LTE_FEC_RING_CTRL_REGS + 889 (sizeof(struct fpga_ring_ctrl_reg) * q->q_idx); 890 uint8_t payload = 0x01; 891 uint8_t counter = 0; 892 uint8_t timeout = FPGA_QUEUE_FLUSH_TIMEOUT_US / 893 FPGA_TIMEOUT_CHECK_INTERVAL; 894 895 /* Set flush_queue_en bit to trigger queue flushing */ 896 fpga_reg_write_8(d->mmio_base, 897 offset + FPGA_LTE_FEC_RING_FLUSH_QUEUE_EN, payload); 898 899 /** Check if queue flush is completed. 900 * FPGA will update the completion flag after queue flushing is 901 * completed. If completion flag is not updated within 1ms it is 902 * considered as a failure. 903 */ 904 while (!(*((volatile uint8_t *)d->flush_queue_status + q->q_idx) & payload)) { 905 if (counter > timeout) { 906 rte_bbdev_log(ERR, "FPGA Queue Flush failed for queue %d", 907 queue_id); 908 return -1; 909 } 910 usleep(FPGA_TIMEOUT_CHECK_INTERVAL); 911 counter++; 912 } 913 914 /* Disable queue */ 915 payload = 0x00; 916 fpga_reg_write_8(d->mmio_base, offset + FPGA_LTE_FEC_RING_ENABLE, 917 payload); 918 919 rte_bbdev_log_debug("FPGA Queue[%d] stopped", queue_id); 920 return 0; 921 } 922 923 static inline uint16_t 924 get_queue_id(struct rte_bbdev_data *data, uint8_t q_idx) 925 { 926 uint16_t queue_id; 927 928 for (queue_id = 0; queue_id < data->num_queues; ++queue_id) { 929 struct fpga_queue *q = data->queues[queue_id].queue_private; 930 if (q != NULL && q->q_idx == q_idx) 931 return queue_id; 932 } 933 934 return -1; 935 } 936 937 /* Interrupt handler triggered by FPGA dev for handling specific interrupt */ 938 static void 939 fpga_dev_interrupt_handler(void *cb_arg) 940 { 941 struct rte_bbdev *dev = cb_arg; 942 struct fpga_lte_fec_device *fpga_dev = dev->data->dev_private; 943 struct fpga_queue *q; 944 uint64_t ring_head; 945 uint64_t q_idx; 946 uint16_t queue_id; 947 uint8_t i; 948 949 /* Scan queue assigned to this device */ 950 for (i = 0; i < FPGA_TOTAL_NUM_QUEUES; ++i) { 951 q_idx = 1ULL << i; 952 if (fpga_dev->q_bound_bit_map & q_idx) { 953 queue_id = get_queue_id(dev->data, i); 954 if (queue_id == (uint16_t) -1) 955 continue; 956 957 /* Check if completion head was changed */ 958 q = dev->data->queues[queue_id].queue_private; 959 ring_head = *q->ring_head_addr; 960 if (q->shadow_completion_head != ring_head && 961 q->irq_enable == 1) { 962 q->shadow_completion_head = ring_head; 963 rte_bbdev_pmd_callback_process( 964 dev, 965 RTE_BBDEV_EVENT_DEQUEUE, 966 &queue_id); 967 } 968 } 969 } 970 } 971 972 static int 973 fpga_queue_intr_enable(struct rte_bbdev *dev, uint16_t queue_id) 974 { 975 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 976 977 if (!rte_intr_cap_multiple(dev->intr_handle)) 978 return -ENOTSUP; 979 980 q->irq_enable = 1; 981 982 return 0; 983 } 984 985 static int 986 fpga_queue_intr_disable(struct rte_bbdev *dev, uint16_t queue_id) 987 { 988 struct fpga_queue *q = dev->data->queues[queue_id].queue_private; 989 q->irq_enable = 0; 990 991 return 0; 992 } 993 994 static int 995 fpga_intr_enable(struct rte_bbdev *dev) 996 { 997 int ret; 998 uint8_t i; 999 1000 if (!rte_intr_cap_multiple(dev->intr_handle)) { 1001 rte_bbdev_log(ERR, "Multiple intr vector is not supported by FPGA (%s)", 1002 dev->data->name); 1003 return -ENOTSUP; 1004 } 1005 1006 /* Create event file descriptors for each of 64 queue. Event fds will be 1007 * mapped to FPGA IRQs in rte_intr_enable(). This is a 1:1 mapping where 1008 * the IRQ number is a direct translation to the queue number. 1009 * 1010 * 63 (FPGA_NUM_INTR_VEC) event fds are created as rte_intr_enable() 1011 * mapped the first IRQ to already created interrupt event file 1012 * descriptor (intr_handle->fd). 1013 */ 1014 if (rte_intr_efd_enable(dev->intr_handle, FPGA_NUM_INTR_VEC)) { 1015 rte_bbdev_log(ERR, "Failed to create fds for %u queues", 1016 dev->data->num_queues); 1017 return -1; 1018 } 1019 1020 /* TODO Each event file descriptor is overwritten by interrupt event 1021 * file descriptor. That descriptor is added to epoll observed list. 1022 * It ensures that callback function assigned to that descriptor will 1023 * invoked when any FPGA queue issues interrupt. 1024 */ 1025 for (i = 0; i < FPGA_NUM_INTR_VEC; ++i) { 1026 if (rte_intr_efds_index_set(dev->intr_handle, i, 1027 rte_intr_fd_get(dev->intr_handle))) 1028 return -rte_errno; 1029 } 1030 1031 if (rte_intr_vec_list_alloc(dev->intr_handle, "intr_vec", 1032 dev->data->num_queues)) { 1033 rte_bbdev_log(ERR, "Failed to allocate %u vectors", 1034 dev->data->num_queues); 1035 return -ENOMEM; 1036 } 1037 1038 ret = rte_intr_enable(dev->intr_handle); 1039 if (ret < 0) { 1040 rte_bbdev_log(ERR, 1041 "Couldn't enable interrupts for device: %s", 1042 dev->data->name); 1043 return ret; 1044 } 1045 1046 ret = rte_intr_callback_register(dev->intr_handle, 1047 fpga_dev_interrupt_handler, dev); 1048 if (ret < 0) { 1049 rte_bbdev_log(ERR, 1050 "Couldn't register interrupt callback for device: %s", 1051 dev->data->name); 1052 return ret; 1053 } 1054 1055 return 0; 1056 } 1057 1058 static const struct rte_bbdev_ops fpga_ops = { 1059 .setup_queues = fpga_setup_queues, 1060 .intr_enable = fpga_intr_enable, 1061 .close = fpga_dev_close, 1062 .info_get = fpga_dev_info_get, 1063 .queue_setup = fpga_queue_setup, 1064 .queue_stop = fpga_queue_stop, 1065 .queue_start = fpga_queue_start, 1066 .queue_release = fpga_queue_release, 1067 .queue_intr_enable = fpga_queue_intr_enable, 1068 .queue_intr_disable = fpga_queue_intr_disable 1069 }; 1070 1071 static inline void 1072 fpga_dma_enqueue(struct fpga_queue *q, uint16_t num_desc, 1073 struct rte_bbdev_stats *queue_stats) 1074 { 1075 uint64_t start_time = 0; 1076 queue_stats->acc_offload_cycles = 0; 1077 1078 /* Update tail and shadow_tail register */ 1079 q->tail = (q->tail + num_desc) & q->sw_ring_wrap_mask; 1080 1081 rte_wmb(); 1082 1083 /* Start time measurement for enqueue function offload. */ 1084 start_time = rte_rdtsc_precise(); 1085 mmio_write_16(q->shadow_tail_addr, q->tail); 1086 1087 rte_wmb(); 1088 queue_stats->acc_offload_cycles += rte_rdtsc_precise() - start_time; 1089 } 1090 1091 /* Calculates number of CBs in processed encoder TB based on 'r' and input 1092 * length. 1093 */ 1094 static inline uint8_t 1095 get_num_cbs_in_op_enc(struct rte_bbdev_op_turbo_enc *turbo_enc) 1096 { 1097 uint8_t c, c_neg, r, crc24_bits = 0; 1098 uint16_t k, k_neg, k_pos; 1099 uint8_t cbs_in_op = 0; 1100 int32_t length; 1101 1102 length = turbo_enc->input.length; 1103 r = turbo_enc->tb_params.r; 1104 c = turbo_enc->tb_params.c; 1105 c_neg = turbo_enc->tb_params.c_neg; 1106 k_neg = turbo_enc->tb_params.k_neg; 1107 k_pos = turbo_enc->tb_params.k_pos; 1108 crc24_bits = 24; 1109 while (length > 0 && r < c) { 1110 k = (r < c_neg) ? k_neg : k_pos; 1111 length -= (k - crc24_bits) >> 3; 1112 r++; 1113 cbs_in_op++; 1114 } 1115 1116 return cbs_in_op; 1117 } 1118 1119 /* Calculates number of CBs in processed decoder TB based on 'r' and input 1120 * length. 1121 */ 1122 static inline uint16_t 1123 get_num_cbs_in_op_dec(struct rte_bbdev_op_turbo_dec *turbo_dec) 1124 { 1125 uint8_t c, c_neg, r = 0; 1126 uint16_t kw, k, k_neg, k_pos, cbs_in_op = 0; 1127 int32_t length; 1128 1129 length = turbo_dec->input.length; 1130 r = turbo_dec->tb_params.r; 1131 c = turbo_dec->tb_params.c; 1132 c_neg = turbo_dec->tb_params.c_neg; 1133 k_neg = turbo_dec->tb_params.k_neg; 1134 k_pos = turbo_dec->tb_params.k_pos; 1135 while (length > 0 && r < c) { 1136 k = (r < c_neg) ? k_neg : k_pos; 1137 kw = RTE_ALIGN_CEIL(k + 4, 32) * 3; 1138 length -= kw; 1139 r++; 1140 cbs_in_op++; 1141 } 1142 1143 return cbs_in_op; 1144 } 1145 1146 /* Read flag value 0/1/ from bitmap */ 1147 static inline bool 1148 check_bit(uint32_t bitmap, uint32_t bitmask) 1149 { 1150 return bitmap & bitmask; 1151 } 1152 1153 /* Print an error if a descriptor error has occurred. 1154 * Return 0 on success, 1 on failure 1155 */ 1156 static inline int 1157 check_desc_error(uint32_t error_code) { 1158 switch (error_code) { 1159 case DESC_ERR_NO_ERR: 1160 return 0; 1161 case DESC_ERR_K_OUT_OF_RANGE: 1162 rte_bbdev_log(ERR, "Block_size_k is out of range (k<40 or k>6144)"); 1163 break; 1164 case DESC_ERR_K_NOT_NORMAL: 1165 rte_bbdev_log(ERR, "Block_size_k is not a normal value within normal range"); 1166 break; 1167 case DESC_ERR_KPAI_NOT_NORMAL: 1168 rte_bbdev_log(ERR, "Three_kpai is not a normal value for UL only"); 1169 break; 1170 case DESC_ERR_DESC_OFFSET_ERR: 1171 rte_bbdev_log(ERR, "Queue offset does not meet the expectation in the FPGA"); 1172 break; 1173 case (DESC_ERR_K_OUT_OF_RANGE | DESC_ERR_DESC_OFFSET_ERR): 1174 rte_bbdev_log(ERR, "Block_size_k is out of range (k<40 or k>6144) and queue offset error"); 1175 break; 1176 case (DESC_ERR_K_NOT_NORMAL | DESC_ERR_DESC_OFFSET_ERR): 1177 rte_bbdev_log(ERR, "Block_size_k is not a normal value within normal range and queue offset error"); 1178 break; 1179 case (DESC_ERR_KPAI_NOT_NORMAL | DESC_ERR_DESC_OFFSET_ERR): 1180 rte_bbdev_log(ERR, "Three_kpai is not a normal value for UL only and queue offset error"); 1181 break; 1182 case DESC_ERR_DESC_READ_FAIL: 1183 rte_bbdev_log(ERR, "Unsuccessful completion for descriptor read"); 1184 break; 1185 case DESC_ERR_DESC_READ_TIMEOUT: 1186 rte_bbdev_log(ERR, "Descriptor read time-out"); 1187 break; 1188 case DESC_ERR_DESC_READ_TLP_POISONED: 1189 rte_bbdev_log(ERR, "Descriptor read TLP poisoned"); 1190 break; 1191 case DESC_ERR_CB_READ_FAIL: 1192 rte_bbdev_log(ERR, "Unsuccessful completion for code block"); 1193 break; 1194 case DESC_ERR_CB_READ_TIMEOUT: 1195 rte_bbdev_log(ERR, "Code block read time-out"); 1196 break; 1197 case DESC_ERR_CB_READ_TLP_POISONED: 1198 rte_bbdev_log(ERR, "Code block read TLP poisoned"); 1199 break; 1200 default: 1201 rte_bbdev_log(ERR, "Descriptor error unknown error code %u", 1202 error_code); 1203 break; 1204 } 1205 return 1; 1206 } 1207 1208 /** 1209 * Set DMA descriptor for encode operation (1 Code Block) 1210 * 1211 * @param op 1212 * Pointer to a single encode operation. 1213 * @param desc 1214 * Pointer to DMA descriptor. 1215 * @param input 1216 * Pointer to pointer to input data which will be decoded. 1217 * @param k 1218 * K value (length of input in bits). 1219 * @param e 1220 * E value (length of output in bits). 1221 * @param ncb 1222 * Ncb value (size of the soft buffer). 1223 * @param out_length 1224 * Length of output buffer 1225 * @param in_offset 1226 * Input offset in rte_mbuf structure. It is used for calculating the point 1227 * where data is starting. 1228 * @param out_offset 1229 * Output offset in rte_mbuf structure. It is used for calculating the point 1230 * where hard output data will be stored. 1231 * @param cbs_in_op 1232 * Number of CBs contained in one operation. 1233 */ 1234 static inline int 1235 fpga_dma_desc_te_fill(struct rte_bbdev_enc_op *op, 1236 struct fpga_dma_enc_desc *desc, struct rte_mbuf *input, 1237 struct rte_mbuf *output, uint16_t k, uint16_t e, uint16_t ncb, 1238 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset, 1239 uint8_t cbs_in_op) 1240 1241 { 1242 /* reset */ 1243 desc->done = 0; 1244 desc->crc_en = check_bit(op->turbo_enc.op_flags, 1245 RTE_BBDEV_TURBO_CRC_24B_ATTACH); 1246 desc->bypass_rm = !check_bit(op->turbo_enc.op_flags, 1247 RTE_BBDEV_TURBO_RATE_MATCH); 1248 desc->k = k; 1249 desc->e = e; 1250 desc->ncb = ncb; 1251 desc->rv = op->turbo_enc.rv_index; 1252 desc->offset = desc_offset; 1253 /* Set inbound data buffer address */ 1254 desc->in_addr_hi = (uint32_t)( 1255 rte_pktmbuf_iova_offset(input, in_offset) >> 32); 1256 desc->in_addr_lw = (uint32_t)( 1257 rte_pktmbuf_iova_offset(input, in_offset)); 1258 1259 desc->out_addr_hi = (uint32_t)( 1260 rte_pktmbuf_iova_offset(output, out_offset) >> 32); 1261 desc->out_addr_lw = (uint32_t)( 1262 rte_pktmbuf_iova_offset(output, out_offset)); 1263 1264 /* Save software context needed for dequeue */ 1265 desc->op_addr = op; 1266 1267 /* Set total number of CBs in an op */ 1268 desc->cbs_in_op = cbs_in_op; 1269 1270 return 0; 1271 } 1272 1273 /** 1274 * Set DMA descriptor for encode operation (1 Code Block) 1275 * 1276 * @param op 1277 * Pointer to a single encode operation. 1278 * @param desc 1279 * Pointer to DMA descriptor. 1280 * @param input 1281 * Pointer to pointer to input data which will be decoded. 1282 * @param in_length 1283 * Length of an input. 1284 * @param k 1285 * K value (length of an output in bits). 1286 * @param in_offset 1287 * Input offset in rte_mbuf structure. It is used for calculating the point 1288 * where data is starting. 1289 * @param out_offset 1290 * Output offset in rte_mbuf structure. It is used for calculating the point 1291 * where hard output data will be stored. 1292 * @param cbs_in_op 1293 * Number of CBs contained in one operation. 1294 */ 1295 static inline int 1296 fpga_dma_desc_td_fill(struct rte_bbdev_dec_op *op, 1297 struct fpga_dma_dec_desc *desc, struct rte_mbuf *input, 1298 struct rte_mbuf *output, uint16_t in_length, uint16_t k, 1299 uint32_t in_offset, uint32_t out_offset, uint16_t desc_offset, 1300 uint8_t cbs_in_op) 1301 { 1302 /* reset */ 1303 desc->done = 0; 1304 /* Set inbound data buffer address */ 1305 desc->in_addr_hi = (uint32_t)( 1306 rte_pktmbuf_iova_offset(input, in_offset) >> 32); 1307 desc->in_addr_lw = (uint32_t)( 1308 rte_pktmbuf_iova_offset(input, in_offset)); 1309 desc->in_len = in_length; 1310 desc->k = k; 1311 desc->crc_type = !check_bit(op->turbo_dec.op_flags, 1312 RTE_BBDEV_TURBO_CRC_TYPE_24B); 1313 if ((op->turbo_dec.code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) 1314 && !check_bit(op->turbo_dec.op_flags, 1315 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP)) 1316 desc->drop_crc = 1; 1317 desc->max_iter = op->turbo_dec.iter_max * 2; 1318 desc->offset = desc_offset; 1319 desc->out_addr_hi = (uint32_t)( 1320 rte_pktmbuf_iova_offset(output, out_offset) >> 32); 1321 desc->out_addr_lw = (uint32_t)( 1322 rte_pktmbuf_iova_offset(output, out_offset)); 1323 1324 /* Save software context needed for dequeue */ 1325 desc->op_addr = op; 1326 1327 /* Set total number of CBs in an op */ 1328 desc->cbs_in_op = cbs_in_op; 1329 1330 return 0; 1331 } 1332 1333 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1334 /* Validates turbo encoder parameters */ 1335 static int 1336 validate_enc_op(struct rte_bbdev_enc_op *op) 1337 { 1338 struct rte_bbdev_op_turbo_enc *turbo_enc = &op->turbo_enc; 1339 struct rte_bbdev_op_enc_turbo_cb_params *cb = NULL; 1340 struct rte_bbdev_op_enc_turbo_tb_params *tb = NULL; 1341 uint16_t kw, kw_neg, kw_pos; 1342 1343 if (turbo_enc->input.length > 1344 RTE_BBDEV_TURBO_MAX_TB_SIZE >> 3) { 1345 rte_bbdev_log(ERR, "TB size (%u) is too big, max: %d", 1346 turbo_enc->input.length, 1347 RTE_BBDEV_TURBO_MAX_TB_SIZE); 1348 op->status = 1 << RTE_BBDEV_DATA_ERROR; 1349 return -1; 1350 } 1351 1352 if (op->mempool == NULL) { 1353 rte_bbdev_log(ERR, "Invalid mempool pointer"); 1354 return -1; 1355 } 1356 if (turbo_enc->input.data == NULL) { 1357 rte_bbdev_log(ERR, "Invalid input pointer"); 1358 return -1; 1359 } 1360 if (turbo_enc->output.data == NULL) { 1361 rte_bbdev_log(ERR, "Invalid output pointer"); 1362 return -1; 1363 } 1364 if (turbo_enc->rv_index > 3) { 1365 rte_bbdev_log(ERR, 1366 "rv_index (%u) is out of range 0 <= value <= 3", 1367 turbo_enc->rv_index); 1368 return -1; 1369 } 1370 if (turbo_enc->code_block_mode != RTE_BBDEV_TRANSPORT_BLOCK && 1371 turbo_enc->code_block_mode != RTE_BBDEV_CODE_BLOCK) { 1372 rte_bbdev_log(ERR, 1373 "code_block_mode (%u) is out of range 0 <= value <= 1", 1374 turbo_enc->code_block_mode); 1375 return -1; 1376 } 1377 1378 if (turbo_enc->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1379 tb = &turbo_enc->tb_params; 1380 if ((tb->k_neg < RTE_BBDEV_TURBO_MIN_CB_SIZE 1381 || tb->k_neg > RTE_BBDEV_TURBO_MAX_CB_SIZE) 1382 && tb->c_neg > 0) { 1383 rte_bbdev_log(ERR, 1384 "k_neg (%u) is out of range %u <= value <= %u", 1385 tb->k_neg, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1386 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1387 return -1; 1388 } 1389 if (tb->k_pos < RTE_BBDEV_TURBO_MIN_CB_SIZE 1390 || tb->k_pos > RTE_BBDEV_TURBO_MAX_CB_SIZE) { 1391 rte_bbdev_log(ERR, 1392 "k_pos (%u) is out of range %u <= value <= %u", 1393 tb->k_pos, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1394 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1395 return -1; 1396 } 1397 if (tb->c_neg > (RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1)) 1398 rte_bbdev_log(ERR, 1399 "c_neg (%u) is out of range 0 <= value <= %u", 1400 tb->c_neg, 1401 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1); 1402 if (tb->c < 1 || tb->c > RTE_BBDEV_TURBO_MAX_CODE_BLOCKS) { 1403 rte_bbdev_log(ERR, 1404 "c (%u) is out of range 1 <= value <= %u", 1405 tb->c, RTE_BBDEV_TURBO_MAX_CODE_BLOCKS); 1406 return -1; 1407 } 1408 if (tb->cab > tb->c) { 1409 rte_bbdev_log(ERR, 1410 "cab (%u) is greater than c (%u)", 1411 tb->cab, tb->c); 1412 return -1; 1413 } 1414 if ((tb->ea < RTE_BBDEV_TURBO_MIN_CB_SIZE || (tb->ea % 2)) 1415 && tb->r < tb->cab) { 1416 rte_bbdev_log(ERR, 1417 "ea (%u) is less than %u or it is not even", 1418 tb->ea, RTE_BBDEV_TURBO_MIN_CB_SIZE); 1419 return -1; 1420 } 1421 if ((tb->eb < RTE_BBDEV_TURBO_MIN_CB_SIZE || (tb->eb % 2)) 1422 && tb->c > tb->cab) { 1423 rte_bbdev_log(ERR, 1424 "eb (%u) is less than %u or it is not even", 1425 tb->eb, RTE_BBDEV_TURBO_MIN_CB_SIZE); 1426 return -1; 1427 } 1428 1429 kw_neg = 3 * RTE_ALIGN_CEIL(tb->k_neg + 4, 1430 RTE_BBDEV_TURBO_C_SUBBLOCK); 1431 if (tb->ncb_neg < tb->k_neg || tb->ncb_neg > kw_neg) { 1432 rte_bbdev_log(ERR, 1433 "ncb_neg (%u) is out of range (%u) k_neg <= value <= (%u) kw_neg", 1434 tb->ncb_neg, tb->k_neg, kw_neg); 1435 return -1; 1436 } 1437 1438 kw_pos = 3 * RTE_ALIGN_CEIL(tb->k_pos + 4, 1439 RTE_BBDEV_TURBO_C_SUBBLOCK); 1440 if (tb->ncb_pos < tb->k_pos || tb->ncb_pos > kw_pos) { 1441 rte_bbdev_log(ERR, 1442 "ncb_pos (%u) is out of range (%u) k_pos <= value <= (%u) kw_pos", 1443 tb->ncb_pos, tb->k_pos, kw_pos); 1444 return -1; 1445 } 1446 if (tb->r > (tb->c - 1)) { 1447 rte_bbdev_log(ERR, 1448 "r (%u) is greater than c - 1 (%u)", 1449 tb->r, tb->c - 1); 1450 return -1; 1451 } 1452 } else { 1453 cb = &turbo_enc->cb_params; 1454 if (cb->k < RTE_BBDEV_TURBO_MIN_CB_SIZE 1455 || cb->k > RTE_BBDEV_TURBO_MAX_CB_SIZE) { 1456 rte_bbdev_log(ERR, 1457 "k (%u) is out of range %u <= value <= %u", 1458 cb->k, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1459 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1460 return -1; 1461 } 1462 1463 if (cb->e < RTE_BBDEV_TURBO_MIN_CB_SIZE || (cb->e % 2)) { 1464 rte_bbdev_log(ERR, 1465 "e (%u) is less than %u or it is not even", 1466 cb->e, RTE_BBDEV_TURBO_MIN_CB_SIZE); 1467 return -1; 1468 } 1469 1470 kw = RTE_ALIGN_CEIL(cb->k + 4, RTE_BBDEV_TURBO_C_SUBBLOCK) * 3; 1471 if (cb->ncb < cb->k || cb->ncb > kw) { 1472 rte_bbdev_log(ERR, 1473 "ncb (%u) is out of range (%u) k <= value <= (%u) kw", 1474 cb->ncb, cb->k, kw); 1475 return -1; 1476 } 1477 } 1478 1479 return 0; 1480 } 1481 #endif 1482 1483 static inline char * 1484 mbuf_append(struct rte_mbuf *m_head, struct rte_mbuf *m, uint16_t len) 1485 { 1486 if (unlikely(len > rte_pktmbuf_tailroom(m))) 1487 return NULL; 1488 1489 char *tail = (char *)m->buf_addr + m->data_off + m->data_len; 1490 m->data_len = (uint16_t)(m->data_len + len); 1491 m_head->pkt_len = (m_head->pkt_len + len); 1492 return tail; 1493 } 1494 1495 static inline int 1496 enqueue_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op *op, 1497 uint16_t desc_offset) 1498 { 1499 union fpga_dma_desc *desc; 1500 struct rte_mbuf *input; 1501 struct rte_mbuf *output; 1502 int ret; 1503 uint16_t k, e, ncb, ring_offset; 1504 uint32_t total_left, in_length, out_length, in_offset, out_offset; 1505 1506 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1507 /* Validate op structure */ 1508 if (validate_enc_op(op) == -1) { 1509 rte_bbdev_log(ERR, "Turbo encoder validation failed"); 1510 return -EINVAL; 1511 } 1512 #endif 1513 1514 input = op->turbo_enc.input.data; 1515 output = op->turbo_enc.output.data; 1516 in_offset = op->turbo_enc.input.offset; 1517 out_offset = op->turbo_enc.output.offset; 1518 total_left = op->turbo_enc.input.length; 1519 k = op->turbo_enc.cb_params.k; 1520 e = op->turbo_enc.cb_params.e; 1521 ncb = op->turbo_enc.cb_params.ncb; 1522 1523 if (check_bit(op->turbo_enc.op_flags, RTE_BBDEV_TURBO_CRC_24B_ATTACH)) 1524 in_length = ((k - 24) >> 3); 1525 else 1526 in_length = k >> 3; 1527 1528 if (check_bit(op->turbo_enc.op_flags, RTE_BBDEV_TURBO_RATE_MATCH)) 1529 out_length = (e + 7) >> 3; 1530 else 1531 out_length = (k >> 3) * 3 + 2; 1532 1533 mbuf_append(output, output, out_length); 1534 1535 /* Offset into the ring */ 1536 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1537 /* Setup DMA Descriptor */ 1538 desc = q->ring_addr + ring_offset; 1539 1540 ret = fpga_dma_desc_te_fill(op, &desc->enc_req, input, output, k, e, 1541 ncb, in_offset, out_offset, ring_offset, 1); 1542 if (unlikely(ret < 0)) 1543 return ret; 1544 1545 /* Update lengths */ 1546 total_left -= in_length; 1547 op->turbo_enc.output.length += out_length; 1548 1549 if (total_left > 0) { 1550 rte_bbdev_log(ERR, 1551 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u", 1552 total_left, in_length); 1553 return -1; 1554 } 1555 1556 return 1; 1557 } 1558 1559 static inline int 1560 enqueue_enc_one_op_tb(struct fpga_queue *q, struct rte_bbdev_enc_op *op, 1561 uint16_t desc_offset, uint8_t cbs_in_op) 1562 { 1563 union fpga_dma_desc *desc; 1564 struct rte_mbuf *input, *output_head, *output; 1565 int ret; 1566 uint8_t r, c, crc24_bits = 0; 1567 uint16_t k, e, ncb, ring_offset; 1568 uint32_t mbuf_total_left, in_length, out_length, in_offset, out_offset; 1569 uint32_t seg_total_left; 1570 uint16_t current_enqueued_cbs = 0; 1571 1572 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1573 /* Validate op structure */ 1574 if (validate_enc_op(op) == -1) { 1575 rte_bbdev_log(ERR, "Turbo encoder validation failed"); 1576 return -EINVAL; 1577 } 1578 #endif 1579 1580 input = op->turbo_enc.input.data; 1581 output_head = output = op->turbo_enc.output.data; 1582 in_offset = op->turbo_enc.input.offset; 1583 out_offset = op->turbo_enc.output.offset; 1584 mbuf_total_left = op->turbo_enc.input.length; 1585 1586 c = op->turbo_enc.tb_params.c; 1587 r = op->turbo_enc.tb_params.r; 1588 1589 if (check_bit(op->turbo_enc.op_flags, RTE_BBDEV_TURBO_CRC_24B_ATTACH)) 1590 crc24_bits = 24; 1591 1592 while (mbuf_total_left > 0 && r < c && input != NULL) { 1593 seg_total_left = rte_pktmbuf_data_len(input) - in_offset; 1594 1595 e = (r < op->turbo_enc.tb_params.cab) ? 1596 op->turbo_enc.tb_params.ea : 1597 op->turbo_enc.tb_params.eb; 1598 k = (r < op->turbo_enc.tb_params.c_neg) ? 1599 op->turbo_enc.tb_params.k_neg : 1600 op->turbo_enc.tb_params.k_pos; 1601 ncb = (r < op->turbo_enc.tb_params.c_neg) ? 1602 op->turbo_enc.tb_params.ncb_neg : 1603 op->turbo_enc.tb_params.ncb_pos; 1604 1605 in_length = ((k - crc24_bits) >> 3); 1606 1607 if (check_bit(op->turbo_enc.op_flags, 1608 RTE_BBDEV_TURBO_RATE_MATCH)) 1609 out_length = (e + 7) >> 3; 1610 else 1611 out_length = (k >> 3) * 3 + 2; 1612 1613 mbuf_append(output_head, output, out_length); 1614 1615 /* Setup DMA Descriptor */ 1616 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1617 desc = q->ring_addr + ring_offset; 1618 ret = fpga_dma_desc_te_fill(op, &desc->enc_req, input, output, 1619 k, e, ncb, in_offset, out_offset, ring_offset, 1620 cbs_in_op); 1621 if (unlikely(ret < 0)) 1622 return ret; 1623 1624 rte_bbdev_log_debug("DMA request desc %p", desc); 1625 1626 /* Update lengths */ 1627 op->turbo_enc.output.length += out_length; 1628 mbuf_total_left -= in_length; 1629 1630 /* Update offsets */ 1631 if (seg_total_left == in_length) { 1632 /* Go to the next mbuf */ 1633 input = input->next; 1634 output = output->next; 1635 in_offset = 0; 1636 out_offset = 0; 1637 } else { 1638 in_offset += in_length; 1639 out_offset += out_length; 1640 } 1641 1642 r++; 1643 desc_offset++; 1644 current_enqueued_cbs++; 1645 } 1646 1647 if (mbuf_total_left > 0) { 1648 rte_bbdev_log(ERR, 1649 "Some date still left for processing: mbuf_total_left = %u", 1650 mbuf_total_left); 1651 return -1; 1652 } 1653 1654 return current_enqueued_cbs; 1655 } 1656 1657 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1658 /* Validates turbo decoder parameters */ 1659 static int 1660 validate_dec_op(struct rte_bbdev_dec_op *op) 1661 { 1662 struct rte_bbdev_op_turbo_dec *turbo_dec = &op->turbo_dec; 1663 struct rte_bbdev_op_dec_turbo_cb_params *cb = NULL; 1664 struct rte_bbdev_op_dec_turbo_tb_params *tb = NULL; 1665 1666 if (op->mempool == NULL) { 1667 rte_bbdev_log(ERR, "Invalid mempool pointer"); 1668 return -1; 1669 } 1670 if (turbo_dec->input.data == NULL) { 1671 rte_bbdev_log(ERR, "Invalid input pointer"); 1672 return -1; 1673 } 1674 if (turbo_dec->hard_output.data == NULL) { 1675 rte_bbdev_log(ERR, "Invalid hard_output pointer"); 1676 return -1; 1677 } 1678 if (turbo_dec->rv_index > 3) { 1679 rte_bbdev_log(ERR, 1680 "rv_index (%u) is out of range 0 <= value <= 3", 1681 turbo_dec->rv_index); 1682 return -1; 1683 } 1684 if (turbo_dec->iter_min < 1) { 1685 rte_bbdev_log(ERR, 1686 "iter_min (%u) is less than 1", 1687 turbo_dec->iter_min); 1688 return -1; 1689 } 1690 if (turbo_dec->iter_max <= 2) { 1691 rte_bbdev_log(ERR, 1692 "iter_max (%u) is less than or equal to 2", 1693 turbo_dec->iter_max); 1694 return -1; 1695 } 1696 if (turbo_dec->iter_min > turbo_dec->iter_max) { 1697 rte_bbdev_log(ERR, 1698 "iter_min (%u) is greater than iter_max (%u)", 1699 turbo_dec->iter_min, turbo_dec->iter_max); 1700 return -1; 1701 } 1702 if (turbo_dec->code_block_mode != RTE_BBDEV_TRANSPORT_BLOCK && 1703 turbo_dec->code_block_mode != RTE_BBDEV_CODE_BLOCK) { 1704 rte_bbdev_log(ERR, 1705 "code_block_mode (%u) is out of range 0 <= value <= 1", 1706 turbo_dec->code_block_mode); 1707 return -1; 1708 } 1709 1710 if (turbo_dec->code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) { 1711 1712 if ((turbo_dec->op_flags & 1713 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP) && 1714 !(turbo_dec->op_flags & RTE_BBDEV_TURBO_CRC_TYPE_24B)) { 1715 rte_bbdev_log(ERR, 1716 "RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP should accompany RTE_BBDEV_TURBO_CRC_TYPE_24B"); 1717 return -1; 1718 } 1719 1720 tb = &turbo_dec->tb_params; 1721 if ((tb->k_neg < RTE_BBDEV_TURBO_MIN_CB_SIZE 1722 || tb->k_neg > RTE_BBDEV_TURBO_MAX_CB_SIZE) 1723 && tb->c_neg > 0) { 1724 rte_bbdev_log(ERR, 1725 "k_neg (%u) is out of range %u <= value <= %u", 1726 tb->k_neg, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1727 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1728 return -1; 1729 } 1730 if ((tb->k_pos < RTE_BBDEV_TURBO_MIN_CB_SIZE 1731 || tb->k_pos > RTE_BBDEV_TURBO_MAX_CB_SIZE) 1732 && tb->c > tb->c_neg) { 1733 rte_bbdev_log(ERR, 1734 "k_pos (%u) is out of range %u <= value <= %u", 1735 tb->k_pos, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1736 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1737 return -1; 1738 } 1739 if (tb->c_neg > (RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1)) 1740 rte_bbdev_log(ERR, 1741 "c_neg (%u) is out of range 0 <= value <= %u", 1742 tb->c_neg, 1743 RTE_BBDEV_TURBO_MAX_CODE_BLOCKS - 1); 1744 if (tb->c < 1 || tb->c > RTE_BBDEV_TURBO_MAX_CODE_BLOCKS) { 1745 rte_bbdev_log(ERR, 1746 "c (%u) is out of range 1 <= value <= %u", 1747 tb->c, RTE_BBDEV_TURBO_MAX_CODE_BLOCKS); 1748 return -1; 1749 } 1750 if (tb->cab > tb->c) { 1751 rte_bbdev_log(ERR, 1752 "cab (%u) is greater than c (%u)", 1753 tb->cab, tb->c); 1754 return -1; 1755 } 1756 } else { 1757 1758 if (turbo_dec->op_flags & RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP) { 1759 rte_bbdev_log(ERR, 1760 "RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP is invalid in CB-mode"); 1761 return -1; 1762 } 1763 1764 cb = &turbo_dec->cb_params; 1765 if (cb->k < RTE_BBDEV_TURBO_MIN_CB_SIZE 1766 || cb->k > RTE_BBDEV_TURBO_MAX_CB_SIZE) { 1767 rte_bbdev_log(ERR, 1768 "k (%u) is out of range %u <= value <= %u", 1769 cb->k, RTE_BBDEV_TURBO_MIN_CB_SIZE, 1770 RTE_BBDEV_TURBO_MAX_CB_SIZE); 1771 return -1; 1772 } 1773 } 1774 1775 return 0; 1776 } 1777 #endif 1778 1779 static inline int 1780 enqueue_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op *op, 1781 uint16_t desc_offset) 1782 { 1783 union fpga_dma_desc *desc; 1784 struct rte_mbuf *input; 1785 struct rte_mbuf *output; 1786 int ret; 1787 uint16_t k, kw, ring_offset; 1788 uint32_t total_left, in_length, out_length, in_offset, out_offset; 1789 1790 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1791 /* Validate op structure */ 1792 if (validate_dec_op(op) == -1) { 1793 rte_bbdev_log(ERR, "Turbo decoder validation failed"); 1794 return -EINVAL; 1795 } 1796 #endif 1797 1798 input = op->turbo_dec.input.data; 1799 output = op->turbo_dec.hard_output.data; 1800 total_left = op->turbo_dec.input.length; 1801 in_offset = op->turbo_dec.input.offset; 1802 out_offset = op->turbo_dec.hard_output.offset; 1803 1804 k = op->turbo_dec.cb_params.k; 1805 kw = RTE_ALIGN_CEIL(k + 4, 32) * 3; 1806 in_length = kw; 1807 out_length = k >> 3; 1808 1809 mbuf_append(output, output, out_length); 1810 1811 /* Setup DMA Descriptor */ 1812 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1813 desc = q->ring_addr + ring_offset; 1814 ret = fpga_dma_desc_td_fill(op, &desc->dec_req, input, output, 1815 in_length, k, in_offset, out_offset, ring_offset, 1); 1816 if (unlikely(ret < 0)) 1817 return ret; 1818 1819 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1820 print_dma_dec_desc_debug_info(desc); 1821 #endif 1822 1823 /* Update lengths */ 1824 total_left -= in_length; 1825 op->turbo_dec.hard_output.length += out_length; 1826 1827 if (total_left > 0) { 1828 rte_bbdev_log(ERR, 1829 "Mismatch between mbuf length and included CB sizes: mbuf len %u, cb len %u", 1830 total_left, in_length); 1831 return -1; 1832 } 1833 1834 return 1; 1835 } 1836 1837 1838 static inline int 1839 enqueue_dec_one_op_tb(struct fpga_queue *q, struct rte_bbdev_dec_op *op, 1840 uint16_t desc_offset, uint8_t cbs_in_op) 1841 { 1842 union fpga_dma_desc *desc; 1843 struct rte_mbuf *input, *output_head, *output; 1844 int ret; 1845 uint8_t r, c; 1846 uint16_t k, kw, in_length, out_length, ring_offset; 1847 uint32_t mbuf_total_left, seg_total_left, in_offset, out_offset; 1848 uint16_t current_enqueued_cbs = 0; 1849 uint16_t crc24_overlap = 0; 1850 1851 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1852 /* Validate op structure */ 1853 if (validate_dec_op(op) == -1) { 1854 rte_bbdev_log(ERR, "Turbo decoder validation failed"); 1855 return -EINVAL; 1856 } 1857 #endif 1858 1859 input = op->turbo_dec.input.data; 1860 output_head = output = op->turbo_dec.hard_output.data; 1861 mbuf_total_left = op->turbo_dec.input.length; 1862 in_offset = op->turbo_dec.input.offset; 1863 out_offset = op->turbo_dec.hard_output.offset; 1864 1865 if (!check_bit(op->turbo_dec.op_flags, 1866 RTE_BBDEV_TURBO_DEC_TB_CRC_24B_KEEP)) 1867 crc24_overlap = 24; 1868 1869 c = op->turbo_dec.tb_params.c; 1870 r = op->turbo_dec.tb_params.r; 1871 1872 while (mbuf_total_left > 0 && r < c && input != NULL) { 1873 seg_total_left = rte_pktmbuf_data_len(input) - in_offset; 1874 k = (r < op->turbo_dec.tb_params.c_neg) ? 1875 op->turbo_dec.tb_params.k_neg : 1876 op->turbo_dec.tb_params.k_pos; 1877 kw = RTE_ALIGN_CEIL(k + 4, 32) * 3; 1878 1879 in_length = kw; 1880 out_length = (k - crc24_overlap) >> 3; 1881 1882 mbuf_append(output_head, output, out_length); 1883 1884 if (seg_total_left < in_length) { 1885 rte_bbdev_log(ERR, 1886 "Partial CB found in a TB. FPGA Driver doesn't support scatter-gather operations!"); 1887 return -1; 1888 } 1889 1890 /* Setup DMA Descriptor */ 1891 ring_offset = ((q->tail + desc_offset) & q->sw_ring_wrap_mask); 1892 desc = q->ring_addr + ring_offset; 1893 ret = fpga_dma_desc_td_fill(op, &desc->dec_req, input, output, 1894 in_length, k, in_offset, out_offset, 1895 ring_offset, cbs_in_op); 1896 if (unlikely(ret < 0)) 1897 return ret; 1898 1899 /* Update lengths */ 1900 ret = rte_pktmbuf_trim(op->turbo_dec.hard_output.data, 1901 (crc24_overlap >> 3)); 1902 #ifdef RTE_LIBRTE_BBDEV_DEBUG 1903 if (ret < 0) { 1904 rte_bbdev_log(ERR, 1905 "The length to remove is greater than the length of the last segment"); 1906 return -EINVAL; 1907 } 1908 #endif 1909 op->turbo_dec.hard_output.length += out_length; 1910 mbuf_total_left -= in_length; 1911 1912 /* Update offsets */ 1913 if (seg_total_left == in_length) { 1914 /* Go to the next mbuf */ 1915 input = input->next; 1916 output = output->next; 1917 in_offset = 0; 1918 out_offset = 0; 1919 } else { 1920 in_offset += in_length; 1921 out_offset += out_length; 1922 } 1923 1924 r++; 1925 desc_offset++; 1926 current_enqueued_cbs++; 1927 } 1928 1929 if (mbuf_total_left > 0) { 1930 rte_bbdev_log(ERR, 1931 "Some date still left for processing: mbuf_total_left = %u", 1932 mbuf_total_left); 1933 return -1; 1934 } 1935 1936 return current_enqueued_cbs; 1937 } 1938 1939 static uint16_t 1940 fpga_enqueue_enc(struct rte_bbdev_queue_data *q_data, 1941 struct rte_bbdev_enc_op **ops, uint16_t num) 1942 { 1943 uint8_t cbs_in_op; 1944 uint16_t i, total_enqueued_cbs = 0; 1945 int32_t avail; 1946 int enqueued_cbs; 1947 struct fpga_queue *q = q_data->queue_private; 1948 union fpga_dma_desc *desc; 1949 1950 /* Check if queue is not full */ 1951 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) == 1952 q->head_free_desc)) 1953 return 0; 1954 1955 /* Calculates available space */ 1956 avail = (q->head_free_desc > q->tail) ? 1957 q->head_free_desc - q->tail - 1 : 1958 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1; 1959 1960 for (i = 0; i < num; ++i) { 1961 if (ops[i]->turbo_enc.code_block_mode == 1962 RTE_BBDEV_TRANSPORT_BLOCK) { 1963 cbs_in_op = get_num_cbs_in_op_enc(&ops[i]->turbo_enc); 1964 /* Check if there is available space for further 1965 * processing 1966 */ 1967 if (unlikely(avail - cbs_in_op < 0)) 1968 break; 1969 avail -= cbs_in_op; 1970 enqueued_cbs = enqueue_enc_one_op_tb(q, ops[i], 1971 total_enqueued_cbs, cbs_in_op); 1972 } else { 1973 /* Check if there is available space for further 1974 * processing 1975 */ 1976 if (unlikely(avail - 1 < 0)) 1977 break; 1978 avail -= 1; 1979 enqueued_cbs = enqueue_enc_one_op_cb(q, ops[i], 1980 total_enqueued_cbs); 1981 } 1982 1983 if (enqueued_cbs < 0) 1984 break; 1985 1986 total_enqueued_cbs += enqueued_cbs; 1987 1988 rte_bbdev_log_debug("enqueuing enc ops [%d/%d] | head %d | tail %d", 1989 total_enqueued_cbs, num, 1990 q->head_free_desc, q->tail); 1991 } 1992 1993 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt 1994 * only when all previous CBs were already processed. 1995 */ 1996 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1) 1997 & q->sw_ring_wrap_mask); 1998 desc->enc_req.irq_en = q->irq_enable; 1999 2000 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats); 2001 2002 /* Update stats */ 2003 q_data->queue_stats.enqueued_count += i; 2004 q_data->queue_stats.enqueue_err_count += num - i; 2005 2006 return i; 2007 } 2008 2009 static uint16_t 2010 fpga_enqueue_dec(struct rte_bbdev_queue_data *q_data, 2011 struct rte_bbdev_dec_op **ops, uint16_t num) 2012 { 2013 uint8_t cbs_in_op; 2014 uint16_t i, total_enqueued_cbs = 0; 2015 int32_t avail; 2016 int enqueued_cbs; 2017 struct fpga_queue *q = q_data->queue_private; 2018 union fpga_dma_desc *desc; 2019 2020 /* Check if queue is not full */ 2021 if (unlikely(((q->tail + 1) & q->sw_ring_wrap_mask) == 2022 q->head_free_desc)) 2023 return 0; 2024 2025 /* Calculates available space */ 2026 avail = (q->head_free_desc > q->tail) ? 2027 q->head_free_desc - q->tail - 1 : 2028 q->ring_ctrl_reg.ring_size + q->head_free_desc - q->tail - 1; 2029 2030 for (i = 0; i < num; ++i) { 2031 if (ops[i]->turbo_dec.code_block_mode == 2032 RTE_BBDEV_TRANSPORT_BLOCK) { 2033 cbs_in_op = get_num_cbs_in_op_dec(&ops[i]->turbo_dec); 2034 /* Check if there is available space for further 2035 * processing 2036 */ 2037 if (unlikely(avail - cbs_in_op < 0)) 2038 break; 2039 avail -= cbs_in_op; 2040 enqueued_cbs = enqueue_dec_one_op_tb(q, ops[i], 2041 total_enqueued_cbs, cbs_in_op); 2042 } else { 2043 /* Check if there is available space for further 2044 * processing 2045 */ 2046 if (unlikely(avail - 1 < 0)) 2047 break; 2048 avail -= 1; 2049 enqueued_cbs = enqueue_dec_one_op_cb(q, ops[i], 2050 total_enqueued_cbs); 2051 } 2052 2053 if (enqueued_cbs < 0) 2054 break; 2055 2056 total_enqueued_cbs += enqueued_cbs; 2057 2058 rte_bbdev_log_debug("enqueuing dec ops [%d/%d] | head %d | tail %d", 2059 total_enqueued_cbs, num, 2060 q->head_free_desc, q->tail); 2061 } 2062 2063 /* Set interrupt bit for last CB in enqueued ops. FPGA issues interrupt 2064 * only when all previous CBs were already processed. 2065 */ 2066 desc = q->ring_addr + ((q->tail + total_enqueued_cbs - 1) 2067 & q->sw_ring_wrap_mask); 2068 desc->dec_req.irq_en = q->irq_enable; 2069 2070 fpga_dma_enqueue(q, total_enqueued_cbs, &q_data->queue_stats); 2071 2072 /* Update stats */ 2073 q_data->queue_stats.enqueued_count += i; 2074 q_data->queue_stats.enqueue_err_count += num - i; 2075 2076 return i; 2077 } 2078 2079 static inline int 2080 dequeue_enc_one_op_cb(struct fpga_queue *q, struct rte_bbdev_enc_op **op, 2081 uint16_t desc_offset) 2082 { 2083 union fpga_dma_desc *desc; 2084 int desc_error = 0; 2085 2086 /* Set current desc */ 2087 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2088 & q->sw_ring_wrap_mask); 2089 2090 /*check if done */ 2091 if (desc->enc_req.done == 0) 2092 return -1; 2093 2094 /* make sure the response is read atomically */ 2095 rte_smp_rmb(); 2096 2097 rte_bbdev_log_debug("DMA response desc %p", desc); 2098 2099 *op = desc->enc_req.op_addr; 2100 /* Check the descriptor error field, return 1 on error */ 2101 desc_error = check_desc_error(desc->enc_req.error); 2102 (*op)->status = desc_error << RTE_BBDEV_DATA_ERROR; 2103 2104 return 1; 2105 } 2106 2107 static inline int 2108 dequeue_enc_one_op_tb(struct fpga_queue *q, struct rte_bbdev_enc_op **op, 2109 uint16_t desc_offset) 2110 { 2111 union fpga_dma_desc *desc; 2112 uint8_t cbs_in_op, cb_idx; 2113 int desc_error = 0; 2114 int status = 0; 2115 2116 /* Set descriptor */ 2117 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2118 & q->sw_ring_wrap_mask); 2119 2120 /* Verify if done bit is set */ 2121 if (desc->enc_req.done == 0) 2122 return -1; 2123 2124 /* Make sure the response is read atomically */ 2125 rte_smp_rmb(); 2126 2127 /* Verify if done bit in all CBs is set */ 2128 cbs_in_op = desc->enc_req.cbs_in_op; 2129 for (cb_idx = 1; cb_idx < cbs_in_op; ++cb_idx) { 2130 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2131 cb_idx) & q->sw_ring_wrap_mask); 2132 if (desc->enc_req.done == 0) 2133 return -1; 2134 } 2135 2136 /* Make sure the response is read atomically */ 2137 rte_smp_rmb(); 2138 2139 for (cb_idx = 0; cb_idx < cbs_in_op; ++cb_idx) { 2140 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2141 cb_idx) & q->sw_ring_wrap_mask); 2142 /* Check the descriptor error field, return 1 on error */ 2143 desc_error = check_desc_error(desc->enc_req.error); 2144 status |= desc_error << RTE_BBDEV_DATA_ERROR; 2145 rte_bbdev_log_debug("DMA response desc %p", desc); 2146 } 2147 2148 *op = desc->enc_req.op_addr; 2149 (*op)->status = status; 2150 return cbs_in_op; 2151 } 2152 2153 static inline int 2154 dequeue_dec_one_op_cb(struct fpga_queue *q, struct rte_bbdev_dec_op **op, 2155 uint16_t desc_offset) 2156 { 2157 union fpga_dma_desc *desc; 2158 int desc_error = 0; 2159 /* Set descriptor */ 2160 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2161 & q->sw_ring_wrap_mask); 2162 2163 /* Verify done bit is set */ 2164 if (desc->dec_req.done == 0) 2165 return -1; 2166 2167 /* make sure the response is read atomically */ 2168 rte_smp_rmb(); 2169 2170 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2171 print_dma_dec_desc_debug_info(desc); 2172 2173 #endif 2174 2175 *op = desc->dec_req.op_addr; 2176 /* FPGA reports in half-iterations, from 0 to 31. get ceiling */ 2177 (*op)->turbo_dec.iter_count = (desc->dec_req.iter + 2) >> 1; 2178 /* crc_pass = 0 when decoder fails */ 2179 (*op)->status = !(desc->dec_req.crc_pass) << RTE_BBDEV_CRC_ERROR; 2180 /* Check the descriptor error field, return 1 on error */ 2181 desc_error = check_desc_error(desc->enc_req.error); 2182 (*op)->status |= desc_error << RTE_BBDEV_DATA_ERROR; 2183 return 1; 2184 } 2185 2186 static inline int 2187 dequeue_dec_one_op_tb(struct fpga_queue *q, struct rte_bbdev_dec_op **op, 2188 uint16_t desc_offset) 2189 { 2190 union fpga_dma_desc *desc; 2191 uint8_t cbs_in_op, cb_idx, iter_count = 0; 2192 int status = 0; 2193 int desc_error = 0; 2194 /* Set descriptor */ 2195 desc = q->ring_addr + ((q->head_free_desc + desc_offset) 2196 & q->sw_ring_wrap_mask); 2197 2198 /* Verify if done bit is set */ 2199 if (desc->dec_req.done == 0) 2200 return -1; 2201 2202 /* Make sure the response is read atomically */ 2203 rte_smp_rmb(); 2204 2205 /* Verify if done bit in all CBs is set */ 2206 cbs_in_op = desc->dec_req.cbs_in_op; 2207 for (cb_idx = 1; cb_idx < cbs_in_op; ++cb_idx) { 2208 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2209 cb_idx) & q->sw_ring_wrap_mask); 2210 if (desc->dec_req.done == 0) 2211 return -1; 2212 } 2213 2214 /* Make sure the response is read atomically */ 2215 rte_smp_rmb(); 2216 2217 for (cb_idx = 0; cb_idx < cbs_in_op; ++cb_idx) { 2218 desc = q->ring_addr + ((q->head_free_desc + desc_offset + 2219 cb_idx) & q->sw_ring_wrap_mask); 2220 /* get max iter_count for all CBs in op */ 2221 iter_count = RTE_MAX(iter_count, (uint8_t) desc->dec_req.iter); 2222 /* crc_pass = 0 when decoder fails, one fails all */ 2223 status |= !(desc->dec_req.crc_pass) << RTE_BBDEV_CRC_ERROR; 2224 /* Check the descriptor error field, return 1 on error */ 2225 desc_error = check_desc_error(desc->enc_req.error); 2226 status |= desc_error << RTE_BBDEV_DATA_ERROR; 2227 rte_bbdev_log_debug("DMA response desc %p", desc); 2228 } 2229 2230 *op = desc->dec_req.op_addr; 2231 2232 /* FPGA reports in half-iterations, get ceiling */ 2233 (*op)->turbo_dec.iter_count = (iter_count + 2) >> 1; 2234 (*op)->status = status; 2235 return cbs_in_op; 2236 } 2237 2238 static uint16_t 2239 fpga_dequeue_enc(struct rte_bbdev_queue_data *q_data, 2240 struct rte_bbdev_enc_op **ops, uint16_t num) 2241 { 2242 struct fpga_queue *q = q_data->queue_private; 2243 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask; 2244 uint16_t i; 2245 uint16_t dequeued_cbs = 0; 2246 struct rte_bbdev_enc_op *op; 2247 int ret; 2248 2249 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) { 2250 op = (q->ring_addr + ((q->head_free_desc + dequeued_cbs) 2251 & q->sw_ring_wrap_mask))->enc_req.op_addr; 2252 if (op->turbo_enc.code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) 2253 ret = dequeue_enc_one_op_tb(q, &ops[i], dequeued_cbs); 2254 else 2255 ret = dequeue_enc_one_op_cb(q, &ops[i], dequeued_cbs); 2256 2257 if (ret < 0) 2258 break; 2259 2260 dequeued_cbs += ret; 2261 2262 rte_bbdev_log_debug("dequeuing enc ops [%d/%d] | head %d | tail %d", 2263 dequeued_cbs, num, q->head_free_desc, q->tail); 2264 } 2265 2266 /* Update head */ 2267 q->head_free_desc = (q->head_free_desc + dequeued_cbs) & 2268 q->sw_ring_wrap_mask; 2269 2270 /* Update stats */ 2271 q_data->queue_stats.dequeued_count += i; 2272 2273 return i; 2274 } 2275 2276 static uint16_t 2277 fpga_dequeue_dec(struct rte_bbdev_queue_data *q_data, 2278 struct rte_bbdev_dec_op **ops, uint16_t num) 2279 { 2280 struct fpga_queue *q = q_data->queue_private; 2281 uint32_t avail = (q->tail - q->head_free_desc) & q->sw_ring_wrap_mask; 2282 uint16_t i; 2283 uint16_t dequeued_cbs = 0; 2284 struct rte_bbdev_dec_op *op; 2285 int ret; 2286 2287 for (i = 0; (i < num) && (dequeued_cbs < avail); ++i) { 2288 op = (q->ring_addr + ((q->head_free_desc + dequeued_cbs) 2289 & q->sw_ring_wrap_mask))->dec_req.op_addr; 2290 if (op->turbo_dec.code_block_mode == RTE_BBDEV_TRANSPORT_BLOCK) 2291 ret = dequeue_dec_one_op_tb(q, &ops[i], dequeued_cbs); 2292 else 2293 ret = dequeue_dec_one_op_cb(q, &ops[i], dequeued_cbs); 2294 2295 if (ret < 0) 2296 break; 2297 2298 dequeued_cbs += ret; 2299 2300 rte_bbdev_log_debug("dequeuing dec ops [%d/%d] | head %d | tail %d", 2301 dequeued_cbs, num, q->head_free_desc, q->tail); 2302 } 2303 2304 /* Update head */ 2305 q->head_free_desc = (q->head_free_desc + dequeued_cbs) & 2306 q->sw_ring_wrap_mask; 2307 2308 /* Update stats */ 2309 q_data->queue_stats.dequeued_count += i; 2310 2311 return i; 2312 } 2313 2314 /* Initialization Function */ 2315 static void 2316 fpga_lte_fec_init(struct rte_bbdev *dev, struct rte_pci_driver *drv) 2317 { 2318 struct rte_pci_device *pci_dev = RTE_DEV_TO_PCI(dev->device); 2319 2320 dev->dev_ops = &fpga_ops; 2321 dev->enqueue_enc_ops = fpga_enqueue_enc; 2322 dev->enqueue_dec_ops = fpga_enqueue_dec; 2323 dev->dequeue_enc_ops = fpga_dequeue_enc; 2324 dev->dequeue_dec_ops = fpga_dequeue_dec; 2325 2326 ((struct fpga_lte_fec_device *) dev->data->dev_private)->pf_device = 2327 !strcmp(drv->driver.name, 2328 RTE_STR(FPGA_LTE_FEC_PF_DRIVER_NAME)); 2329 ((struct fpga_lte_fec_device *) dev->data->dev_private)->mmio_base = 2330 pci_dev->mem_resource[0].addr; 2331 2332 rte_bbdev_log_debug( 2333 "Init device %s [%s] @ virtaddr %p phyaddr %#"PRIx64, 2334 drv->driver.name, dev->data->name, 2335 (void *)pci_dev->mem_resource[0].addr, 2336 pci_dev->mem_resource[0].phys_addr); 2337 } 2338 2339 static int 2340 fpga_lte_fec_probe(struct rte_pci_driver *pci_drv, 2341 struct rte_pci_device *pci_dev) 2342 { 2343 struct rte_bbdev *bbdev = NULL; 2344 char dev_name[RTE_BBDEV_NAME_MAX_LEN]; 2345 2346 if (pci_dev == NULL) { 2347 rte_bbdev_log(ERR, "NULL PCI device"); 2348 return -EINVAL; 2349 } 2350 2351 rte_pci_device_name(&pci_dev->addr, dev_name, sizeof(dev_name)); 2352 2353 /* Allocate memory to be used privately by drivers */ 2354 bbdev = rte_bbdev_allocate(pci_dev->device.name); 2355 if (bbdev == NULL) 2356 return -ENODEV; 2357 2358 /* allocate device private memory */ 2359 bbdev->data->dev_private = rte_zmalloc_socket(dev_name, 2360 sizeof(struct fpga_lte_fec_device), RTE_CACHE_LINE_SIZE, 2361 pci_dev->device.numa_node); 2362 2363 if (bbdev->data->dev_private == NULL) { 2364 rte_bbdev_log(CRIT, 2365 "Allocate of %zu bytes for device \"%s\" failed", 2366 sizeof(struct fpga_lte_fec_device), dev_name); 2367 rte_bbdev_release(bbdev); 2368 return -ENOMEM; 2369 } 2370 2371 /* Fill HW specific part of device structure */ 2372 bbdev->device = &pci_dev->device; 2373 bbdev->intr_handle = pci_dev->intr_handle; 2374 bbdev->data->socket_id = pci_dev->device.numa_node; 2375 2376 /* Invoke FEC FPGA device initialization function */ 2377 fpga_lte_fec_init(bbdev, pci_drv); 2378 2379 rte_bbdev_log_debug("bbdev id = %u [%s]", 2380 bbdev->data->dev_id, dev_name); 2381 2382 struct fpga_lte_fec_device *d = bbdev->data->dev_private; 2383 uint32_t version_id = fpga_reg_read_32(d->mmio_base, 2384 FPGA_LTE_FEC_VERSION_ID); 2385 rte_bbdev_log(INFO, "FEC FPGA RTL v%u.%u", 2386 ((uint16_t)(version_id >> 16)), ((uint16_t)version_id)); 2387 2388 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2389 if (!strcmp(pci_drv->driver.name, 2390 RTE_STR(FPGA_LTE_FEC_PF_DRIVER_NAME))) 2391 print_static_reg_debug_info(d->mmio_base); 2392 #endif 2393 return 0; 2394 } 2395 2396 static int 2397 fpga_lte_fec_remove(struct rte_pci_device *pci_dev) 2398 { 2399 struct rte_bbdev *bbdev; 2400 int ret; 2401 uint8_t dev_id; 2402 2403 if (pci_dev == NULL) 2404 return -EINVAL; 2405 2406 /* Find device */ 2407 bbdev = rte_bbdev_get_named_dev(pci_dev->device.name); 2408 if (bbdev == NULL) { 2409 rte_bbdev_log(CRIT, 2410 "Couldn't find HW dev \"%s\" to uninitialise it", 2411 pci_dev->device.name); 2412 return -ENODEV; 2413 } 2414 dev_id = bbdev->data->dev_id; 2415 2416 /* free device private memory before close */ 2417 rte_free(bbdev->data->dev_private); 2418 2419 /* Close device */ 2420 ret = rte_bbdev_close(dev_id); 2421 if (ret < 0) 2422 rte_bbdev_log(ERR, 2423 "Device %i failed to close during uninit: %i", 2424 dev_id, ret); 2425 2426 /* release bbdev from library */ 2427 ret = rte_bbdev_release(bbdev); 2428 if (ret) 2429 rte_bbdev_log(ERR, "Device %i failed to uninit: %i", dev_id, 2430 ret); 2431 2432 rte_bbdev_log_debug("Destroyed bbdev = %u", dev_id); 2433 2434 return 0; 2435 } 2436 2437 static inline void 2438 set_default_fpga_conf(struct rte_fpga_lte_fec_conf *def_conf) 2439 { 2440 /* clear default configuration before initialization */ 2441 memset(def_conf, 0, sizeof(struct rte_fpga_lte_fec_conf)); 2442 /* Set pf mode to true */ 2443 def_conf->pf_mode_en = true; 2444 2445 /* Set ratio between UL and DL to 1:1 (unit of weight is 3 CBs) */ 2446 def_conf->ul_bandwidth = 3; 2447 def_conf->dl_bandwidth = 3; 2448 2449 /* Set Load Balance Factor to 64 */ 2450 def_conf->dl_load_balance = 64; 2451 def_conf->ul_load_balance = 64; 2452 } 2453 2454 /* Initial configuration of FPGA LTE FEC device */ 2455 int 2456 rte_fpga_lte_fec_configure(const char *dev_name, 2457 const struct rte_fpga_lte_fec_conf *conf) 2458 { 2459 uint32_t payload_32, address; 2460 uint16_t payload_16; 2461 uint8_t payload_8; 2462 uint16_t q_id, vf_id, total_q_id, total_ul_q_id, total_dl_q_id; 2463 struct rte_bbdev *bbdev = rte_bbdev_get_named_dev(dev_name); 2464 struct rte_fpga_lte_fec_conf def_conf; 2465 2466 if (bbdev == NULL) { 2467 rte_bbdev_log(ERR, 2468 "Invalid dev_name (%s), or device is not yet initialised", 2469 dev_name); 2470 return -ENODEV; 2471 } 2472 2473 struct fpga_lte_fec_device *d = bbdev->data->dev_private; 2474 2475 if (conf == NULL) { 2476 rte_bbdev_log(ERR, 2477 "FPGA Configuration was not provided. Default configuration will be loaded."); 2478 set_default_fpga_conf(&def_conf); 2479 conf = &def_conf; 2480 } 2481 2482 /* 2483 * Configure UL:DL ratio. 2484 * [7:0]: UL weight 2485 * [15:8]: DL weight 2486 */ 2487 payload_16 = (conf->dl_bandwidth << 8) | conf->ul_bandwidth; 2488 address = FPGA_LTE_FEC_CONFIGURATION; 2489 fpga_reg_write_16(d->mmio_base, address, payload_16); 2490 2491 /* Clear all queues registers */ 2492 payload_32 = FPGA_INVALID_HW_QUEUE_ID; 2493 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 2494 address = (q_id << 2) + FPGA_LTE_FEC_QUEUE_MAP; 2495 fpga_reg_write_32(d->mmio_base, address, payload_32); 2496 } 2497 2498 /* 2499 * If PF mode is enabled allocate all queues for PF only. 2500 * 2501 * For VF mode each VF can have different number of UL and DL queues. 2502 * Total number of queues to configure cannot exceed FPGA 2503 * capabilities - 64 queues - 32 queues for UL and 32 queues for DL. 2504 * Queues mapping is done according to configuration: 2505 * 2506 * UL queues: 2507 * | Q_ID | VF_ID | 2508 * | 0 | 0 | 2509 * | ... | 0 | 2510 * | conf->vf_dl_queues_number[0] - 1 | 0 | 2511 * | conf->vf_dl_queues_number[0] | 1 | 2512 * | ... | 1 | 2513 * | conf->vf_dl_queues_number[1] - 1 | 1 | 2514 * | ... | ... | 2515 * | conf->vf_dl_queues_number[7] - 1 | 7 | 2516 * 2517 * DL queues: 2518 * | Q_ID | VF_ID | 2519 * | 32 | 0 | 2520 * | ... | 0 | 2521 * | conf->vf_ul_queues_number[0] - 1 | 0 | 2522 * | conf->vf_ul_queues_number[0] | 1 | 2523 * | ... | 1 | 2524 * | conf->vf_ul_queues_number[1] - 1 | 1 | 2525 * | ... | ... | 2526 * | conf->vf_ul_queues_number[7] - 1 | 7 | 2527 * 2528 * Example of configuration: 2529 * conf->vf_ul_queues_number[0] = 4; -> 4 UL queues for VF0 2530 * conf->vf_dl_queues_number[0] = 4; -> 4 DL queues for VF0 2531 * conf->vf_ul_queues_number[1] = 2; -> 2 UL queues for VF1 2532 * conf->vf_dl_queues_number[1] = 2; -> 2 DL queues for VF1 2533 * 2534 * UL: 2535 * | Q_ID | VF_ID | 2536 * | 0 | 0 | 2537 * | 1 | 0 | 2538 * | 2 | 0 | 2539 * | 3 | 0 | 2540 * | 4 | 1 | 2541 * | 5 | 1 | 2542 * 2543 * DL: 2544 * | Q_ID | VF_ID | 2545 * | 32 | 0 | 2546 * | 33 | 0 | 2547 * | 34 | 0 | 2548 * | 35 | 0 | 2549 * | 36 | 1 | 2550 * | 37 | 1 | 2551 */ 2552 if (conf->pf_mode_en) { 2553 payload_32 = 0x1; 2554 for (q_id = 0; q_id < FPGA_TOTAL_NUM_QUEUES; ++q_id) { 2555 address = (q_id << 2) + FPGA_LTE_FEC_QUEUE_MAP; 2556 fpga_reg_write_32(d->mmio_base, address, payload_32); 2557 } 2558 } else { 2559 /* Calculate total number of UL and DL queues to configure */ 2560 total_ul_q_id = total_dl_q_id = 0; 2561 for (vf_id = 0; vf_id < FPGA_LTE_FEC_NUM_VFS; ++vf_id) { 2562 total_ul_q_id += conf->vf_ul_queues_number[vf_id]; 2563 total_dl_q_id += conf->vf_dl_queues_number[vf_id]; 2564 } 2565 total_q_id = total_dl_q_id + total_ul_q_id; 2566 /* 2567 * Check if total number of queues to configure does not exceed 2568 * FPGA capabilities (64 queues - 32 UL and 32 DL queues) 2569 */ 2570 if ((total_ul_q_id > FPGA_NUM_UL_QUEUES) || 2571 (total_dl_q_id > FPGA_NUM_DL_QUEUES) || 2572 (total_q_id > FPGA_TOTAL_NUM_QUEUES)) { 2573 rte_bbdev_log(ERR, 2574 "FPGA Configuration failed. Too many queues to configure: UL_Q %u, DL_Q %u, FPGA_Q %u", 2575 total_ul_q_id, total_dl_q_id, 2576 FPGA_TOTAL_NUM_QUEUES); 2577 return -EINVAL; 2578 } 2579 total_ul_q_id = 0; 2580 for (vf_id = 0; vf_id < FPGA_LTE_FEC_NUM_VFS; ++vf_id) { 2581 for (q_id = 0; q_id < conf->vf_ul_queues_number[vf_id]; 2582 ++q_id, ++total_ul_q_id) { 2583 address = (total_ul_q_id << 2) + 2584 FPGA_LTE_FEC_QUEUE_MAP; 2585 payload_32 = ((0x80 + vf_id) << 16) | 0x1; 2586 fpga_reg_write_32(d->mmio_base, address, 2587 payload_32); 2588 } 2589 } 2590 total_dl_q_id = 0; 2591 for (vf_id = 0; vf_id < FPGA_LTE_FEC_NUM_VFS; ++vf_id) { 2592 for (q_id = 0; q_id < conf->vf_dl_queues_number[vf_id]; 2593 ++q_id, ++total_dl_q_id) { 2594 address = ((total_dl_q_id + FPGA_NUM_UL_QUEUES) 2595 << 2) + FPGA_LTE_FEC_QUEUE_MAP; 2596 payload_32 = ((0x80 + vf_id) << 16) | 0x1; 2597 fpga_reg_write_32(d->mmio_base, address, 2598 payload_32); 2599 } 2600 } 2601 } 2602 2603 /* Setting Load Balance Factor */ 2604 payload_16 = (conf->dl_load_balance << 8) | (conf->ul_load_balance); 2605 address = FPGA_LTE_FEC_LOAD_BALANCE_FACTOR; 2606 fpga_reg_write_16(d->mmio_base, address, payload_16); 2607 2608 /* Setting length of ring descriptor entry */ 2609 payload_16 = FPGA_RING_DESC_ENTRY_LENGTH; 2610 address = FPGA_LTE_FEC_RING_DESC_LEN; 2611 fpga_reg_write_16(d->mmio_base, address, payload_16); 2612 2613 /* Setting FLR timeout value */ 2614 payload_16 = conf->flr_time_out; 2615 address = FPGA_LTE_FEC_FLR_TIME_OUT; 2616 fpga_reg_write_16(d->mmio_base, address, payload_16); 2617 2618 /* Queue PF/VF mapping table is ready */ 2619 payload_8 = 0x1; 2620 address = FPGA_LTE_FEC_QUEUE_PF_VF_MAP_DONE; 2621 fpga_reg_write_8(d->mmio_base, address, payload_8); 2622 2623 rte_bbdev_log_debug("PF FPGA LTE FEC configuration complete for %s", 2624 dev_name); 2625 2626 #ifdef RTE_LIBRTE_BBDEV_DEBUG 2627 print_static_reg_debug_info(d->mmio_base); 2628 #endif 2629 return 0; 2630 } 2631 2632 /* FPGA LTE FEC PCI PF address map */ 2633 static struct rte_pci_id pci_id_fpga_lte_fec_pf_map[] = { 2634 { 2635 RTE_PCI_DEVICE(FPGA_LTE_FEC_VENDOR_ID, 2636 FPGA_LTE_FEC_PF_DEVICE_ID) 2637 }, 2638 {.device_id = 0}, 2639 }; 2640 2641 static struct rte_pci_driver fpga_lte_fec_pci_pf_driver = { 2642 .probe = fpga_lte_fec_probe, 2643 .remove = fpga_lte_fec_remove, 2644 .id_table = pci_id_fpga_lte_fec_pf_map, 2645 .drv_flags = RTE_PCI_DRV_NEED_MAPPING 2646 }; 2647 2648 /* FPGA LTE FEC PCI VF address map */ 2649 static struct rte_pci_id pci_id_fpga_lte_fec_vf_map[] = { 2650 { 2651 RTE_PCI_DEVICE(FPGA_LTE_FEC_VENDOR_ID, 2652 FPGA_LTE_FEC_VF_DEVICE_ID) 2653 }, 2654 {.device_id = 0}, 2655 }; 2656 2657 static struct rte_pci_driver fpga_lte_fec_pci_vf_driver = { 2658 .probe = fpga_lte_fec_probe, 2659 .remove = fpga_lte_fec_remove, 2660 .id_table = pci_id_fpga_lte_fec_vf_map, 2661 .drv_flags = RTE_PCI_DRV_NEED_MAPPING 2662 }; 2663 2664 2665 RTE_PMD_REGISTER_PCI(FPGA_LTE_FEC_PF_DRIVER_NAME, fpga_lte_fec_pci_pf_driver); 2666 RTE_PMD_REGISTER_PCI_TABLE(FPGA_LTE_FEC_PF_DRIVER_NAME, 2667 pci_id_fpga_lte_fec_pf_map); 2668 RTE_PMD_REGISTER_PCI(FPGA_LTE_FEC_VF_DRIVER_NAME, fpga_lte_fec_pci_vf_driver); 2669 RTE_PMD_REGISTER_PCI_TABLE(FPGA_LTE_FEC_VF_DRIVER_NAME, 2670 pci_id_fpga_lte_fec_vf_map); 2671