1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 #ifndef _OTX_EP_COMMON_H_ 5 #define _OTX_EP_COMMON_H_ 6 7 #include <rte_bitops.h> 8 #include <rte_spinlock.h> 9 #include <unistd.h> 10 #include <assert.h> 11 #include <rte_eal.h> 12 #include <rte_kvargs.h> 13 #include <rte_mempool.h> 14 #include <rte_mbuf.h> 15 #include <rte_io.h> 16 #include <rte_net.h> 17 #include <ethdev_pci.h> 18 19 #define OTX_EP_CN8XX RTE_BIT32(0) 20 #define OTX_EP_CN9XX RTE_BIT32(1) 21 #define OTX_EP_CN10XX RTE_BIT32(2) 22 23 #define OTX_EP_NW_PKT_OP 0x1220 24 #define OTX_EP_NW_CMD_OP 0x1221 25 26 #define OTX_EP_MAX_RINGS_PER_VF (8) 27 #define OTX_EP_CFG_IO_QUEUES OTX_EP_MAX_RINGS_PER_VF 28 #define OTX_EP_32BYTE_INSTR (32) 29 #define OTX_EP_64BYTE_INSTR (64) 30 /* 31 * Backpressure for SDP is configured on Octeon, and the minimum queue sizes 32 * must be much larger than the backpressure watermark configured in the Octeon 33 * SDP driver. IQ and OQ backpressure configurations are separate. 34 */ 35 #define OTX_EP_MIN_IQ_DESCRIPTORS (2048) 36 #define OTX_EP_MIN_OQ_DESCRIPTORS (2048) 37 #define OTX_EP_MAX_IQ_DESCRIPTORS (8192) 38 #define OTX_EP_MAX_OQ_DESCRIPTORS (8192) 39 #define OTX_EP_OQ_BUF_SIZE (2048) 40 #define OTX_EP_MIN_RX_BUF_SIZE (64) 41 #define OTX_EP_OQ_WMARK_MIN (256) 42 43 #define OTX_EP_OQ_INFOPTR_MODE (0) 44 #define OTX_EP_OQ_REFIL_THRESHOLD (16) 45 46 /* IQ instruction req types */ 47 #define OTX_EP_REQTYPE_NONE (0) 48 #define OTX_EP_REQTYPE_NORESP_INSTR (1) 49 #define OTX_EP_REQTYPE_NORESP_NET_DIRECT (2) 50 #define OTX_EP_REQTYPE_NORESP_NET OTX_EP_REQTYPE_NORESP_NET_DIRECT 51 #define OTX_EP_REQTYPE_NORESP_GATHER (3) 52 #define OTX_EP_NORESP_OHSM_SEND (4) 53 #define OTX_EP_NORESP_LAST (4) 54 #define OTX_EP_PCI_RING_ALIGN 65536 55 #define OTX_EP_MAX_SG_LISTS 6 56 #define OTX_EP_NUM_SG_PTRS 4 57 #define SDP_PKIND 40 58 #define SDP_OTX2_PKIND 57 59 #define SDP_OTX2_PKIND_FS0 0 60 61 #define ORDERED_TAG 0 62 #define ATOMIC_TAG 1 63 #define NULL_TAG 2 64 #define NULL_NULL_TAG 3 65 66 #define OTX_EP_BUSY_LOOP_COUNT (10000) 67 #define OTX_EP_MAX_IOQS_PER_VF 8 68 #define OTX_CUST_DATA_LEN 0 69 70 #define otx_ep_info(fmt, args...) \ 71 rte_log(RTE_LOG_INFO, otx_net_ep_logtype, \ 72 "%s():%u " fmt "\n", \ 73 __func__, __LINE__, ##args) 74 75 #define otx_ep_err(fmt, args...) \ 76 rte_log(RTE_LOG_ERR, otx_net_ep_logtype, \ 77 "%s():%u " fmt "\n", \ 78 __func__, __LINE__, ##args) 79 80 #define otx_ep_dbg(fmt, args...) \ 81 rte_log(RTE_LOG_DEBUG, otx_net_ep_logtype, \ 82 "%s():%u " fmt "\n", \ 83 __func__, __LINE__, ##args) 84 85 /* IO Access */ 86 #define oct_ep_read64(addr) rte_read64_relaxed((void *)(addr)) 87 #define oct_ep_write64(val, addr) rte_write64_relaxed((val), (void *)(addr)) 88 89 /* Mailbox maximum data size */ 90 #define MBOX_MAX_DATA_BUF_SIZE 320 91 92 /* Input Request Header format */ 93 union otx_ep_instr_irh { 94 uint64_t u64; 95 struct { 96 /* Request ID */ 97 uint64_t rid:16; 98 99 /* PCIe port to use for response */ 100 uint64_t pcie_port:3; 101 102 /* Scatter indicator 1=scatter */ 103 uint64_t scatter:1; 104 105 /* Size of Expected result OR no. of entries in scatter list */ 106 uint64_t rlenssz:14; 107 108 /* Desired destination port for result */ 109 uint64_t dport:6; 110 111 /* Opcode Specific parameters */ 112 uint64_t param:8; 113 114 /* Opcode for the return packet */ 115 uint64_t opcode:16; 116 } s; 117 }; 118 119 #define otx_ep_write64(value, base_addr, reg_off) \ 120 {\ 121 typeof(value) val = (value); \ 122 typeof(reg_off) off = (reg_off); \ 123 otx_ep_dbg("octeon_write_csr64: reg: 0x%08lx val: 0x%016llx\n", \ 124 (unsigned long)off, (unsigned long long)val); \ 125 rte_write64(val, ((base_addr) + off)); \ 126 } 127 128 /* Instruction Header - for OCTEON-TX models */ 129 typedef union otx_ep_instr_ih { 130 uint64_t u64; 131 struct { 132 /** Data Len */ 133 uint64_t tlen:16; 134 135 /** Reserved */ 136 uint64_t rsvd:20; 137 138 /** PKIND for OTX_EP */ 139 uint64_t pkind:6; 140 141 /** Front Data size */ 142 uint64_t fsz:6; 143 144 /** No. of entries in gather list */ 145 uint64_t gsz:14; 146 147 /** Gather indicator 1=gather*/ 148 uint64_t gather:1; 149 150 /** Reserved3 */ 151 uint64_t reserved3:1; 152 } s; 153 } otx_ep_instr_ih_t; 154 155 struct otx_ep_sg_entry { 156 /** The first 64 bit gives the size of data in each dptr. */ 157 union { 158 uint16_t size[OTX_EP_NUM_SG_PTRS]; 159 uint64_t size64; 160 } u; 161 162 /** The 4 dptr pointers for this entry. */ 163 uint64_t ptr[OTX_EP_NUM_SG_PTRS]; 164 }; 165 166 #define OTX_EP_SG_ENTRY_SIZE (sizeof(struct otx_ep_sg_entry)) 167 168 /** Structure of a node in list of gather components maintained by 169 * driver for each network device. 170 */ 171 struct otx_ep_gather { 172 /** number of gather entries. */ 173 int num_sg; 174 175 /** Gather component that can accommodate max sized fragment list 176 * received from the IP layer. 177 */ 178 struct otx_ep_sg_entry *sg; 179 }; 180 181 struct otx_ep_buf_free_info { 182 struct rte_mbuf *mbuf; 183 struct otx_ep_gather g; 184 }; 185 186 /* OTX_EP IQ request list */ 187 struct otx_ep_instr_list { 188 struct otx_ep_buf_free_info finfo; 189 uint32_t reqtype; 190 }; 191 #define OTX_EP_IQREQ_LIST_SIZE (sizeof(struct otx_ep_instr_list)) 192 193 /* Input Queue statistics. Each input queue has four stats fields. */ 194 struct otx_ep_iq_stats { 195 uint64_t instr_posted; /* Instructions posted to this queue. */ 196 uint64_t instr_processed; /* Instructions processed in this queue. */ 197 uint64_t instr_dropped; /* Instructions that could not be processed */ 198 uint64_t tx_pkts; 199 uint64_t tx_bytes; 200 }; 201 202 /* Structure to define the configuration attributes for each Input queue. */ 203 struct otx_ep_iq_config { 204 /* Max number of IQs available */ 205 uint16_t max_iqs; 206 207 /* Command size - 32 or 64 bytes */ 208 uint16_t instr_type; 209 210 /* Pending list size, usually set to the sum of the size of all IQs */ 211 uint32_t pending_list_size; 212 }; 213 214 /** The instruction (input) queue. 215 * The input queue is used to post raw (instruction) mode data or packet data 216 * to OCTEON 9 device from the host. Each IQ of a OTX_EP EP VF device has one 217 * such structure to represent it. 218 */ 219 struct otx_ep_instr_queue { 220 /* Location in memory updated by SDP ISM */ 221 uint32_t *inst_cnt_ism; 222 struct rte_mbuf **mbuf_list; 223 /* Pointer to the Virtual Base addr of the input ring. */ 224 uint8_t *base_addr; 225 226 /* track inst count locally to consolidate HW counter updates */ 227 uint32_t inst_cnt_prev; 228 229 /* Input ring index, where the driver should write the next packet */ 230 uint32_t host_write_index; 231 232 /* Input ring index, where the OCTEON 9 should read the next packet */ 233 uint32_t otx_read_index; 234 /** This index aids in finding the window in the queue where OCTEON 9 235 * has read the commands. 236 */ 237 uint32_t flush_index; 238 /* This keeps track of the instructions pending in this queue. */ 239 uint64_t instr_pending; 240 241 /* Memory zone */ 242 const struct rte_memzone *iq_mz; 243 /* OTX_EP doorbell register for the ring. */ 244 void *doorbell_reg; 245 246 /* OTX_EP instruction count register for this ring. */ 247 void *inst_cnt_reg; 248 249 /* Number of instructions pending to be posted to OCTEON 9. */ 250 uint32_t fill_cnt; 251 252 struct otx_ep_device *otx_ep_dev; 253 254 uint32_t q_no; 255 uint32_t pkt_in_done; 256 257 /* Flag for 64 byte commands. */ 258 uint32_t iqcmd_64B:1; 259 uint32_t rsvd:17; 260 uint32_t status:8; 261 262 /* Number of descriptors in this ring. */ 263 uint32_t nb_desc; 264 265 /* Use ISM memory */ 266 uint8_t ism_ena; 267 268 /* Size of the descriptor. */ 269 uint8_t desc_size; 270 271 uint32_t reset_instr_cnt; 272 273 /* Free-running/wrapping instruction counter for IQ. */ 274 uint32_t inst_cnt; 275 276 uint64_t partial_ih; 277 278 /* This IQ request list */ 279 struct otx_ep_instr_list *req_list; 280 281 /* Statistics for this input queue. */ 282 struct otx_ep_iq_stats stats; 283 284 /* DMA mapped base address of the input descriptor ring. */ 285 uint64_t base_addr_dma; 286 }; 287 288 /** Descriptor format. 289 * The descriptor ring is made of descriptors which have 2 64-bit values: 290 * -# Physical (bus) address of the data buffer. 291 * -# Physical (bus) address of a otx_ep_droq_info structure. 292 * The device DMA's incoming packets and its information at the address 293 * given by these descriptor fields. 294 */ 295 struct otx_ep_droq_desc { 296 /* The buffer pointer */ 297 uint64_t buffer_ptr; 298 299 /* The Info pointer */ 300 uint64_t info_ptr; 301 }; 302 #define OTX_EP_DROQ_DESC_SIZE (sizeof(struct otx_ep_droq_desc)) 303 304 /* Receive Header */ 305 union otx_ep_rh { 306 uint64_t rh64; 307 }; 308 #define OTX_EP_RH_SIZE (sizeof(union otx_ep_rh)) 309 310 /** Information about packet DMA'ed by OCTEON 9. 311 * The format of the information available at Info Pointer after OCTEON 9 312 * has posted a packet. Not all descriptors have valid information. Only 313 * the Info field of the first descriptor for a packet has information 314 * about the packet. 315 */ 316 struct otx_ep_droq_info { 317 /* The Length of the packet. */ 318 uint64_t length; 319 320 /* The Output Receive Header. */ 321 union otx_ep_rh rh; 322 }; 323 #define OTX_EP_DROQ_INFO_SIZE (sizeof(struct otx_ep_droq_info)) 324 325 /* DROQ statistics. Each output queue has four stats fields. */ 326 struct otx_ep_droq_stats { 327 /* Number of packets received in this queue. */ 328 uint64_t pkts_received; 329 330 /* Bytes received by this queue. */ 331 uint64_t bytes_received; 332 333 /* Num of failures of rte_pktmbuf_alloc() */ 334 uint64_t rx_alloc_failure; 335 336 /* Rx error */ 337 uint64_t rx_err; 338 339 /* packets with data got ready after interrupt arrived */ 340 uint64_t pkts_delayed_data; 341 342 /* packets dropped due to zero length */ 343 uint64_t dropped_zlp; 344 }; 345 346 /* Structure to define the configuration attributes for each Output queue. */ 347 struct otx_ep_oq_config { 348 /* Max number of OQs available */ 349 uint16_t max_oqs; 350 351 /* If set, the Output queue uses info-pointer mode. (Default: 1 ) */ 352 uint16_t info_ptr; 353 354 /** The number of buffers that were consumed during packet processing by 355 * the driver on this Output queue before the driver attempts to 356 * replenish the descriptor ring with new buffers. 357 */ 358 uint32_t refill_threshold; 359 }; 360 361 /* The Descriptor Ring Output Queue(DROQ) structure. */ 362 struct otx_ep_droq { 363 /* The 8B aligned descriptor ring starts at this address. */ 364 struct otx_ep_droq_desc *desc_ring; 365 366 /* The 8B aligned info ptrs begin from this address. */ 367 struct otx_ep_droq_info *info_list; 368 369 /* receive buffer list contains mbuf ptr list */ 370 struct rte_mbuf **recv_buf_list; 371 372 /* Packet re-arm data. */ 373 uint64_t rearm_data; 374 375 /* Packets pending to be processed */ 376 uint64_t pkts_pending; 377 378 /* Driver should read the next packet at this index */ 379 uint32_t read_idx; 380 381 /* OCTEON 9 will write the next packet at this index */ 382 uint32_t write_idx; 383 384 /* At this index, the driver will refill the descriptor's buffer */ 385 uint32_t refill_idx; 386 387 /* The number of descriptors pending to refill. */ 388 uint32_t refill_count; 389 390 /* Number of descriptors in this ring. */ 391 uint32_t nb_desc; 392 393 uint32_t refill_threshold; 394 395 uint64_t last_pkt_count; 396 397 struct rte_mempool *mpool; 398 399 /* The size of each buffer pointed by the buffer pointer. */ 400 uint32_t buffer_size; 401 402 /** Pointer to the mapped packet credit register. 403 * Host writes number of info/buffer ptrs available to this register 404 */ 405 void *pkts_credit_reg; 406 407 /** Pointer to the mapped packet sent register. OCTEON 9 writes the 408 * number of packets DMA'ed to host memory in this register. 409 */ 410 void *pkts_sent_reg; 411 412 /* Use ISM memory */ 413 uint8_t ism_ena; 414 415 /* Pointer to host memory copy of output packet count, set by ISM */ 416 uint32_t *pkts_sent_ism; 417 uint32_t pkts_sent_prev; 418 419 /* Statistics for this DROQ. */ 420 struct otx_ep_droq_stats stats; 421 422 /** Handle DMA incompletion during pkt reads. 423 * This variable is used to initiate a sent_reg_read 424 * that completes pending dma 425 * this variable is used as lvalue so compiler cannot optimize 426 * the reads. 427 */ 428 uint32_t sent_reg_val; 429 430 uint32_t q_no; 431 432 struct otx_ep_device *otx_ep_dev; 433 434 /* DMA mapped address of the DROQ descriptor ring. */ 435 size_t desc_ring_dma; 436 437 /* Info_ptr list is allocated at this virtual address. */ 438 size_t info_base_addr; 439 440 /* DMA mapped address of the info list */ 441 size_t info_list_dma; 442 443 /* Allocated size of info list. */ 444 uint32_t info_alloc_size; 445 446 /* Memory zone **/ 447 const struct rte_memzone *desc_ring_mz; 448 449 const struct rte_memzone *info_mz; 450 }; 451 #define OTX_EP_DROQ_SIZE (sizeof(struct otx_ep_droq)) 452 453 /* IQ/OQ mask */ 454 struct otx_ep_io_enable { 455 uint64_t iq; 456 uint64_t oq; 457 uint64_t iq64B; 458 }; 459 460 /* Structure to define the configuration. */ 461 struct otx_ep_config { 462 /* Input Queue attributes. */ 463 struct otx_ep_iq_config iq; 464 465 /* Output Queue attributes. */ 466 struct otx_ep_oq_config oq; 467 468 /* Num of desc for IQ rings */ 469 uint32_t num_iqdef_descs; 470 471 /* Num of desc for OQ rings */ 472 uint32_t num_oqdef_descs; 473 474 /* OQ buffer size */ 475 uint32_t oqdef_buf_size; 476 }; 477 478 /* SRIOV information */ 479 struct otx_ep_sriov_info { 480 /* Number of rings assigned to VF */ 481 uint32_t rings_per_vf; 482 483 /* Number of VF devices enabled */ 484 uint32_t num_vfs; 485 }; 486 487 /* Required functions for each VF device */ 488 struct otx_ep_fn_list { 489 int (*setup_iq_regs)(struct otx_ep_device *otx_ep, uint32_t q_no); 490 491 int (*setup_oq_regs)(struct otx_ep_device *otx_ep, uint32_t q_no); 492 493 int (*setup_device_regs)(struct otx_ep_device *otx_ep); 494 495 int (*enable_io_queues)(struct otx_ep_device *otx_ep); 496 void (*disable_io_queues)(struct otx_ep_device *otx_ep); 497 498 int (*enable_iq)(struct otx_ep_device *otx_ep, uint32_t q_no); 499 void (*disable_iq)(struct otx_ep_device *otx_ep, uint32_t q_no); 500 501 int (*enable_oq)(struct otx_ep_device *otx_ep, uint32_t q_no); 502 void (*disable_oq)(struct otx_ep_device *otx_ep, uint32_t q_no); 503 504 int (*enable_rxq_intr)(struct otx_ep_device *otx_epvf, uint16_t q_no); 505 int (*disable_rxq_intr)(struct otx_ep_device *otx_epvf, uint16_t q_no); 506 }; 507 508 /* OTX_EP EP VF device data structure */ 509 struct otx_ep_device { 510 /* PCI device pointer */ 511 struct rte_pci_device *pdev; 512 513 uint16_t chip_id; 514 515 uint32_t pkind; 516 517 struct rte_eth_dev *eth_dev; 518 519 int port_id; 520 521 /* Memory mapped h/w address */ 522 uint8_t *hw_addr; 523 524 struct otx_ep_fn_list fn_list; 525 526 uint32_t max_tx_queues; 527 528 uint32_t max_rx_queues; 529 530 /* Num IQs */ 531 uint32_t nb_tx_queues; 532 533 /* The input instruction queues */ 534 struct otx_ep_instr_queue *instr_queue[OTX_EP_MAX_IOQS_PER_VF]; 535 536 /* Num OQs */ 537 uint32_t nb_rx_queues; 538 539 /* The DROQ output queues */ 540 struct otx_ep_droq *droq[OTX_EP_MAX_IOQS_PER_VF]; 541 542 /* IOQ mask */ 543 struct otx_ep_io_enable io_qmask; 544 545 /* SR-IOV info */ 546 struct otx_ep_sriov_info sriov_info; 547 548 /* Device configuration */ 549 const struct otx_ep_config *conf; 550 551 uint64_t rx_offloads; 552 553 uint64_t tx_offloads; 554 555 /* DMA buffer for SDP ISM messages */ 556 const struct rte_memzone *ism_buffer_mz; 557 558 /* Mailbox lock */ 559 rte_spinlock_t mbox_lock; 560 561 /* Mailbox data */ 562 uint8_t mbox_data_buf[MBOX_MAX_DATA_BUF_SIZE]; 563 564 /* Mailbox data index */ 565 int32_t mbox_data_index; 566 567 /* Mailbox receive message length */ 568 int32_t mbox_rcv_message_len; 569 570 /* Negotiated Mbox version */ 571 uint32_t mbox_neg_ver; 572 573 /* Generation */ 574 uint32_t chip_gen; 575 576 /* Use ISM memory */ 577 uint8_t ism_ena; 578 }; 579 580 int otx_ep_setup_iqs(struct otx_ep_device *otx_ep, uint32_t iq_no, 581 int num_descs, unsigned int socket_id); 582 int otx_ep_delete_iqs(struct otx_ep_device *otx_ep, uint32_t iq_no); 583 584 int otx_ep_setup_oqs(struct otx_ep_device *otx_ep, int oq_no, int num_descs, 585 int desc_size, struct rte_mempool *mpool, 586 unsigned int socket_id); 587 int otx_ep_delete_oqs(struct otx_ep_device *otx_ep, uint32_t oq_no); 588 589 #define OTX_EP_MAX_PKT_SZ 65498U 590 #define OTX_EP_MAX_MAC_ADDRS 1 591 #define OTX_EP_SG_ALIGN 8 592 #define OTX_EP_CLEAR_ISIZE_BSIZE 0x7FFFFFULL 593 #define OTX_EP_CLEAR_OUT_INT_LVLS 0x3FFFFFFFFFFFFFULL 594 #define OTX_EP_CLEAR_IN_INT_LVLS 0xFFFFFFFF 595 #define OTX_EP_CLEAR_SDP_IN_INT_LVLS 0x3FFFFFFFFFFFFFUL 596 #define OTX_EP_DROQ_BUFSZ_MASK 0xFFFF 597 #define OTX_EP_CLEAR_SLIST_DBELL 0xFFFFFFFF 598 #define OTX_EP_CLEAR_SDP_OUT_PKT_CNT 0xFFFFFFFFF 599 600 /* Max overhead includes 601 * - Ethernet hdr 602 * - CRC 603 * - nested VLANs 604 * - octeon rx info 605 */ 606 #define OTX_EP_ETH_OVERHEAD \ 607 (RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + \ 608 (2 * RTE_VLAN_HLEN) + OTX_EP_DROQ_INFO_SIZE) 609 610 /* PCI IDs */ 611 #define PCI_VENDOR_ID_CAVIUM 0x177D 612 613 extern int otx_net_ep_logtype; 614 #define RTE_LOGTYPE_OTX_NET_EP otx_net_ep_logtype 615 616 #endif /* _OTX_EP_COMMON_H_ */ 617