1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright(C) 2021 Marvell. 3 */ 4 #ifndef _OTX_EP_COMMON_H_ 5 #define _OTX_EP_COMMON_H_ 6 7 #include <rte_bitops.h> 8 #include <rte_spinlock.h> 9 #include <unistd.h> 10 #include <assert.h> 11 #include <rte_eal.h> 12 #include <rte_kvargs.h> 13 #include <rte_mempool.h> 14 #include <rte_mbuf.h> 15 #include <rte_io.h> 16 #include <rte_net.h> 17 #include <ethdev_pci.h> 18 19 #define OTX_EP_CN8XX RTE_BIT32(0) 20 #define OTX_EP_CN9XX RTE_BIT32(1) 21 #define OTX_EP_CN10XX RTE_BIT32(2) 22 23 #define OTX_EP_NW_PKT_OP 0x1220 24 #define OTX_EP_NW_CMD_OP 0x1221 25 26 #define OTX_EP_MAX_RINGS_PER_VF (8) 27 #define OTX_EP_CFG_IO_QUEUES OTX_EP_MAX_RINGS_PER_VF 28 #define OTX_EP_32BYTE_INSTR (32) 29 #define OTX_EP_64BYTE_INSTR (64) 30 /* 31 * Backpressure for SDP is configured on Octeon, and the minimum queue sizes 32 * must be much larger than the backpressure watermark configured in the Octeon 33 * SDP driver. IQ and OQ backpressure configurations are separate. 34 */ 35 #define OTX_EP_MIN_IQ_DESCRIPTORS (2048) 36 #define OTX_EP_MIN_OQ_DESCRIPTORS (2048) 37 #define OTX_EP_MAX_IQ_DESCRIPTORS (8192) 38 #define OTX_EP_MAX_OQ_DESCRIPTORS (8192) 39 #define OTX_EP_OQ_BUF_SIZE (2048) 40 #define OTX_EP_MIN_RX_BUF_SIZE (64) 41 #define OTX_EP_OQ_WMARK_MIN (256) 42 43 #define OTX_EP_OQ_INFOPTR_MODE (0) 44 #define OTX_EP_OQ_REFIL_THRESHOLD (16) 45 46 /* IQ instruction req types */ 47 #define OTX_EP_REQTYPE_NONE (0) 48 #define OTX_EP_REQTYPE_NORESP_INSTR (1) 49 #define OTX_EP_REQTYPE_NORESP_NET_DIRECT (2) 50 #define OTX_EP_REQTYPE_NORESP_NET OTX_EP_REQTYPE_NORESP_NET_DIRECT 51 #define OTX_EP_REQTYPE_NORESP_GATHER (3) 52 #define OTX_EP_NORESP_OHSM_SEND (4) 53 #define OTX_EP_NORESP_LAST (4) 54 #define OTX_EP_PCI_RING_ALIGN 65536 55 #define OTX_EP_MAX_SG_LISTS 6 56 #define OTX_EP_NUM_SG_PTRS 4 57 #define SDP_PKIND 40 58 #define SDP_OTX2_PKIND 57 59 #define SDP_OTX2_PKIND_FS0 0 60 61 #define ORDERED_TAG 0 62 #define ATOMIC_TAG 1 63 #define NULL_TAG 2 64 #define NULL_NULL_TAG 3 65 66 #define OTX_EP_BUSY_LOOP_COUNT (10000) 67 #define OTX_EP_MAX_IOQS_PER_VF 8 68 #define OTX_CUST_DATA_LEN 0 69 70 #define otx_ep_info(...) \ 71 RTE_LOG_LINE_PREFIX(INFO, OTX_NET_EP, "%s():%u ", \ 72 __func__ RTE_LOG_COMMA __LINE__, __VA_ARGS__) 73 74 #define otx_ep_err(...) \ 75 RTE_LOG_LINE_PREFIX(ERR, OTX_NET_EP, "%s():%u ", \ 76 __func__ RTE_LOG_COMMA __LINE__, __VA_ARGS__) 77 78 #define otx_ep_dbg(...) \ 79 RTE_LOG_LINE_PREFIX(DEBUG, OTX_NET_EP, "%s():%u ", \ 80 __func__ RTE_LOG_COMMA __LINE__, __VA_ARGS__) 81 82 /* IO Access */ 83 #define oct_ep_read64(addr) rte_read64_relaxed((void *)(addr)) 84 #define oct_ep_write64(val, addr) rte_write64_relaxed((val), (void *)(addr)) 85 86 /* Mailbox maximum data size */ 87 #define MBOX_MAX_DATA_BUF_SIZE 320 88 89 /* Input Request Header format */ 90 union otx_ep_instr_irh { 91 uint64_t u64; 92 struct { 93 /* Request ID */ 94 uint64_t rid:16; 95 96 /* PCIe port to use for response */ 97 uint64_t pcie_port:3; 98 99 /* Scatter indicator 1=scatter */ 100 uint64_t scatter:1; 101 102 /* Size of Expected result OR no. of entries in scatter list */ 103 uint64_t rlenssz:14; 104 105 /* Desired destination port for result */ 106 uint64_t dport:6; 107 108 /* Opcode Specific parameters */ 109 uint64_t param:8; 110 111 /* Opcode for the return packet */ 112 uint64_t opcode:16; 113 } s; 114 }; 115 116 #define otx_ep_write64(value, base_addr, reg_off) \ 117 {\ 118 typeof(value) val = (value); \ 119 typeof(reg_off) off = (reg_off); \ 120 otx_ep_dbg("octeon_write_csr64: reg: 0x%08lx val: 0x%016llx", \ 121 (unsigned long)off, (unsigned long long)val); \ 122 rte_write64(val, ((base_addr) + off)); \ 123 } 124 125 /* Instruction Header - for OCTEON-TX models */ 126 typedef union otx_ep_instr_ih { 127 uint64_t u64; 128 struct { 129 /** Data Len */ 130 uint64_t tlen:16; 131 132 /** Reserved */ 133 uint64_t rsvd:20; 134 135 /** PKIND for OTX_EP */ 136 uint64_t pkind:6; 137 138 /** Front Data size */ 139 uint64_t fsz:6; 140 141 /** No. of entries in gather list */ 142 uint64_t gsz:14; 143 144 /** Gather indicator 1=gather*/ 145 uint64_t gather:1; 146 147 /** Reserved3 */ 148 uint64_t reserved3:1; 149 } s; 150 } otx_ep_instr_ih_t; 151 152 struct otx_ep_sg_entry { 153 /** The first 64 bit gives the size of data in each dptr. */ 154 union { 155 uint16_t size[OTX_EP_NUM_SG_PTRS]; 156 uint64_t size64; 157 } u; 158 159 /** The 4 dptr pointers for this entry. */ 160 uint64_t ptr[OTX_EP_NUM_SG_PTRS]; 161 }; 162 163 #define OTX_EP_SG_ENTRY_SIZE (sizeof(struct otx_ep_sg_entry)) 164 165 /** Structure of a node in list of gather components maintained by 166 * driver for each network device. 167 */ 168 struct otx_ep_gather { 169 /** number of gather entries. */ 170 int num_sg; 171 172 /** Gather component that can accommodate max sized fragment list 173 * received from the IP layer. 174 */ 175 struct otx_ep_sg_entry *sg; 176 }; 177 178 struct otx_ep_buf_free_info { 179 struct rte_mbuf *mbuf; 180 struct otx_ep_gather g; 181 }; 182 183 /* OTX_EP IQ request list */ 184 struct otx_ep_instr_list { 185 struct otx_ep_buf_free_info finfo; 186 uint32_t reqtype; 187 }; 188 #define OTX_EP_IQREQ_LIST_SIZE (sizeof(struct otx_ep_instr_list)) 189 190 /* Input Queue statistics. Each input queue has four stats fields. */ 191 struct otx_ep_iq_stats { 192 uint64_t instr_posted; /* Instructions posted to this queue. */ 193 uint64_t instr_processed; /* Instructions processed in this queue. */ 194 uint64_t instr_dropped; /* Instructions that could not be processed */ 195 uint64_t tx_pkts; 196 uint64_t tx_bytes; 197 }; 198 199 /* Structure to define the configuration attributes for each Input queue. */ 200 struct otx_ep_iq_config { 201 /* Max number of IQs available */ 202 uint16_t max_iqs; 203 204 /* Command size - 32 or 64 bytes */ 205 uint16_t instr_type; 206 207 /* Pending list size, usually set to the sum of the size of all IQs */ 208 uint32_t pending_list_size; 209 }; 210 211 /** The instruction (input) queue. 212 * The input queue is used to post raw (instruction) mode data or packet data 213 * to OCTEON 9 device from the host. Each IQ of a OTX_EP EP VF device has one 214 * such structure to represent it. 215 */ 216 struct otx_ep_instr_queue { 217 /* Location in memory updated by SDP ISM */ 218 RTE_ATOMIC(uint32_t) *inst_cnt_ism; 219 struct rte_mbuf **mbuf_list; 220 /* Pointer to the Virtual Base addr of the input ring. */ 221 uint8_t *base_addr; 222 223 /* track inst count locally to consolidate HW counter updates */ 224 uint32_t inst_cnt_prev; 225 226 /* Input ring index, where the driver should write the next packet */ 227 uint32_t host_write_index; 228 229 /* Input ring index, where the OCTEON 9 should read the next packet */ 230 uint32_t otx_read_index; 231 /** This index aids in finding the window in the queue where OCTEON 9 232 * has read the commands. 233 */ 234 uint32_t flush_index; 235 /* This keeps track of the instructions pending in this queue. */ 236 uint64_t instr_pending; 237 238 /* Memory zone */ 239 const struct rte_memzone *iq_mz; 240 /* OTX_EP doorbell register for the ring. */ 241 void *doorbell_reg; 242 243 /* OTX_EP instruction count register for this ring. */ 244 void *inst_cnt_reg; 245 246 /* Number of instructions pending to be posted to OCTEON 9. */ 247 uint32_t fill_cnt; 248 249 struct otx_ep_device *otx_ep_dev; 250 251 uint32_t q_no; 252 uint32_t pkt_in_done; 253 254 /* Flag for 64 byte commands. */ 255 uint32_t iqcmd_64B:1; 256 uint32_t rsvd:17; 257 uint32_t status:8; 258 259 /* Number of descriptors in this ring. */ 260 uint32_t nb_desc; 261 262 /* Use ISM memory */ 263 uint8_t ism_ena; 264 265 /* Size of the descriptor. */ 266 uint8_t desc_size; 267 268 uint32_t reset_instr_cnt; 269 270 /* Free-running/wrapping instruction counter for IQ. */ 271 uint32_t inst_cnt; 272 273 uint64_t partial_ih; 274 275 /* This IQ request list */ 276 struct otx_ep_instr_list *req_list; 277 278 /* Statistics for this input queue. */ 279 struct otx_ep_iq_stats stats; 280 281 /* DMA mapped base address of the input descriptor ring. */ 282 uint64_t base_addr_dma; 283 }; 284 285 /** Descriptor format. 286 * The descriptor ring is made of descriptors which have 2 64-bit values: 287 * -# Physical (bus) address of the data buffer. 288 * -# Physical (bus) address of a otx_ep_droq_info structure. 289 * The device DMA's incoming packets and its information at the address 290 * given by these descriptor fields. 291 */ 292 struct otx_ep_droq_desc { 293 /* The buffer pointer */ 294 uint64_t buffer_ptr; 295 296 /* The Info pointer */ 297 uint64_t info_ptr; 298 }; 299 #define OTX_EP_DROQ_DESC_SIZE (sizeof(struct otx_ep_droq_desc)) 300 301 /* Receive Header */ 302 union otx_ep_rh { 303 uint64_t rh64; 304 }; 305 #define OTX_EP_RH_SIZE (sizeof(union otx_ep_rh)) 306 307 /** Information about packet DMA'ed by OCTEON 9. 308 * The format of the information available at Info Pointer after OCTEON 9 309 * has posted a packet. Not all descriptors have valid information. Only 310 * the Info field of the first descriptor for a packet has information 311 * about the packet. 312 */ 313 struct otx_ep_droq_info { 314 /* The Length of the packet. */ 315 uint64_t length; 316 317 /* The Output Receive Header. */ 318 union otx_ep_rh rh; 319 }; 320 #define OTX_EP_DROQ_INFO_SIZE (sizeof(struct otx_ep_droq_info)) 321 322 /* DROQ statistics. Each output queue has four stats fields. */ 323 struct otx_ep_droq_stats { 324 /* Number of packets received in this queue. */ 325 uint64_t pkts_received; 326 327 /* Bytes received by this queue. */ 328 uint64_t bytes_received; 329 330 /* Num of failures of rte_pktmbuf_alloc() */ 331 uint64_t rx_alloc_failure; 332 333 /* Rx error */ 334 uint64_t rx_err; 335 336 /* packets with data got ready after interrupt arrived */ 337 uint64_t pkts_delayed_data; 338 339 /* packets dropped due to zero length */ 340 uint64_t dropped_zlp; 341 }; 342 343 /* Structure to define the configuration attributes for each Output queue. */ 344 struct otx_ep_oq_config { 345 /* Max number of OQs available */ 346 uint16_t max_oqs; 347 348 /* If set, the Output queue uses info-pointer mode. (Default: 1 ) */ 349 uint16_t info_ptr; 350 351 /** The number of buffers that were consumed during packet processing by 352 * the driver on this Output queue before the driver attempts to 353 * replenish the descriptor ring with new buffers. 354 */ 355 uint32_t refill_threshold; 356 }; 357 358 /* The Descriptor Ring Output Queue(DROQ) structure. */ 359 struct otx_ep_droq { 360 /* The 8B aligned descriptor ring starts at this address. */ 361 struct otx_ep_droq_desc *desc_ring; 362 363 /* The 8B aligned info ptrs begin from this address. */ 364 struct otx_ep_droq_info *info_list; 365 366 /* receive buffer list contains mbuf ptr list */ 367 struct rte_mbuf **recv_buf_list; 368 369 /* Packet re-arm data. */ 370 uint64_t rearm_data; 371 372 /* Packets pending to be processed */ 373 uint64_t pkts_pending; 374 375 /* Driver should read the next packet at this index */ 376 uint32_t read_idx; 377 378 /* OCTEON 9 will write the next packet at this index */ 379 uint32_t write_idx; 380 381 /* At this index, the driver will refill the descriptor's buffer */ 382 uint32_t refill_idx; 383 384 /* The number of descriptors pending to refill. */ 385 uint32_t refill_count; 386 387 /* Number of descriptors in this ring. */ 388 uint32_t nb_desc; 389 390 uint32_t refill_threshold; 391 392 uint64_t last_pkt_count; 393 394 struct rte_mempool *mpool; 395 396 /* The size of each buffer pointed by the buffer pointer. */ 397 uint32_t buffer_size; 398 399 /** Pointer to the mapped packet credit register. 400 * Host writes number of info/buffer ptrs available to this register 401 */ 402 void *pkts_credit_reg; 403 404 /** Pointer to the mapped packet sent register. OCTEON 9 writes the 405 * number of packets DMA'ed to host memory in this register. 406 */ 407 void *pkts_sent_reg; 408 409 /* Use ISM memory */ 410 uint8_t ism_ena; 411 412 /* Pointer to host memory copy of output packet count, set by ISM */ 413 RTE_ATOMIC(uint32_t) *pkts_sent_ism; 414 uint32_t pkts_sent_prev; 415 416 /* Statistics for this DROQ. */ 417 struct otx_ep_droq_stats stats; 418 419 /** Handle DMA incompletion during pkt reads. 420 * This variable is used to initiate a sent_reg_read 421 * that completes pending dma 422 * this variable is used as lvalue so compiler cannot optimize 423 * the reads. 424 */ 425 uint32_t sent_reg_val; 426 427 uint32_t q_no; 428 429 struct otx_ep_device *otx_ep_dev; 430 431 /* DMA mapped address of the DROQ descriptor ring. */ 432 size_t desc_ring_dma; 433 434 /* Info_ptr list is allocated at this virtual address. */ 435 size_t info_base_addr; 436 437 /* DMA mapped address of the info list */ 438 size_t info_list_dma; 439 440 /* Allocated size of info list. */ 441 uint32_t info_alloc_size; 442 443 /* Memory zone **/ 444 const struct rte_memzone *desc_ring_mz; 445 446 const struct rte_memzone *info_mz; 447 }; 448 #define OTX_EP_DROQ_SIZE (sizeof(struct otx_ep_droq)) 449 450 /* IQ/OQ mask */ 451 struct otx_ep_io_enable { 452 uint64_t iq; 453 uint64_t oq; 454 uint64_t iq64B; 455 }; 456 457 /* Structure to define the configuration. */ 458 struct otx_ep_config { 459 /* Input Queue attributes. */ 460 struct otx_ep_iq_config iq; 461 462 /* Output Queue attributes. */ 463 struct otx_ep_oq_config oq; 464 465 /* Num of desc for IQ rings */ 466 uint32_t num_iqdef_descs; 467 468 /* Num of desc for OQ rings */ 469 uint32_t num_oqdef_descs; 470 471 /* OQ buffer size */ 472 uint32_t oqdef_buf_size; 473 }; 474 475 /* SRIOV information */ 476 struct otx_ep_sriov_info { 477 /* Number of rings assigned to VF */ 478 uint32_t rings_per_vf; 479 480 /* Number of VF devices enabled */ 481 uint32_t num_vfs; 482 }; 483 484 /* Required functions for each VF device */ 485 struct otx_ep_fn_list { 486 int (*setup_iq_regs)(struct otx_ep_device *otx_ep, uint32_t q_no); 487 488 int (*setup_oq_regs)(struct otx_ep_device *otx_ep, uint32_t q_no); 489 490 int (*setup_device_regs)(struct otx_ep_device *otx_ep); 491 492 int (*enable_io_queues)(struct otx_ep_device *otx_ep); 493 void (*disable_io_queues)(struct otx_ep_device *otx_ep); 494 495 int (*enable_iq)(struct otx_ep_device *otx_ep, uint32_t q_no); 496 void (*disable_iq)(struct otx_ep_device *otx_ep, uint32_t q_no); 497 498 int (*enable_oq)(struct otx_ep_device *otx_ep, uint32_t q_no); 499 void (*disable_oq)(struct otx_ep_device *otx_ep, uint32_t q_no); 500 501 int (*enable_rxq_intr)(struct otx_ep_device *otx_epvf, uint16_t q_no); 502 int (*disable_rxq_intr)(struct otx_ep_device *otx_epvf, uint16_t q_no); 503 }; 504 505 /* OTX_EP EP VF device data structure */ 506 struct otx_ep_device { 507 /* PCI device pointer */ 508 struct rte_pci_device *pdev; 509 510 uint16_t chip_id; 511 512 uint32_t pkind; 513 514 struct rte_eth_dev *eth_dev; 515 516 int port_id; 517 518 /* Memory mapped h/w address */ 519 uint8_t *hw_addr; 520 521 struct otx_ep_fn_list fn_list; 522 523 uint32_t max_tx_queues; 524 525 uint32_t max_rx_queues; 526 527 /* Num IQs */ 528 uint32_t nb_tx_queues; 529 530 /* The input instruction queues */ 531 struct otx_ep_instr_queue *instr_queue[OTX_EP_MAX_IOQS_PER_VF]; 532 533 /* Num OQs */ 534 uint32_t nb_rx_queues; 535 536 /* The DROQ output queues */ 537 struct otx_ep_droq *droq[OTX_EP_MAX_IOQS_PER_VF]; 538 539 /* IOQ mask */ 540 struct otx_ep_io_enable io_qmask; 541 542 /* SR-IOV info */ 543 struct otx_ep_sriov_info sriov_info; 544 545 /* Device configuration */ 546 const struct otx_ep_config *conf; 547 548 uint64_t rx_offloads; 549 550 uint64_t tx_offloads; 551 552 /* DMA buffer for SDP ISM messages */ 553 const struct rte_memzone *ism_buffer_mz; 554 555 /* Mailbox lock */ 556 rte_spinlock_t mbox_lock; 557 558 /* Mailbox data */ 559 uint8_t mbox_data_buf[MBOX_MAX_DATA_BUF_SIZE]; 560 561 /* Mailbox data index */ 562 int32_t mbox_data_index; 563 564 /* Mailbox receive message length */ 565 int32_t mbox_rcv_message_len; 566 567 /* Negotiated Mbox version */ 568 uint32_t mbox_neg_ver; 569 570 /* Generation */ 571 uint32_t chip_gen; 572 573 /* Use ISM memory */ 574 uint8_t ism_ena; 575 }; 576 577 int otx_ep_setup_iqs(struct otx_ep_device *otx_ep, uint32_t iq_no, 578 int num_descs, unsigned int socket_id); 579 int otx_ep_delete_iqs(struct otx_ep_device *otx_ep, uint32_t iq_no); 580 581 int otx_ep_setup_oqs(struct otx_ep_device *otx_ep, int oq_no, int num_descs, 582 int desc_size, struct rte_mempool *mpool, 583 unsigned int socket_id); 584 int otx_ep_delete_oqs(struct otx_ep_device *otx_ep, uint32_t oq_no); 585 586 #define OTX_EP_MAX_PKT_SZ 65498U 587 #define OTX_EP_MAX_MAC_ADDRS 1 588 #define OTX_EP_SG_ALIGN 8 589 #define OTX_EP_CLEAR_ISIZE_BSIZE 0x7FFFFFULL 590 #define OTX_EP_CLEAR_OUT_INT_LVLS 0x3FFFFFFFFFFFFFULL 591 #define OTX_EP_CLEAR_IN_INT_LVLS 0xFFFFFFFF 592 #define OTX_EP_CLEAR_SDP_IN_INT_LVLS 0x3FFFFFFFFFFFFFUL 593 #define OTX_EP_DROQ_BUFSZ_MASK 0xFFFF 594 #define OTX_EP_CLEAR_SLIST_DBELL 0xFFFFFFFF 595 #define OTX_EP_CLEAR_SDP_OUT_PKT_CNT 0xFFFFFFFFF 596 597 /* Max overhead includes 598 * - Ethernet hdr 599 * - CRC 600 * - nested VLANs 601 * - octeon rx info 602 */ 603 #define OTX_EP_ETH_OVERHEAD \ 604 (RTE_ETHER_HDR_LEN + RTE_ETHER_CRC_LEN + \ 605 (2 * RTE_VLAN_HLEN) + OTX_EP_DROQ_INFO_SIZE) 606 607 /* PCI IDs */ 608 #define PCI_VENDOR_ID_CAVIUM 0x177D 609 610 extern int otx_net_ep_logtype; 611 #define RTE_LOGTYPE_OTX_NET_EP otx_net_ep_logtype 612 613 #endif /* _OTX_EP_COMMON_H_ */ 614