1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2016 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * This software was jointly developed between OKTET Labs (under contract 8 * for Solarflare) and Solarflare Communications, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright notice, 14 * this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright notice, 16 * this list of conditions and the following disclaimer in the documentation 17 * and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <stdbool.h> 33 34 #include <rte_mbuf.h> 35 #include <rte_io.h> 36 37 #include "efx.h" 38 #include "efx_types.h" 39 #include "efx_regs.h" 40 #include "efx_regs_ef10.h" 41 42 #include "sfc_dp_tx.h" 43 #include "sfc_tweak.h" 44 #include "sfc_kvargs.h" 45 #include "sfc_ef10.h" 46 47 #define sfc_ef10_tx_err(dpq, ...) \ 48 SFC_DP_LOG(SFC_KVARG_DATAPATH_EF10, ERR, dpq, __VA_ARGS__) 49 50 /** Maximum length of the mbuf segment data */ 51 #define SFC_MBUF_SEG_LEN_MAX \ 52 ((1u << (8 * sizeof(((struct rte_mbuf *)0)->data_len))) - 1) 53 54 /** Maximum length of the DMA descriptor data */ 55 #define SFC_EF10_TX_DMA_DESC_LEN_MAX \ 56 ((1u << ESF_DZ_TX_KER_BYTE_CNT_WIDTH) - 1) 57 58 /** Maximum number of DMA descriptors per mbuf segment */ 59 #define SFC_EF10_TX_MBUF_SEG_DESCS_MAX \ 60 SFC_DIV_ROUND_UP(SFC_MBUF_SEG_LEN_MAX, \ 61 SFC_EF10_TX_DMA_DESC_LEN_MAX) 62 63 /** 64 * Maximum number of descriptors/buffers in the Tx ring. 65 * It should guarantee that corresponding event queue never overfill. 66 * EF10 native datapath uses event queue of the same size as Tx queue. 67 * Maximum number of events on datapath can be estimated as number of 68 * Tx queue entries (one event per Tx buffer in the worst case) plus 69 * Tx error and flush events. 70 */ 71 #define SFC_EF10_TXQ_LIMIT(_ndesc) \ 72 ((_ndesc) - 1 /* head must not step on tail */ - \ 73 (SFC_EF10_EV_PER_CACHE_LINE - 1) /* max unused EvQ entries */ - \ 74 1 /* Rx error */ - 1 /* flush */) 75 76 struct sfc_ef10_tx_sw_desc { 77 struct rte_mbuf *mbuf; 78 }; 79 80 struct sfc_ef10_txq { 81 unsigned int flags; 82 #define SFC_EF10_TXQ_STARTED 0x1 83 #define SFC_EF10_TXQ_NOT_RUNNING 0x2 84 #define SFC_EF10_TXQ_EXCEPTION 0x4 85 86 unsigned int ptr_mask; 87 unsigned int added; 88 unsigned int completed; 89 unsigned int free_thresh; 90 unsigned int evq_read_ptr; 91 struct sfc_ef10_tx_sw_desc *sw_ring; 92 efx_qword_t *txq_hw_ring; 93 volatile void *doorbell; 94 efx_qword_t *evq_hw_ring; 95 96 /* Datapath transmit queue anchor */ 97 struct sfc_dp_txq dp; 98 }; 99 100 static inline struct sfc_ef10_txq * 101 sfc_ef10_txq_by_dp_txq(struct sfc_dp_txq *dp_txq) 102 { 103 return container_of(dp_txq, struct sfc_ef10_txq, dp); 104 } 105 106 static bool 107 sfc_ef10_tx_get_event(struct sfc_ef10_txq *txq, efx_qword_t *tx_ev) 108 { 109 volatile efx_qword_t *evq_hw_ring = txq->evq_hw_ring; 110 111 /* 112 * Exception flag is set when reap is done. 113 * It is never done twice per packet burst get and absence of 114 * the flag is checked on burst get entry. 115 */ 116 SFC_ASSERT((txq->flags & SFC_EF10_TXQ_EXCEPTION) == 0); 117 118 *tx_ev = evq_hw_ring[txq->evq_read_ptr & txq->ptr_mask]; 119 120 if (!sfc_ef10_ev_present(*tx_ev)) 121 return false; 122 123 if (unlikely(EFX_QWORD_FIELD(*tx_ev, FSF_AZ_EV_CODE) != 124 FSE_AZ_EV_CODE_TX_EV)) { 125 /* 126 * Do not move read_ptr to keep the event for exception 127 * handling by the control path. 128 */ 129 txq->flags |= SFC_EF10_TXQ_EXCEPTION; 130 sfc_ef10_tx_err(&txq->dp.dpq, 131 "TxQ exception at EvQ read ptr %#x", 132 txq->evq_read_ptr); 133 return false; 134 } 135 136 txq->evq_read_ptr++; 137 return true; 138 } 139 140 static void 141 sfc_ef10_tx_reap(struct sfc_ef10_txq *txq) 142 { 143 const unsigned int old_read_ptr = txq->evq_read_ptr; 144 const unsigned int ptr_mask = txq->ptr_mask; 145 unsigned int completed = txq->completed; 146 unsigned int pending = completed; 147 const unsigned int curr_done = pending - 1; 148 unsigned int anew_done = curr_done; 149 efx_qword_t tx_ev; 150 151 while (sfc_ef10_tx_get_event(txq, &tx_ev)) { 152 /* 153 * DROP_EVENT is an internal to the NIC, software should 154 * never see it and, therefore, may ignore it. 155 */ 156 157 /* Update the latest done descriptor */ 158 anew_done = EFX_QWORD_FIELD(tx_ev, ESF_DZ_TX_DESCR_INDX); 159 } 160 pending += (anew_done - curr_done) & ptr_mask; 161 162 if (pending != completed) { 163 do { 164 struct sfc_ef10_tx_sw_desc *txd; 165 166 txd = &txq->sw_ring[completed & ptr_mask]; 167 168 if (txd->mbuf != NULL) { 169 rte_pktmbuf_free(txd->mbuf); 170 txd->mbuf = NULL; 171 } 172 } while (++completed != pending); 173 174 txq->completed = completed; 175 } 176 177 sfc_ef10_ev_qclear(txq->evq_hw_ring, ptr_mask, old_read_ptr, 178 txq->evq_read_ptr); 179 } 180 181 static void 182 sfc_ef10_tx_qdesc_dma_create(phys_addr_t addr, uint16_t size, bool eop, 183 efx_qword_t *edp) 184 { 185 EFX_POPULATE_QWORD_4(*edp, 186 ESF_DZ_TX_KER_TYPE, 0, 187 ESF_DZ_TX_KER_CONT, !eop, 188 ESF_DZ_TX_KER_BYTE_CNT, size, 189 ESF_DZ_TX_KER_BUF_ADDR, addr); 190 } 191 192 static inline void 193 sfc_ef10_tx_qpush(struct sfc_ef10_txq *txq, unsigned int added, 194 unsigned int pushed) 195 { 196 efx_qword_t desc; 197 efx_oword_t oword; 198 199 /* 200 * This improves performance by pushing a TX descriptor at the same 201 * time as the doorbell. The descriptor must be added to the TXQ, 202 * so that can be used if the hardware decides not to use the pushed 203 * descriptor. 204 */ 205 desc.eq_u64[0] = txq->txq_hw_ring[pushed & txq->ptr_mask].eq_u64[0]; 206 EFX_POPULATE_OWORD_3(oword, 207 ERF_DZ_TX_DESC_WPTR, added & txq->ptr_mask, 208 ERF_DZ_TX_DESC_HWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_1), 209 ERF_DZ_TX_DESC_LWORD, EFX_QWORD_FIELD(desc, EFX_DWORD_0)); 210 211 /* DMA sync to device is not required */ 212 213 /* 214 * rte_io_wmb() which guarantees that the STORE operations 215 * (i.e. Tx and event descriptor updates) that precede 216 * the rte_io_wmb() call are visible to NIC before the STORE 217 * operations that follow it (i.e. doorbell write). 218 */ 219 rte_io_wmb(); 220 221 *(volatile __m128i *)txq->doorbell = oword.eo_u128[0]; 222 } 223 224 static uint16_t 225 sfc_ef10_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, uint16_t nb_pkts) 226 { 227 struct sfc_ef10_txq * const txq = sfc_ef10_txq_by_dp_txq(tx_queue); 228 unsigned int ptr_mask; 229 unsigned int added; 230 unsigned int dma_desc_space; 231 bool reap_done; 232 struct rte_mbuf **pktp; 233 struct rte_mbuf **pktp_end; 234 235 if (unlikely(txq->flags & 236 (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION))) 237 return 0; 238 239 ptr_mask = txq->ptr_mask; 240 added = txq->added; 241 dma_desc_space = SFC_EF10_TXQ_LIMIT(ptr_mask + 1) - 242 (added - txq->completed); 243 244 reap_done = (dma_desc_space < txq->free_thresh); 245 if (reap_done) { 246 sfc_ef10_tx_reap(txq); 247 dma_desc_space = SFC_EF10_TXQ_LIMIT(ptr_mask + 1) - 248 (added - txq->completed); 249 } 250 251 for (pktp = &tx_pkts[0], pktp_end = &tx_pkts[nb_pkts]; 252 pktp != pktp_end; 253 ++pktp) { 254 struct rte_mbuf *m_seg = *pktp; 255 unsigned int pkt_start = added; 256 uint32_t pkt_len; 257 258 if (likely(pktp + 1 != pktp_end)) 259 rte_mbuf_prefetch_part1(pktp[1]); 260 261 if (m_seg->nb_segs * SFC_EF10_TX_MBUF_SEG_DESCS_MAX > 262 dma_desc_space) { 263 if (reap_done) 264 break; 265 266 /* Push already prepared descriptors before polling */ 267 if (added != txq->added) { 268 sfc_ef10_tx_qpush(txq, added, txq->added); 269 txq->added = added; 270 } 271 272 sfc_ef10_tx_reap(txq); 273 reap_done = true; 274 dma_desc_space = SFC_EF10_TXQ_LIMIT(ptr_mask + 1) - 275 (added - txq->completed); 276 if (m_seg->nb_segs * SFC_EF10_TX_MBUF_SEG_DESCS_MAX > 277 dma_desc_space) 278 break; 279 } 280 281 pkt_len = m_seg->pkt_len; 282 do { 283 phys_addr_t seg_addr = rte_mbuf_data_dma_addr(m_seg); 284 unsigned int seg_len = rte_pktmbuf_data_len(m_seg); 285 286 SFC_ASSERT(seg_len <= SFC_EF10_TX_DMA_DESC_LEN_MAX); 287 288 pkt_len -= seg_len; 289 290 sfc_ef10_tx_qdesc_dma_create(seg_addr, 291 seg_len, (pkt_len == 0), 292 &txq->txq_hw_ring[added & ptr_mask]); 293 ++added; 294 295 } while ((m_seg = m_seg->next) != 0); 296 297 dma_desc_space -= (added - pkt_start); 298 299 /* Assign mbuf to the last used desc */ 300 txq->sw_ring[(added - 1) & ptr_mask].mbuf = *pktp; 301 } 302 303 if (likely(added != txq->added)) { 304 sfc_ef10_tx_qpush(txq, added, txq->added); 305 txq->added = added; 306 } 307 308 #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE 309 if (!reap_done) 310 sfc_ef10_tx_reap(txq); 311 #endif 312 313 return pktp - &tx_pkts[0]; 314 } 315 316 static uint16_t 317 sfc_ef10_simple_xmit_pkts(void *tx_queue, struct rte_mbuf **tx_pkts, 318 uint16_t nb_pkts) 319 { 320 struct sfc_ef10_txq * const txq = sfc_ef10_txq_by_dp_txq(tx_queue); 321 unsigned int ptr_mask; 322 unsigned int added; 323 unsigned int dma_desc_space; 324 bool reap_done; 325 struct rte_mbuf **pktp; 326 struct rte_mbuf **pktp_end; 327 328 if (unlikely(txq->flags & 329 (SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION))) 330 return 0; 331 332 ptr_mask = txq->ptr_mask; 333 added = txq->added; 334 dma_desc_space = SFC_EF10_TXQ_LIMIT(ptr_mask + 1) - 335 (added - txq->completed); 336 337 reap_done = (dma_desc_space < RTE_MAX(txq->free_thresh, nb_pkts)); 338 if (reap_done) { 339 sfc_ef10_tx_reap(txq); 340 dma_desc_space = SFC_EF10_TXQ_LIMIT(ptr_mask + 1) - 341 (added - txq->completed); 342 } 343 344 pktp_end = &tx_pkts[MIN(nb_pkts, dma_desc_space)]; 345 for (pktp = &tx_pkts[0]; pktp != pktp_end; ++pktp) { 346 struct rte_mbuf *pkt = *pktp; 347 unsigned int id = added & ptr_mask; 348 349 SFC_ASSERT(rte_pktmbuf_data_len(pkt) <= 350 SFC_EF10_TX_DMA_DESC_LEN_MAX); 351 352 sfc_ef10_tx_qdesc_dma_create(rte_mbuf_data_dma_addr(pkt), 353 rte_pktmbuf_data_len(pkt), 354 true, &txq->txq_hw_ring[id]); 355 356 txq->sw_ring[id].mbuf = pkt; 357 358 ++added; 359 } 360 361 if (likely(added != txq->added)) { 362 sfc_ef10_tx_qpush(txq, added, txq->added); 363 txq->added = added; 364 } 365 366 #if SFC_TX_XMIT_PKTS_REAP_AT_LEAST_ONCE 367 if (!reap_done) 368 sfc_ef10_tx_reap(txq); 369 #endif 370 371 return pktp - &tx_pkts[0]; 372 } 373 374 375 static sfc_dp_tx_qcreate_t sfc_ef10_tx_qcreate; 376 static int 377 sfc_ef10_tx_qcreate(uint16_t port_id, uint16_t queue_id, 378 const struct rte_pci_addr *pci_addr, int socket_id, 379 const struct sfc_dp_tx_qcreate_info *info, 380 struct sfc_dp_txq **dp_txqp) 381 { 382 struct sfc_ef10_txq *txq; 383 int rc; 384 385 rc = EINVAL; 386 if (info->txq_entries != info->evq_entries) 387 goto fail_bad_args; 388 389 rc = ENOMEM; 390 txq = rte_zmalloc_socket("sfc-ef10-txq", sizeof(*txq), 391 RTE_CACHE_LINE_SIZE, socket_id); 392 if (txq == NULL) 393 goto fail_txq_alloc; 394 395 sfc_dp_queue_init(&txq->dp.dpq, port_id, queue_id, pci_addr); 396 397 rc = ENOMEM; 398 txq->sw_ring = rte_calloc_socket("sfc-ef10-txq-sw_ring", 399 info->txq_entries, 400 sizeof(*txq->sw_ring), 401 RTE_CACHE_LINE_SIZE, socket_id); 402 if (txq->sw_ring == NULL) 403 goto fail_sw_ring_alloc; 404 405 txq->flags = SFC_EF10_TXQ_NOT_RUNNING; 406 txq->ptr_mask = info->txq_entries - 1; 407 txq->free_thresh = info->free_thresh; 408 txq->txq_hw_ring = info->txq_hw_ring; 409 txq->doorbell = (volatile uint8_t *)info->mem_bar + 410 ER_DZ_TX_DESC_UPD_REG_OFST + 411 info->hw_index * ER_DZ_TX_DESC_UPD_REG_STEP; 412 txq->evq_hw_ring = info->evq_hw_ring; 413 414 *dp_txqp = &txq->dp; 415 return 0; 416 417 fail_sw_ring_alloc: 418 rte_free(txq); 419 420 fail_txq_alloc: 421 fail_bad_args: 422 return rc; 423 } 424 425 static sfc_dp_tx_qdestroy_t sfc_ef10_tx_qdestroy; 426 static void 427 sfc_ef10_tx_qdestroy(struct sfc_dp_txq *dp_txq) 428 { 429 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); 430 431 rte_free(txq->sw_ring); 432 rte_free(txq); 433 } 434 435 static sfc_dp_tx_qstart_t sfc_ef10_tx_qstart; 436 static int 437 sfc_ef10_tx_qstart(struct sfc_dp_txq *dp_txq, unsigned int evq_read_ptr, 438 unsigned int txq_desc_index) 439 { 440 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); 441 442 txq->evq_read_ptr = evq_read_ptr; 443 txq->added = txq->completed = txq_desc_index; 444 445 txq->flags |= SFC_EF10_TXQ_STARTED; 446 txq->flags &= ~(SFC_EF10_TXQ_NOT_RUNNING | SFC_EF10_TXQ_EXCEPTION); 447 448 return 0; 449 } 450 451 static sfc_dp_tx_qstop_t sfc_ef10_tx_qstop; 452 static void 453 sfc_ef10_tx_qstop(struct sfc_dp_txq *dp_txq, unsigned int *evq_read_ptr) 454 { 455 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); 456 457 txq->flags |= SFC_EF10_TXQ_NOT_RUNNING; 458 459 *evq_read_ptr = txq->evq_read_ptr; 460 } 461 462 static sfc_dp_tx_qtx_ev_t sfc_ef10_tx_qtx_ev; 463 static bool 464 sfc_ef10_tx_qtx_ev(struct sfc_dp_txq *dp_txq, __rte_unused unsigned int id) 465 { 466 __rte_unused struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); 467 468 SFC_ASSERT(txq->flags & SFC_EF10_TXQ_NOT_RUNNING); 469 470 /* 471 * It is safe to ignore Tx event since we reap all mbufs on 472 * queue purge anyway. 473 */ 474 475 return false; 476 } 477 478 static sfc_dp_tx_qreap_t sfc_ef10_tx_qreap; 479 static void 480 sfc_ef10_tx_qreap(struct sfc_dp_txq *dp_txq) 481 { 482 struct sfc_ef10_txq *txq = sfc_ef10_txq_by_dp_txq(dp_txq); 483 unsigned int txds; 484 485 for (txds = 0; txds <= txq->ptr_mask; ++txds) { 486 if (txq->sw_ring[txds].mbuf != NULL) { 487 rte_pktmbuf_free(txq->sw_ring[txds].mbuf); 488 txq->sw_ring[txds].mbuf = NULL; 489 } 490 } 491 492 txq->flags &= ~SFC_EF10_TXQ_STARTED; 493 } 494 495 struct sfc_dp_tx sfc_ef10_tx = { 496 .dp = { 497 .name = SFC_KVARG_DATAPATH_EF10, 498 .type = SFC_DP_TX, 499 .hw_fw_caps = SFC_DP_HW_FW_CAP_EF10, 500 }, 501 .features = SFC_DP_TX_FEAT_MULTI_SEG, 502 .qcreate = sfc_ef10_tx_qcreate, 503 .qdestroy = sfc_ef10_tx_qdestroy, 504 .qstart = sfc_ef10_tx_qstart, 505 .qtx_ev = sfc_ef10_tx_qtx_ev, 506 .qstop = sfc_ef10_tx_qstop, 507 .qreap = sfc_ef10_tx_qreap, 508 .pkt_burst = sfc_ef10_xmit_pkts, 509 }; 510 511 struct sfc_dp_tx sfc_ef10_simple_tx = { 512 .dp = { 513 .name = SFC_KVARG_DATAPATH_EF10_SIMPLE, 514 .type = SFC_DP_TX, 515 }, 516 .features = 0, 517 .qcreate = sfc_ef10_tx_qcreate, 518 .qdestroy = sfc_ef10_tx_qdestroy, 519 .qstart = sfc_ef10_tx_qstart, 520 .qtx_ev = sfc_ef10_tx_qtx_ev, 521 .qstop = sfc_ef10_tx_qstop, 522 .qreap = sfc_ef10_tx_qreap, 523 .pkt_burst = sfc_ef10_simple_xmit_pkts, 524 }; 525