1 /*- 2 * BSD LICENSE 3 * 4 * Copyright (c) 2016-2017 Solarflare Communications Inc. 5 * All rights reserved. 6 * 7 * This software was jointly developed between OKTET Labs (under contract 8 * for Solarflare) and Solarflare Communications, Inc. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions are met: 12 * 13 * 1. Redistributions of source code must retain the above copyright notice, 14 * this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright notice, 16 * this list of conditions and the following disclaimer in the documentation 17 * and/or other materials provided with the distribution. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE COPYRIGHT HOLDERS AND CONTRIBUTORS "AS IS" 20 * AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, 21 * THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 22 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE COPYRIGHT OWNER OR 23 * CONTRIBUTORS BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, 24 * EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, 25 * PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; 26 * OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, 27 * WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR 28 * OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, 29 * EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 30 */ 31 32 #include <rte_debug.h> 33 #include <rte_cycles.h> 34 #include <rte_alarm.h> 35 #include <rte_branch_prediction.h> 36 37 #include "efx.h" 38 39 #include "sfc.h" 40 #include "sfc_debug.h" 41 #include "sfc_log.h" 42 #include "sfc_ev.h" 43 #include "sfc_rx.h" 44 #include "sfc_tx.h" 45 #include "sfc_kvargs.h" 46 47 48 /* Initial delay when waiting for event queue init complete event */ 49 #define SFC_EVQ_INIT_BACKOFF_START_US (1) 50 /* Maximum delay between event queue polling attempts */ 51 #define SFC_EVQ_INIT_BACKOFF_MAX_US (10 * 1000) 52 /* Event queue init approx timeout */ 53 #define SFC_EVQ_INIT_TIMEOUT_US (2 * US_PER_S) 54 55 /* Management event queue polling period in microseconds */ 56 #define SFC_MGMT_EV_QPOLL_PERIOD_US (US_PER_S) 57 58 static const char * 59 sfc_evq_type2str(enum sfc_evq_type type) 60 { 61 switch (type) { 62 case SFC_EVQ_TYPE_MGMT: 63 return "mgmt-evq"; 64 case SFC_EVQ_TYPE_RX: 65 return "rx-evq"; 66 case SFC_EVQ_TYPE_TX: 67 return "tx-evq"; 68 default: 69 SFC_ASSERT(B_FALSE); 70 return NULL; 71 } 72 } 73 74 static boolean_t 75 sfc_ev_initialized(void *arg) 76 { 77 struct sfc_evq *evq = arg; 78 79 /* Init done events may be duplicated on SFN7xxx (SFC bug 31631) */ 80 SFC_ASSERT(evq->init_state == SFC_EVQ_STARTING || 81 evq->init_state == SFC_EVQ_STARTED); 82 83 evq->init_state = SFC_EVQ_STARTED; 84 85 return B_FALSE; 86 } 87 88 static boolean_t 89 sfc_ev_nop_rx(void *arg, uint32_t label, uint32_t id, 90 uint32_t size, uint16_t flags) 91 { 92 struct sfc_evq *evq = arg; 93 94 sfc_err(evq->sa, 95 "EVQ %u unexpected Rx event label=%u id=%#x size=%u flags=%#x", 96 evq->evq_index, label, id, size, flags); 97 return B_TRUE; 98 } 99 100 static boolean_t 101 sfc_ev_efx_rx(void *arg, __rte_unused uint32_t label, uint32_t id, 102 uint32_t size, uint16_t flags) 103 { 104 struct sfc_evq *evq = arg; 105 struct sfc_efx_rxq *rxq; 106 unsigned int stop; 107 unsigned int pending_id; 108 unsigned int delta; 109 unsigned int i; 110 struct sfc_efx_rx_sw_desc *rxd; 111 112 if (unlikely(evq->exception)) 113 goto done; 114 115 rxq = sfc_efx_rxq_by_dp_rxq(evq->dp_rxq); 116 117 SFC_ASSERT(rxq != NULL); 118 SFC_ASSERT(rxq->evq == evq); 119 SFC_ASSERT(rxq->flags & SFC_EFX_RXQ_FLAG_STARTED); 120 121 stop = (id + 1) & rxq->ptr_mask; 122 pending_id = rxq->pending & rxq->ptr_mask; 123 delta = (stop >= pending_id) ? (stop - pending_id) : 124 (rxq->ptr_mask + 1 - pending_id + stop); 125 126 if (delta == 0) { 127 /* 128 * Rx event with no new descriptors done and zero length 129 * is used to abort scattered packet when there is no room 130 * for the tail. 131 */ 132 if (unlikely(size != 0)) { 133 evq->exception = B_TRUE; 134 sfc_err(evq->sa, 135 "EVQ %u RxQ %u invalid RX abort " 136 "(id=%#x size=%u flags=%#x); needs restart", 137 evq->evq_index, rxq->dp.dpq.queue_id, 138 id, size, flags); 139 goto done; 140 } 141 142 /* Add discard flag to the first fragment */ 143 rxq->sw_desc[pending_id].flags |= EFX_DISCARD; 144 /* Remove continue flag from the last fragment */ 145 rxq->sw_desc[id].flags &= ~EFX_PKT_CONT; 146 } else if (unlikely(delta > rxq->batch_max)) { 147 evq->exception = B_TRUE; 148 149 sfc_err(evq->sa, 150 "EVQ %u RxQ %u completion out of order " 151 "(id=%#x delta=%u flags=%#x); needs restart", 152 evq->evq_index, rxq->dp.dpq.queue_id, 153 id, delta, flags); 154 155 goto done; 156 } 157 158 for (i = pending_id; i != stop; i = (i + 1) & rxq->ptr_mask) { 159 rxd = &rxq->sw_desc[i]; 160 161 rxd->flags = flags; 162 163 SFC_ASSERT(size < (1 << 16)); 164 rxd->size = (uint16_t)size; 165 } 166 167 rxq->pending += delta; 168 169 done: 170 return B_FALSE; 171 } 172 173 static boolean_t 174 sfc_ev_dp_rx(void *arg, __rte_unused uint32_t label, uint32_t id, 175 __rte_unused uint32_t size, __rte_unused uint16_t flags) 176 { 177 struct sfc_evq *evq = arg; 178 struct sfc_dp_rxq *dp_rxq; 179 180 dp_rxq = evq->dp_rxq; 181 SFC_ASSERT(dp_rxq != NULL); 182 183 SFC_ASSERT(evq->sa->dp_rx->qrx_ev != NULL); 184 return evq->sa->dp_rx->qrx_ev(dp_rxq, id); 185 } 186 187 static boolean_t 188 sfc_ev_nop_tx(void *arg, uint32_t label, uint32_t id) 189 { 190 struct sfc_evq *evq = arg; 191 192 sfc_err(evq->sa, "EVQ %u unexpected Tx event label=%u id=%#x", 193 evq->evq_index, label, id); 194 return B_TRUE; 195 } 196 197 static boolean_t 198 sfc_ev_tx(void *arg, __rte_unused uint32_t label, uint32_t id) 199 { 200 struct sfc_evq *evq = arg; 201 struct sfc_dp_txq *dp_txq; 202 struct sfc_efx_txq *txq; 203 unsigned int stop; 204 unsigned int delta; 205 206 dp_txq = evq->dp_txq; 207 SFC_ASSERT(dp_txq != NULL); 208 209 txq = sfc_efx_txq_by_dp_txq(dp_txq); 210 SFC_ASSERT(txq->evq == evq); 211 212 if (unlikely((txq->flags & SFC_EFX_TXQ_FLAG_STARTED) == 0)) 213 goto done; 214 215 stop = (id + 1) & txq->ptr_mask; 216 id = txq->pending & txq->ptr_mask; 217 218 delta = (stop >= id) ? (stop - id) : (txq->ptr_mask + 1 - id + stop); 219 220 txq->pending += delta; 221 222 done: 223 return B_FALSE; 224 } 225 226 static boolean_t 227 sfc_ev_dp_tx(void *arg, __rte_unused uint32_t label, uint32_t id) 228 { 229 struct sfc_evq *evq = arg; 230 struct sfc_dp_txq *dp_txq; 231 232 dp_txq = evq->dp_txq; 233 SFC_ASSERT(dp_txq != NULL); 234 235 SFC_ASSERT(evq->sa->dp_tx->qtx_ev != NULL); 236 return evq->sa->dp_tx->qtx_ev(dp_txq, id); 237 } 238 239 static boolean_t 240 sfc_ev_exception(void *arg, __rte_unused uint32_t code, 241 __rte_unused uint32_t data) 242 { 243 struct sfc_evq *evq = arg; 244 245 if (code == EFX_EXCEPTION_UNKNOWN_SENSOREVT) 246 return B_FALSE; 247 248 evq->exception = B_TRUE; 249 sfc_warn(evq->sa, 250 "hardware exception %s (code=%u, data=%#x) on EVQ %u;" 251 " needs recovery", 252 (code == EFX_EXCEPTION_RX_RECOVERY) ? "RX_RECOVERY" : 253 (code == EFX_EXCEPTION_RX_DSC_ERROR) ? "RX_DSC_ERROR" : 254 (code == EFX_EXCEPTION_TX_DSC_ERROR) ? "TX_DSC_ERROR" : 255 (code == EFX_EXCEPTION_FWALERT_SRAM) ? "FWALERT_SRAM" : 256 (code == EFX_EXCEPTION_UNKNOWN_FWALERT) ? "UNKNOWN_FWALERT" : 257 (code == EFX_EXCEPTION_RX_ERROR) ? "RX_ERROR" : 258 (code == EFX_EXCEPTION_TX_ERROR) ? "TX_ERROR" : 259 (code == EFX_EXCEPTION_EV_ERROR) ? "EV_ERROR" : 260 "UNKNOWN", 261 code, data, evq->evq_index); 262 263 return B_TRUE; 264 } 265 266 static boolean_t 267 sfc_ev_nop_rxq_flush_done(void *arg, uint32_t rxq_hw_index) 268 { 269 struct sfc_evq *evq = arg; 270 271 sfc_err(evq->sa, "EVQ %u unexpected RxQ %u flush done", 272 evq->evq_index, rxq_hw_index); 273 return B_TRUE; 274 } 275 276 static boolean_t 277 sfc_ev_rxq_flush_done(void *arg, __rte_unused uint32_t rxq_hw_index) 278 { 279 struct sfc_evq *evq = arg; 280 struct sfc_dp_rxq *dp_rxq; 281 struct sfc_rxq *rxq; 282 283 dp_rxq = evq->dp_rxq; 284 SFC_ASSERT(dp_rxq != NULL); 285 286 rxq = sfc_rxq_by_dp_rxq(dp_rxq); 287 SFC_ASSERT(rxq != NULL); 288 SFC_ASSERT(rxq->hw_index == rxq_hw_index); 289 SFC_ASSERT(rxq->evq == evq); 290 sfc_rx_qflush_done(rxq); 291 292 return B_FALSE; 293 } 294 295 static boolean_t 296 sfc_ev_nop_rxq_flush_failed(void *arg, uint32_t rxq_hw_index) 297 { 298 struct sfc_evq *evq = arg; 299 300 sfc_err(evq->sa, "EVQ %u unexpected RxQ %u flush failed", 301 evq->evq_index, rxq_hw_index); 302 return B_TRUE; 303 } 304 305 static boolean_t 306 sfc_ev_rxq_flush_failed(void *arg, __rte_unused uint32_t rxq_hw_index) 307 { 308 struct sfc_evq *evq = arg; 309 struct sfc_dp_rxq *dp_rxq; 310 struct sfc_rxq *rxq; 311 312 dp_rxq = evq->dp_rxq; 313 SFC_ASSERT(dp_rxq != NULL); 314 315 rxq = sfc_rxq_by_dp_rxq(dp_rxq); 316 SFC_ASSERT(rxq != NULL); 317 SFC_ASSERT(rxq->hw_index == rxq_hw_index); 318 SFC_ASSERT(rxq->evq == evq); 319 sfc_rx_qflush_failed(rxq); 320 321 return B_FALSE; 322 } 323 324 static boolean_t 325 sfc_ev_nop_txq_flush_done(void *arg, uint32_t txq_hw_index) 326 { 327 struct sfc_evq *evq = arg; 328 329 sfc_err(evq->sa, "EVQ %u unexpected TxQ %u flush done", 330 evq->evq_index, txq_hw_index); 331 return B_TRUE; 332 } 333 334 static boolean_t 335 sfc_ev_txq_flush_done(void *arg, __rte_unused uint32_t txq_hw_index) 336 { 337 struct sfc_evq *evq = arg; 338 struct sfc_dp_txq *dp_txq; 339 struct sfc_txq *txq; 340 341 dp_txq = evq->dp_txq; 342 SFC_ASSERT(dp_txq != NULL); 343 344 txq = sfc_txq_by_dp_txq(dp_txq); 345 SFC_ASSERT(txq != NULL); 346 SFC_ASSERT(txq->hw_index == txq_hw_index); 347 SFC_ASSERT(txq->evq == evq); 348 sfc_tx_qflush_done(txq); 349 350 return B_FALSE; 351 } 352 353 static boolean_t 354 sfc_ev_software(void *arg, uint16_t magic) 355 { 356 struct sfc_evq *evq = arg; 357 358 sfc_err(evq->sa, "EVQ %u unexpected software event magic=%#.4x", 359 evq->evq_index, magic); 360 return B_TRUE; 361 } 362 363 static boolean_t 364 sfc_ev_sram(void *arg, uint32_t code) 365 { 366 struct sfc_evq *evq = arg; 367 368 sfc_err(evq->sa, "EVQ %u unexpected SRAM event code=%u", 369 evq->evq_index, code); 370 return B_TRUE; 371 } 372 373 static boolean_t 374 sfc_ev_wake_up(void *arg, uint32_t index) 375 { 376 struct sfc_evq *evq = arg; 377 378 sfc_err(evq->sa, "EVQ %u unexpected wake up event index=%u", 379 evq->evq_index, index); 380 return B_TRUE; 381 } 382 383 static boolean_t 384 sfc_ev_timer(void *arg, uint32_t index) 385 { 386 struct sfc_evq *evq = arg; 387 388 sfc_err(evq->sa, "EVQ %u unexpected timer event index=%u", 389 evq->evq_index, index); 390 return B_TRUE; 391 } 392 393 static boolean_t 394 sfc_ev_nop_link_change(void *arg, __rte_unused efx_link_mode_t link_mode) 395 { 396 struct sfc_evq *evq = arg; 397 398 sfc_err(evq->sa, "EVQ %u unexpected link change event", 399 evq->evq_index); 400 return B_TRUE; 401 } 402 403 static boolean_t 404 sfc_ev_link_change(void *arg, efx_link_mode_t link_mode) 405 { 406 struct sfc_evq *evq = arg; 407 struct sfc_adapter *sa = evq->sa; 408 struct rte_eth_link *dev_link = &sa->eth_dev->data->dev_link; 409 struct rte_eth_link new_link; 410 uint64_t new_link_u64; 411 uint64_t old_link_u64; 412 413 EFX_STATIC_ASSERT(sizeof(*dev_link) == sizeof(rte_atomic64_t)); 414 415 sfc_port_link_mode_to_info(link_mode, &new_link); 416 417 new_link_u64 = *(uint64_t *)&new_link; 418 do { 419 old_link_u64 = rte_atomic64_read((rte_atomic64_t *)dev_link); 420 if (old_link_u64 == new_link_u64) 421 break; 422 423 if (rte_atomic64_cmpset((volatile uint64_t *)dev_link, 424 old_link_u64, new_link_u64)) { 425 evq->sa->port.lsc_seq++; 426 break; 427 } 428 } while (B_TRUE); 429 430 return B_FALSE; 431 } 432 433 static const efx_ev_callbacks_t sfc_ev_callbacks = { 434 .eec_initialized = sfc_ev_initialized, 435 .eec_rx = sfc_ev_nop_rx, 436 .eec_tx = sfc_ev_nop_tx, 437 .eec_exception = sfc_ev_exception, 438 .eec_rxq_flush_done = sfc_ev_nop_rxq_flush_done, 439 .eec_rxq_flush_failed = sfc_ev_nop_rxq_flush_failed, 440 .eec_txq_flush_done = sfc_ev_nop_txq_flush_done, 441 .eec_software = sfc_ev_software, 442 .eec_sram = sfc_ev_sram, 443 .eec_wake_up = sfc_ev_wake_up, 444 .eec_timer = sfc_ev_timer, 445 .eec_link_change = sfc_ev_link_change, 446 }; 447 448 static const efx_ev_callbacks_t sfc_ev_callbacks_efx_rx = { 449 .eec_initialized = sfc_ev_initialized, 450 .eec_rx = sfc_ev_efx_rx, 451 .eec_tx = sfc_ev_nop_tx, 452 .eec_exception = sfc_ev_exception, 453 .eec_rxq_flush_done = sfc_ev_rxq_flush_done, 454 .eec_rxq_flush_failed = sfc_ev_rxq_flush_failed, 455 .eec_txq_flush_done = sfc_ev_nop_txq_flush_done, 456 .eec_software = sfc_ev_software, 457 .eec_sram = sfc_ev_sram, 458 .eec_wake_up = sfc_ev_wake_up, 459 .eec_timer = sfc_ev_timer, 460 .eec_link_change = sfc_ev_nop_link_change, 461 }; 462 463 static const efx_ev_callbacks_t sfc_ev_callbacks_dp_rx = { 464 .eec_initialized = sfc_ev_initialized, 465 .eec_rx = sfc_ev_dp_rx, 466 .eec_tx = sfc_ev_nop_tx, 467 .eec_exception = sfc_ev_exception, 468 .eec_rxq_flush_done = sfc_ev_rxq_flush_done, 469 .eec_rxq_flush_failed = sfc_ev_rxq_flush_failed, 470 .eec_txq_flush_done = sfc_ev_nop_txq_flush_done, 471 .eec_software = sfc_ev_software, 472 .eec_sram = sfc_ev_sram, 473 .eec_wake_up = sfc_ev_wake_up, 474 .eec_timer = sfc_ev_timer, 475 .eec_link_change = sfc_ev_nop_link_change, 476 }; 477 478 static const efx_ev_callbacks_t sfc_ev_callbacks_efx_tx = { 479 .eec_initialized = sfc_ev_initialized, 480 .eec_rx = sfc_ev_nop_rx, 481 .eec_tx = sfc_ev_tx, 482 .eec_exception = sfc_ev_exception, 483 .eec_rxq_flush_done = sfc_ev_nop_rxq_flush_done, 484 .eec_rxq_flush_failed = sfc_ev_nop_rxq_flush_failed, 485 .eec_txq_flush_done = sfc_ev_txq_flush_done, 486 .eec_software = sfc_ev_software, 487 .eec_sram = sfc_ev_sram, 488 .eec_wake_up = sfc_ev_wake_up, 489 .eec_timer = sfc_ev_timer, 490 .eec_link_change = sfc_ev_nop_link_change, 491 }; 492 493 static const efx_ev_callbacks_t sfc_ev_callbacks_dp_tx = { 494 .eec_initialized = sfc_ev_initialized, 495 .eec_rx = sfc_ev_nop_rx, 496 .eec_tx = sfc_ev_dp_tx, 497 .eec_exception = sfc_ev_exception, 498 .eec_rxq_flush_done = sfc_ev_nop_rxq_flush_done, 499 .eec_rxq_flush_failed = sfc_ev_nop_rxq_flush_failed, 500 .eec_txq_flush_done = sfc_ev_txq_flush_done, 501 .eec_software = sfc_ev_software, 502 .eec_sram = sfc_ev_sram, 503 .eec_wake_up = sfc_ev_wake_up, 504 .eec_timer = sfc_ev_timer, 505 .eec_link_change = sfc_ev_nop_link_change, 506 }; 507 508 509 void 510 sfc_ev_qpoll(struct sfc_evq *evq) 511 { 512 SFC_ASSERT(evq->init_state == SFC_EVQ_STARTED || 513 evq->init_state == SFC_EVQ_STARTING); 514 515 /* Synchronize the DMA memory for reading not required */ 516 517 efx_ev_qpoll(evq->common, &evq->read_ptr, evq->callbacks, evq); 518 519 if (unlikely(evq->exception) && sfc_adapter_trylock(evq->sa)) { 520 struct sfc_adapter *sa = evq->sa; 521 int rc; 522 523 if (evq->dp_rxq != NULL) { 524 unsigned int rxq_sw_index; 525 526 rxq_sw_index = evq->dp_rxq->dpq.queue_id; 527 528 sfc_warn(sa, 529 "restart RxQ %u because of exception on its EvQ %u", 530 rxq_sw_index, evq->evq_index); 531 532 sfc_rx_qstop(sa, rxq_sw_index); 533 rc = sfc_rx_qstart(sa, rxq_sw_index); 534 if (rc != 0) 535 sfc_err(sa, "cannot restart RxQ %u", 536 rxq_sw_index); 537 } 538 539 if (evq->dp_txq != NULL) { 540 unsigned int txq_sw_index; 541 542 txq_sw_index = evq->dp_txq->dpq.queue_id; 543 544 sfc_warn(sa, 545 "restart TxQ %u because of exception on its EvQ %u", 546 txq_sw_index, evq->evq_index); 547 548 sfc_tx_qstop(sa, txq_sw_index); 549 rc = sfc_tx_qstart(sa, txq_sw_index); 550 if (rc != 0) 551 sfc_err(sa, "cannot restart TxQ %u", 552 txq_sw_index); 553 } 554 555 if (evq->exception) 556 sfc_panic(sa, "unrecoverable exception on EvQ %u", 557 evq->evq_index); 558 559 sfc_adapter_unlock(sa); 560 } 561 562 /* Poll-mode driver does not re-prime the event queue for interrupts */ 563 } 564 565 void 566 sfc_ev_mgmt_qpoll(struct sfc_adapter *sa) 567 { 568 if (rte_spinlock_trylock(&sa->mgmt_evq_lock)) { 569 struct sfc_evq *mgmt_evq = sa->mgmt_evq; 570 571 if (mgmt_evq->init_state == SFC_EVQ_STARTED) 572 sfc_ev_qpoll(mgmt_evq); 573 574 rte_spinlock_unlock(&sa->mgmt_evq_lock); 575 } 576 } 577 578 int 579 sfc_ev_qprime(struct sfc_evq *evq) 580 { 581 SFC_ASSERT(evq->init_state == SFC_EVQ_STARTED); 582 return efx_ev_qprime(evq->common, evq->read_ptr); 583 } 584 585 /* Event queue HW index allocation scheme is described in sfc_ev.h. */ 586 int 587 sfc_ev_qstart(struct sfc_evq *evq, unsigned int hw_index) 588 { 589 struct sfc_adapter *sa = evq->sa; 590 efsys_mem_t *esmp; 591 uint32_t evq_flags = sa->evq_flags; 592 unsigned int total_delay_us; 593 unsigned int delay_us; 594 int rc; 595 596 sfc_log_init(sa, "hw_index=%u", hw_index); 597 598 esmp = &evq->mem; 599 600 evq->evq_index = hw_index; 601 602 /* Clear all events */ 603 (void)memset((void *)esmp->esm_base, 0xff, EFX_EVQ_SIZE(evq->entries)); 604 605 if (sa->intr.lsc_intr && hw_index == sa->mgmt_evq_index) 606 evq_flags |= EFX_EVQ_FLAGS_NOTIFY_INTERRUPT; 607 else 608 evq_flags |= EFX_EVQ_FLAGS_NOTIFY_DISABLED; 609 610 /* Create the common code event queue */ 611 rc = efx_ev_qcreate(sa->nic, hw_index, esmp, evq->entries, 612 0 /* unused on EF10 */, 0, evq_flags, 613 &evq->common); 614 if (rc != 0) 615 goto fail_ev_qcreate; 616 617 SFC_ASSERT(evq->dp_rxq == NULL || evq->dp_txq == NULL); 618 if (evq->dp_rxq != 0) { 619 if (strcmp(sa->dp_rx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0) 620 evq->callbacks = &sfc_ev_callbacks_efx_rx; 621 else 622 evq->callbacks = &sfc_ev_callbacks_dp_rx; 623 } else if (evq->dp_txq != 0) { 624 if (strcmp(sa->dp_tx->dp.name, SFC_KVARG_DATAPATH_EFX) == 0) 625 evq->callbacks = &sfc_ev_callbacks_efx_tx; 626 else 627 evq->callbacks = &sfc_ev_callbacks_dp_tx; 628 } else { 629 evq->callbacks = &sfc_ev_callbacks; 630 } 631 632 evq->init_state = SFC_EVQ_STARTING; 633 634 /* Wait for the initialization event */ 635 total_delay_us = 0; 636 delay_us = SFC_EVQ_INIT_BACKOFF_START_US; 637 do { 638 (void)sfc_ev_qpoll(evq); 639 640 /* Check to see if the initialization complete indication 641 * posted by the hardware. 642 */ 643 if (evq->init_state == SFC_EVQ_STARTED) 644 goto done; 645 646 /* Give event queue some time to init */ 647 rte_delay_us(delay_us); 648 649 total_delay_us += delay_us; 650 651 /* Exponential backoff */ 652 delay_us *= 2; 653 if (delay_us > SFC_EVQ_INIT_BACKOFF_MAX_US) 654 delay_us = SFC_EVQ_INIT_BACKOFF_MAX_US; 655 656 } while (total_delay_us < SFC_EVQ_INIT_TIMEOUT_US); 657 658 rc = ETIMEDOUT; 659 goto fail_timedout; 660 661 done: 662 return 0; 663 664 fail_timedout: 665 evq->init_state = SFC_EVQ_INITIALIZED; 666 efx_ev_qdestroy(evq->common); 667 668 fail_ev_qcreate: 669 sfc_log_init(sa, "failed %d", rc); 670 return rc; 671 } 672 673 void 674 sfc_ev_qstop(struct sfc_evq *evq) 675 { 676 if (evq == NULL) 677 return; 678 679 sfc_log_init(evq->sa, "hw_index=%u", evq->evq_index); 680 681 if (evq->init_state != SFC_EVQ_STARTED) 682 return; 683 684 evq->init_state = SFC_EVQ_INITIALIZED; 685 evq->callbacks = NULL; 686 evq->read_ptr = 0; 687 evq->exception = B_FALSE; 688 689 efx_ev_qdestroy(evq->common); 690 691 evq->evq_index = 0; 692 } 693 694 static void 695 sfc_ev_mgmt_periodic_qpoll(void *arg) 696 { 697 struct sfc_adapter *sa = arg; 698 int rc; 699 700 sfc_ev_mgmt_qpoll(sa); 701 702 rc = rte_eal_alarm_set(SFC_MGMT_EV_QPOLL_PERIOD_US, 703 sfc_ev_mgmt_periodic_qpoll, sa); 704 if (rc == -ENOTSUP) { 705 sfc_warn(sa, "alarms are not supported"); 706 sfc_warn(sa, "management EVQ must be polled indirectly using no-wait link status update"); 707 } else if (rc != 0) { 708 sfc_err(sa, 709 "cannot rearm management EVQ polling alarm (rc=%d)", 710 rc); 711 } 712 } 713 714 static void 715 sfc_ev_mgmt_periodic_qpoll_start(struct sfc_adapter *sa) 716 { 717 sfc_ev_mgmt_periodic_qpoll(sa); 718 } 719 720 static void 721 sfc_ev_mgmt_periodic_qpoll_stop(struct sfc_adapter *sa) 722 { 723 rte_eal_alarm_cancel(sfc_ev_mgmt_periodic_qpoll, sa); 724 } 725 726 int 727 sfc_ev_start(struct sfc_adapter *sa) 728 { 729 int rc; 730 731 sfc_log_init(sa, "entry"); 732 733 rc = efx_ev_init(sa->nic); 734 if (rc != 0) 735 goto fail_ev_init; 736 737 /* Start management EVQ used for global events */ 738 rte_spinlock_lock(&sa->mgmt_evq_lock); 739 740 rc = sfc_ev_qstart(sa->mgmt_evq, sa->mgmt_evq_index); 741 if (rc != 0) 742 goto fail_mgmt_evq_start; 743 744 if (sa->intr.lsc_intr) { 745 rc = sfc_ev_qprime(sa->mgmt_evq); 746 if (rc != 0) 747 goto fail_evq0_prime; 748 } 749 750 rte_spinlock_unlock(&sa->mgmt_evq_lock); 751 752 /* 753 * Start management EVQ polling. If interrupts are disabled 754 * (not used), it is required to process link status change 755 * and other device level events to avoid unrecoverable 756 * error because the event queue overflow. 757 */ 758 sfc_ev_mgmt_periodic_qpoll_start(sa); 759 760 /* 761 * Rx/Tx event queues are started/stopped when corresponding 762 * Rx/Tx queue is started/stopped. 763 */ 764 765 return 0; 766 767 fail_evq0_prime: 768 sfc_ev_qstop(sa->mgmt_evq); 769 770 fail_mgmt_evq_start: 771 rte_spinlock_unlock(&sa->mgmt_evq_lock); 772 efx_ev_fini(sa->nic); 773 774 fail_ev_init: 775 sfc_log_init(sa, "failed %d", rc); 776 return rc; 777 } 778 779 void 780 sfc_ev_stop(struct sfc_adapter *sa) 781 { 782 sfc_log_init(sa, "entry"); 783 784 sfc_ev_mgmt_periodic_qpoll_stop(sa); 785 786 rte_spinlock_lock(&sa->mgmt_evq_lock); 787 sfc_ev_qstop(sa->mgmt_evq); 788 rte_spinlock_unlock(&sa->mgmt_evq_lock); 789 790 efx_ev_fini(sa->nic); 791 } 792 793 int 794 sfc_ev_qinit(struct sfc_adapter *sa, 795 enum sfc_evq_type type, unsigned int type_index, 796 unsigned int entries, int socket_id, struct sfc_evq **evqp) 797 { 798 struct sfc_evq *evq; 799 int rc; 800 801 sfc_log_init(sa, "type=%s type_index=%u", 802 sfc_evq_type2str(type), type_index); 803 804 SFC_ASSERT(rte_is_power_of_2(entries)); 805 806 rc = ENOMEM; 807 evq = rte_zmalloc_socket("sfc-evq", sizeof(*evq), RTE_CACHE_LINE_SIZE, 808 socket_id); 809 if (evq == NULL) 810 goto fail_evq_alloc; 811 812 evq->sa = sa; 813 evq->type = type; 814 evq->entries = entries; 815 816 /* Allocate DMA space */ 817 rc = sfc_dma_alloc(sa, sfc_evq_type2str(type), type_index, 818 EFX_EVQ_SIZE(evq->entries), socket_id, &evq->mem); 819 if (rc != 0) 820 goto fail_dma_alloc; 821 822 evq->init_state = SFC_EVQ_INITIALIZED; 823 824 sa->evq_count++; 825 826 *evqp = evq; 827 828 return 0; 829 830 fail_dma_alloc: 831 rte_free(evq); 832 833 fail_evq_alloc: 834 835 sfc_log_init(sa, "failed %d", rc); 836 return rc; 837 } 838 839 void 840 sfc_ev_qfini(struct sfc_evq *evq) 841 { 842 struct sfc_adapter *sa = evq->sa; 843 844 SFC_ASSERT(evq->init_state == SFC_EVQ_INITIALIZED); 845 846 sfc_dma_free(sa, &evq->mem); 847 848 rte_free(evq); 849 850 SFC_ASSERT(sa->evq_count > 0); 851 sa->evq_count--; 852 } 853 854 static int 855 sfc_kvarg_perf_profile_handler(__rte_unused const char *key, 856 const char *value_str, void *opaque) 857 { 858 uint64_t *value = opaque; 859 860 if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_THROUGHPUT) == 0) 861 *value = EFX_EVQ_FLAGS_TYPE_THROUGHPUT; 862 else if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_LOW_LATENCY) == 0) 863 *value = EFX_EVQ_FLAGS_TYPE_LOW_LATENCY; 864 else if (strcasecmp(value_str, SFC_KVARG_PERF_PROFILE_AUTO) == 0) 865 *value = EFX_EVQ_FLAGS_TYPE_AUTO; 866 else 867 return -EINVAL; 868 869 return 0; 870 } 871 872 int 873 sfc_ev_attach(struct sfc_adapter *sa) 874 { 875 int rc; 876 877 sfc_log_init(sa, "entry"); 878 879 sa->evq_flags = EFX_EVQ_FLAGS_TYPE_THROUGHPUT; 880 rc = sfc_kvargs_process(sa, SFC_KVARG_PERF_PROFILE, 881 sfc_kvarg_perf_profile_handler, 882 &sa->evq_flags); 883 if (rc != 0) { 884 sfc_err(sa, "invalid %s parameter value", 885 SFC_KVARG_PERF_PROFILE); 886 goto fail_kvarg_perf_profile; 887 } 888 889 sa->mgmt_evq_index = 0; 890 rte_spinlock_init(&sa->mgmt_evq_lock); 891 892 rc = sfc_ev_qinit(sa, SFC_EVQ_TYPE_MGMT, 0, SFC_MGMT_EVQ_ENTRIES, 893 sa->socket_id, &sa->mgmt_evq); 894 if (rc != 0) 895 goto fail_mgmt_evq_init; 896 897 /* 898 * Rx/Tx event queues are created/destroyed when corresponding 899 * Rx/Tx queue is created/destroyed. 900 */ 901 902 return 0; 903 904 fail_mgmt_evq_init: 905 906 fail_kvarg_perf_profile: 907 sfc_log_init(sa, "failed %d", rc); 908 return rc; 909 } 910 911 void 912 sfc_ev_detach(struct sfc_adapter *sa) 913 { 914 sfc_log_init(sa, "entry"); 915 916 sfc_ev_qfini(sa->mgmt_evq); 917 918 if (sa->evq_count != 0) 919 sfc_err(sa, "%u EvQs are not destroyed before detach", 920 sa->evq_count); 921 } 922