1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2007-2019 Solarflare Communications Inc. 5 */ 6 7 #include "efx.h" 8 #include "efx_impl.h" 9 10 #if EFSYS_OPT_QSTATS 11 #define EFX_TX_QSTAT_INCR(_etp, _stat) \ 12 do { \ 13 (_etp)->et_stat[_stat]++; \ 14 _NOTE(CONSTANTCONDITION) \ 15 } while (B_FALSE) 16 #else 17 #define EFX_TX_QSTAT_INCR(_etp, _stat) 18 #endif 19 20 #if EFSYS_OPT_SIENA 21 22 static __checkReturn efx_rc_t 23 siena_tx_init( 24 __in efx_nic_t *enp); 25 26 static void 27 siena_tx_fini( 28 __in efx_nic_t *enp); 29 30 static __checkReturn efx_rc_t 31 siena_tx_qcreate( 32 __in efx_nic_t *enp, 33 __in unsigned int index, 34 __in unsigned int label, 35 __in efsys_mem_t *esmp, 36 __in size_t ndescs, 37 __in uint32_t id, 38 __in uint16_t flags, 39 __in efx_evq_t *eep, 40 __in efx_txq_t *etp, 41 __out unsigned int *addedp); 42 43 static void 44 siena_tx_qdestroy( 45 __in efx_txq_t *etp); 46 47 static __checkReturn efx_rc_t 48 siena_tx_qpost( 49 __in efx_txq_t *etp, 50 __in_ecount(ndescs) efx_buffer_t *eb, 51 __in unsigned int ndescs, 52 __in unsigned int completed, 53 __inout unsigned int *addedp); 54 55 static void 56 siena_tx_qpush( 57 __in efx_txq_t *etp, 58 __in unsigned int added, 59 __in unsigned int pushed); 60 61 static __checkReturn efx_rc_t 62 siena_tx_qpace( 63 __in efx_txq_t *etp, 64 __in unsigned int ns); 65 66 static __checkReturn efx_rc_t 67 siena_tx_qflush( 68 __in efx_txq_t *etp); 69 70 static void 71 siena_tx_qenable( 72 __in efx_txq_t *etp); 73 74 __checkReturn efx_rc_t 75 siena_tx_qdesc_post( 76 __in efx_txq_t *etp, 77 __in_ecount(ndescs) efx_desc_t *ed, 78 __in unsigned int ndescs, 79 __in unsigned int completed, 80 __inout unsigned int *addedp); 81 82 void 83 siena_tx_qdesc_dma_create( 84 __in efx_txq_t *etp, 85 __in efsys_dma_addr_t addr, 86 __in size_t size, 87 __in boolean_t eop, 88 __out efx_desc_t *edp); 89 90 #if EFSYS_OPT_QSTATS 91 static void 92 siena_tx_qstats_update( 93 __in efx_txq_t *etp, 94 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat); 95 #endif 96 97 #endif /* EFSYS_OPT_SIENA */ 98 99 100 #if EFSYS_OPT_SIENA 101 static const efx_tx_ops_t __efx_tx_siena_ops = { 102 siena_tx_init, /* etxo_init */ 103 siena_tx_fini, /* etxo_fini */ 104 siena_tx_qcreate, /* etxo_qcreate */ 105 siena_tx_qdestroy, /* etxo_qdestroy */ 106 siena_tx_qpost, /* etxo_qpost */ 107 siena_tx_qpush, /* etxo_qpush */ 108 siena_tx_qpace, /* etxo_qpace */ 109 siena_tx_qflush, /* etxo_qflush */ 110 siena_tx_qenable, /* etxo_qenable */ 111 NULL, /* etxo_qpio_enable */ 112 NULL, /* etxo_qpio_disable */ 113 NULL, /* etxo_qpio_write */ 114 NULL, /* etxo_qpio_post */ 115 siena_tx_qdesc_post, /* etxo_qdesc_post */ 116 siena_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 117 NULL, /* etxo_qdesc_tso_create */ 118 NULL, /* etxo_qdesc_tso2_create */ 119 NULL, /* etxo_qdesc_vlantci_create */ 120 NULL, /* etxo_qdesc_checksum_create */ 121 #if EFSYS_OPT_QSTATS 122 siena_tx_qstats_update, /* etxo_qstats_update */ 123 #endif 124 }; 125 #endif /* EFSYS_OPT_SIENA */ 126 127 #if EFSYS_OPT_HUNTINGTON 128 static const efx_tx_ops_t __efx_tx_hunt_ops = { 129 ef10_tx_init, /* etxo_init */ 130 ef10_tx_fini, /* etxo_fini */ 131 ef10_tx_qcreate, /* etxo_qcreate */ 132 ef10_tx_qdestroy, /* etxo_qdestroy */ 133 ef10_tx_qpost, /* etxo_qpost */ 134 ef10_tx_qpush, /* etxo_qpush */ 135 ef10_tx_qpace, /* etxo_qpace */ 136 ef10_tx_qflush, /* etxo_qflush */ 137 ef10_tx_qenable, /* etxo_qenable */ 138 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 139 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 140 ef10_tx_qpio_write, /* etxo_qpio_write */ 141 ef10_tx_qpio_post, /* etxo_qpio_post */ 142 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 143 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 144 ef10_tx_qdesc_tso_create, /* etxo_qdesc_tso_create */ 145 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 146 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 147 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ 148 #if EFSYS_OPT_QSTATS 149 ef10_tx_qstats_update, /* etxo_qstats_update */ 150 #endif 151 }; 152 #endif /* EFSYS_OPT_HUNTINGTON */ 153 154 #if EFSYS_OPT_MEDFORD 155 static const efx_tx_ops_t __efx_tx_medford_ops = { 156 ef10_tx_init, /* etxo_init */ 157 ef10_tx_fini, /* etxo_fini */ 158 ef10_tx_qcreate, /* etxo_qcreate */ 159 ef10_tx_qdestroy, /* etxo_qdestroy */ 160 ef10_tx_qpost, /* etxo_qpost */ 161 ef10_tx_qpush, /* etxo_qpush */ 162 ef10_tx_qpace, /* etxo_qpace */ 163 ef10_tx_qflush, /* etxo_qflush */ 164 ef10_tx_qenable, /* etxo_qenable */ 165 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 166 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 167 ef10_tx_qpio_write, /* etxo_qpio_write */ 168 ef10_tx_qpio_post, /* etxo_qpio_post */ 169 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 170 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 171 NULL, /* etxo_qdesc_tso_create */ 172 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 173 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 174 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ 175 #if EFSYS_OPT_QSTATS 176 ef10_tx_qstats_update, /* etxo_qstats_update */ 177 #endif 178 }; 179 #endif /* EFSYS_OPT_MEDFORD */ 180 181 #if EFSYS_OPT_MEDFORD2 182 static const efx_tx_ops_t __efx_tx_medford2_ops = { 183 ef10_tx_init, /* etxo_init */ 184 ef10_tx_fini, /* etxo_fini */ 185 ef10_tx_qcreate, /* etxo_qcreate */ 186 ef10_tx_qdestroy, /* etxo_qdestroy */ 187 ef10_tx_qpost, /* etxo_qpost */ 188 ef10_tx_qpush, /* etxo_qpush */ 189 ef10_tx_qpace, /* etxo_qpace */ 190 ef10_tx_qflush, /* etxo_qflush */ 191 ef10_tx_qenable, /* etxo_qenable */ 192 ef10_tx_qpio_enable, /* etxo_qpio_enable */ 193 ef10_tx_qpio_disable, /* etxo_qpio_disable */ 194 ef10_tx_qpio_write, /* etxo_qpio_write */ 195 ef10_tx_qpio_post, /* etxo_qpio_post */ 196 ef10_tx_qdesc_post, /* etxo_qdesc_post */ 197 ef10_tx_qdesc_dma_create, /* etxo_qdesc_dma_create */ 198 NULL, /* etxo_qdesc_tso_create */ 199 ef10_tx_qdesc_tso2_create, /* etxo_qdesc_tso2_create */ 200 ef10_tx_qdesc_vlantci_create, /* etxo_qdesc_vlantci_create */ 201 ef10_tx_qdesc_checksum_create, /* etxo_qdesc_checksum_create */ 202 #if EFSYS_OPT_QSTATS 203 ef10_tx_qstats_update, /* etxo_qstats_update */ 204 #endif 205 }; 206 #endif /* EFSYS_OPT_MEDFORD2 */ 207 208 #if EFSYS_OPT_RIVERHEAD 209 static const efx_tx_ops_t __efx_tx_rhead_ops = { 210 rhead_tx_init, /* etxo_init */ 211 rhead_tx_fini, /* etxo_fini */ 212 rhead_tx_qcreate, /* etxo_qcreate */ 213 rhead_tx_qdestroy, /* etxo_qdestroy */ 214 rhead_tx_qpost, /* etxo_qpost */ 215 rhead_tx_qpush, /* etxo_qpush */ 216 rhead_tx_qpace, /* etxo_qpace */ 217 rhead_tx_qflush, /* etxo_qflush */ 218 rhead_tx_qenable, /* etxo_qenable */ 219 NULL, /* etxo_qpio_enable */ 220 NULL, /* etxo_qpio_disable */ 221 NULL, /* etxo_qpio_write */ 222 NULL, /* etxo_qpio_post */ 223 rhead_tx_qdesc_post, /* etxo_qdesc_post */ 224 NULL, /* etxo_qdesc_dma_create */ 225 NULL, /* etxo_qdesc_tso_create */ 226 NULL, /* etxo_qdesc_tso2_create */ 227 NULL, /* etxo_qdesc_vlantci_create */ 228 NULL, /* etxo_qdesc_checksum_create */ 229 #if EFSYS_OPT_QSTATS 230 rhead_tx_qstats_update, /* etxo_qstats_update */ 231 #endif 232 }; 233 #endif /* EFSYS_OPT_RIVERHEAD */ 234 235 236 __checkReturn efx_rc_t 237 efx_tx_init( 238 __in efx_nic_t *enp) 239 { 240 const efx_tx_ops_t *etxop; 241 efx_rc_t rc; 242 243 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 244 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 245 246 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 247 rc = EINVAL; 248 goto fail1; 249 } 250 251 if (enp->en_mod_flags & EFX_MOD_TX) { 252 rc = EINVAL; 253 goto fail2; 254 } 255 256 switch (enp->en_family) { 257 #if EFSYS_OPT_SIENA 258 case EFX_FAMILY_SIENA: 259 etxop = &__efx_tx_siena_ops; 260 break; 261 #endif /* EFSYS_OPT_SIENA */ 262 263 #if EFSYS_OPT_HUNTINGTON 264 case EFX_FAMILY_HUNTINGTON: 265 etxop = &__efx_tx_hunt_ops; 266 break; 267 #endif /* EFSYS_OPT_HUNTINGTON */ 268 269 #if EFSYS_OPT_MEDFORD 270 case EFX_FAMILY_MEDFORD: 271 etxop = &__efx_tx_medford_ops; 272 break; 273 #endif /* EFSYS_OPT_MEDFORD */ 274 275 #if EFSYS_OPT_MEDFORD2 276 case EFX_FAMILY_MEDFORD2: 277 etxop = &__efx_tx_medford2_ops; 278 break; 279 #endif /* EFSYS_OPT_MEDFORD2 */ 280 281 #if EFSYS_OPT_RIVERHEAD 282 case EFX_FAMILY_RIVERHEAD: 283 etxop = &__efx_tx_rhead_ops; 284 break; 285 #endif /* EFSYS_OPT_RIVERHEAD */ 286 287 default: 288 EFSYS_ASSERT(0); 289 rc = ENOTSUP; 290 goto fail3; 291 } 292 293 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 294 295 if ((rc = etxop->etxo_init(enp)) != 0) 296 goto fail4; 297 298 enp->en_etxop = etxop; 299 enp->en_mod_flags |= EFX_MOD_TX; 300 return (0); 301 302 fail4: 303 EFSYS_PROBE(fail4); 304 fail3: 305 EFSYS_PROBE(fail3); 306 fail2: 307 EFSYS_PROBE(fail2); 308 fail1: 309 EFSYS_PROBE1(fail1, efx_rc_t, rc); 310 311 enp->en_etxop = NULL; 312 enp->en_mod_flags &= ~EFX_MOD_TX; 313 return (rc); 314 } 315 316 void 317 efx_tx_fini( 318 __in efx_nic_t *enp) 319 { 320 const efx_tx_ops_t *etxop = enp->en_etxop; 321 322 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 323 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 324 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 325 EFSYS_ASSERT3U(enp->en_tx_qcount, ==, 0); 326 327 etxop->etxo_fini(enp); 328 329 enp->en_etxop = NULL; 330 enp->en_mod_flags &= ~EFX_MOD_TX; 331 } 332 333 __checkReturn size_t 334 efx_txq_size( 335 __in const efx_nic_t *enp, 336 __in unsigned int ndescs) 337 { 338 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 339 340 return (ndescs * encp->enc_tx_desc_size); 341 } 342 343 __checkReturn unsigned int 344 efx_txq_nbufs( 345 __in const efx_nic_t *enp, 346 __in unsigned int ndescs) 347 { 348 return (EFX_DIV_ROUND_UP(efx_txq_size(enp, ndescs), EFX_BUF_SIZE)); 349 } 350 351 __checkReturn efx_rc_t 352 efx_tx_qcreate( 353 __in efx_nic_t *enp, 354 __in unsigned int index, 355 __in unsigned int label, 356 __in efsys_mem_t *esmp, 357 __in size_t ndescs, 358 __in uint32_t id, 359 __in uint16_t flags, 360 __in efx_evq_t *eep, 361 __deref_out efx_txq_t **etpp, 362 __out unsigned int *addedp) 363 { 364 const efx_tx_ops_t *etxop = enp->en_etxop; 365 efx_txq_t *etp; 366 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 367 efx_rc_t rc; 368 369 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 370 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_TX); 371 372 EFSYS_ASSERT3U(enp->en_tx_qcount + 1, <, 373 enp->en_nic_cfg.enc_txq_limit); 374 375 EFSYS_ASSERT(ISP2(encp->enc_txq_max_ndescs)); 376 EFSYS_ASSERT(ISP2(encp->enc_txq_min_ndescs)); 377 378 if (!ISP2(ndescs) || 379 ndescs < encp->enc_txq_min_ndescs || 380 ndescs > encp->enc_txq_max_ndescs) { 381 rc = EINVAL; 382 goto fail1; 383 } 384 385 /* Allocate an TXQ object */ 386 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_txq_t), etp); 387 388 if (etp == NULL) { 389 rc = ENOMEM; 390 goto fail2; 391 } 392 393 etp->et_magic = EFX_TXQ_MAGIC; 394 etp->et_enp = enp; 395 etp->et_index = index; 396 etp->et_mask = ndescs - 1; 397 etp->et_esmp = esmp; 398 399 /* Initial descriptor index may be modified by etxo_qcreate */ 400 *addedp = 0; 401 402 if ((rc = etxop->etxo_qcreate(enp, index, label, esmp, 403 ndescs, id, flags, eep, etp, addedp)) != 0) 404 goto fail3; 405 406 enp->en_tx_qcount++; 407 *etpp = etp; 408 409 return (0); 410 411 fail3: 412 EFSYS_PROBE(fail3); 413 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 414 fail2: 415 EFSYS_PROBE(fail2); 416 fail1: 417 EFSYS_PROBE1(fail1, efx_rc_t, rc); 418 return (rc); 419 } 420 421 void 422 efx_tx_qdestroy( 423 __in efx_txq_t *etp) 424 { 425 efx_nic_t *enp = etp->et_enp; 426 const efx_tx_ops_t *etxop = enp->en_etxop; 427 428 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 429 430 EFSYS_ASSERT(enp->en_tx_qcount != 0); 431 --enp->en_tx_qcount; 432 433 etxop->etxo_qdestroy(etp); 434 435 /* Free the TXQ object */ 436 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_txq_t), etp); 437 } 438 439 __checkReturn efx_rc_t 440 efx_tx_qpost( 441 __in efx_txq_t *etp, 442 __in_ecount(ndescs) efx_buffer_t *eb, 443 __in unsigned int ndescs, 444 __in unsigned int completed, 445 __inout unsigned int *addedp) 446 { 447 efx_nic_t *enp = etp->et_enp; 448 const efx_tx_ops_t *etxop = enp->en_etxop; 449 efx_rc_t rc; 450 451 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 452 453 if ((rc = etxop->etxo_qpost(etp, eb, ndescs, completed, addedp)) != 0) 454 goto fail1; 455 456 return (0); 457 458 fail1: 459 EFSYS_PROBE1(fail1, efx_rc_t, rc); 460 return (rc); 461 } 462 463 void 464 efx_tx_qpush( 465 __in efx_txq_t *etp, 466 __in unsigned int added, 467 __in unsigned int pushed) 468 { 469 efx_nic_t *enp = etp->et_enp; 470 const efx_tx_ops_t *etxop = enp->en_etxop; 471 472 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 473 474 etxop->etxo_qpush(etp, added, pushed); 475 } 476 477 __checkReturn efx_rc_t 478 efx_tx_qpace( 479 __in efx_txq_t *etp, 480 __in unsigned int ns) 481 { 482 efx_nic_t *enp = etp->et_enp; 483 const efx_tx_ops_t *etxop = enp->en_etxop; 484 efx_rc_t rc; 485 486 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 487 488 if ((rc = etxop->etxo_qpace(etp, ns)) != 0) 489 goto fail1; 490 491 return (0); 492 493 fail1: 494 EFSYS_PROBE1(fail1, efx_rc_t, rc); 495 return (rc); 496 } 497 498 __checkReturn efx_rc_t 499 efx_tx_qflush( 500 __in efx_txq_t *etp) 501 { 502 efx_nic_t *enp = etp->et_enp; 503 const efx_tx_ops_t *etxop = enp->en_etxop; 504 efx_rc_t rc; 505 506 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 507 508 if ((rc = etxop->etxo_qflush(etp)) != 0) 509 goto fail1; 510 511 return (0); 512 513 fail1: 514 EFSYS_PROBE1(fail1, efx_rc_t, rc); 515 return (rc); 516 } 517 518 void 519 efx_tx_qenable( 520 __in efx_txq_t *etp) 521 { 522 efx_nic_t *enp = etp->et_enp; 523 const efx_tx_ops_t *etxop = enp->en_etxop; 524 525 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 526 527 etxop->etxo_qenable(etp); 528 } 529 530 __checkReturn efx_rc_t 531 efx_tx_qpio_enable( 532 __in efx_txq_t *etp) 533 { 534 efx_nic_t *enp = etp->et_enp; 535 const efx_tx_ops_t *etxop = enp->en_etxop; 536 efx_rc_t rc; 537 538 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 539 540 if (~enp->en_features & EFX_FEATURE_PIO_BUFFERS) { 541 rc = ENOTSUP; 542 goto fail1; 543 } 544 if (etxop->etxo_qpio_enable == NULL) { 545 rc = ENOTSUP; 546 goto fail2; 547 } 548 if ((rc = etxop->etxo_qpio_enable(etp)) != 0) 549 goto fail3; 550 551 return (0); 552 553 fail3: 554 EFSYS_PROBE(fail3); 555 fail2: 556 EFSYS_PROBE(fail2); 557 fail1: 558 EFSYS_PROBE1(fail1, efx_rc_t, rc); 559 return (rc); 560 } 561 562 void 563 efx_tx_qpio_disable( 564 __in efx_txq_t *etp) 565 { 566 efx_nic_t *enp = etp->et_enp; 567 const efx_tx_ops_t *etxop = enp->en_etxop; 568 569 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 570 571 if (etxop->etxo_qpio_disable != NULL) 572 etxop->etxo_qpio_disable(etp); 573 } 574 575 __checkReturn efx_rc_t 576 efx_tx_qpio_write( 577 __in efx_txq_t *etp, 578 __in_ecount(buf_length) uint8_t *buffer, 579 __in size_t buf_length, 580 __in size_t pio_buf_offset) 581 { 582 efx_nic_t *enp = etp->et_enp; 583 const efx_tx_ops_t *etxop = enp->en_etxop; 584 efx_rc_t rc; 585 586 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 587 588 if (etxop->etxo_qpio_write != NULL) { 589 if ((rc = etxop->etxo_qpio_write(etp, buffer, buf_length, 590 pio_buf_offset)) != 0) 591 goto fail1; 592 return (0); 593 } 594 595 return (ENOTSUP); 596 597 fail1: 598 EFSYS_PROBE1(fail1, efx_rc_t, rc); 599 return (rc); 600 } 601 602 __checkReturn efx_rc_t 603 efx_tx_qpio_post( 604 __in efx_txq_t *etp, 605 __in size_t pkt_length, 606 __in unsigned int completed, 607 __inout unsigned int *addedp) 608 { 609 efx_nic_t *enp = etp->et_enp; 610 const efx_tx_ops_t *etxop = enp->en_etxop; 611 efx_rc_t rc; 612 613 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 614 615 if (etxop->etxo_qpio_post != NULL) { 616 if ((rc = etxop->etxo_qpio_post(etp, pkt_length, completed, 617 addedp)) != 0) 618 goto fail1; 619 return (0); 620 } 621 622 return (ENOTSUP); 623 624 fail1: 625 EFSYS_PROBE1(fail1, efx_rc_t, rc); 626 return (rc); 627 } 628 629 __checkReturn efx_rc_t 630 efx_tx_qdesc_post( 631 __in efx_txq_t *etp, 632 __in_ecount(ndescs) efx_desc_t *ed, 633 __in unsigned int ndescs, 634 __in unsigned int completed, 635 __inout unsigned int *addedp) 636 { 637 efx_nic_t *enp = etp->et_enp; 638 const efx_tx_ops_t *etxop = enp->en_etxop; 639 640 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 641 642 return (etxop->etxo_qdesc_post(etp, ed, ndescs, completed, addedp)); 643 } 644 645 void 646 efx_tx_qdesc_dma_create( 647 __in efx_txq_t *etp, 648 __in efsys_dma_addr_t addr, 649 __in size_t size, 650 __in boolean_t eop, 651 __out efx_desc_t *edp) 652 { 653 efx_nic_t *enp = etp->et_enp; 654 const efx_tx_ops_t *etxop = enp->en_etxop; 655 656 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 657 EFSYS_ASSERT(etxop->etxo_qdesc_dma_create != NULL); 658 659 etxop->etxo_qdesc_dma_create(etp, addr, size, eop, edp); 660 } 661 662 void 663 efx_tx_qdesc_tso_create( 664 __in efx_txq_t *etp, 665 __in uint16_t ipv4_id, 666 __in uint32_t tcp_seq, 667 __in uint8_t tcp_flags, 668 __out efx_desc_t *edp) 669 { 670 efx_nic_t *enp = etp->et_enp; 671 const efx_tx_ops_t *etxop = enp->en_etxop; 672 673 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 674 EFSYS_ASSERT(etxop->etxo_qdesc_tso_create != NULL); 675 676 etxop->etxo_qdesc_tso_create(etp, ipv4_id, tcp_seq, tcp_flags, edp); 677 } 678 679 void 680 efx_tx_qdesc_tso2_create( 681 __in efx_txq_t *etp, 682 __in uint16_t ipv4_id, 683 __in uint16_t outer_ipv4_id, 684 __in uint32_t tcp_seq, 685 __in uint16_t mss, 686 __out_ecount(count) efx_desc_t *edp, 687 __in int count) 688 { 689 efx_nic_t *enp = etp->et_enp; 690 const efx_tx_ops_t *etxop = enp->en_etxop; 691 692 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 693 EFSYS_ASSERT(etxop->etxo_qdesc_tso2_create != NULL); 694 695 etxop->etxo_qdesc_tso2_create(etp, ipv4_id, outer_ipv4_id, 696 tcp_seq, mss, edp, count); 697 } 698 699 void 700 efx_tx_qdesc_vlantci_create( 701 __in efx_txq_t *etp, 702 __in uint16_t tci, 703 __out efx_desc_t *edp) 704 { 705 efx_nic_t *enp = etp->et_enp; 706 const efx_tx_ops_t *etxop = enp->en_etxop; 707 708 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 709 EFSYS_ASSERT(etxop->etxo_qdesc_vlantci_create != NULL); 710 711 etxop->etxo_qdesc_vlantci_create(etp, tci, edp); 712 } 713 714 void 715 efx_tx_qdesc_checksum_create( 716 __in efx_txq_t *etp, 717 __in uint16_t flags, 718 __out efx_desc_t *edp) 719 { 720 efx_nic_t *enp = etp->et_enp; 721 const efx_tx_ops_t *etxop = enp->en_etxop; 722 723 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 724 EFSYS_ASSERT(etxop->etxo_qdesc_checksum_create != NULL); 725 726 etxop->etxo_qdesc_checksum_create(etp, flags, edp); 727 } 728 729 730 #if EFSYS_OPT_QSTATS 731 void 732 efx_tx_qstats_update( 733 __in efx_txq_t *etp, 734 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 735 { 736 efx_nic_t *enp = etp->et_enp; 737 const efx_tx_ops_t *etxop = enp->en_etxop; 738 739 EFSYS_ASSERT3U(etp->et_magic, ==, EFX_TXQ_MAGIC); 740 741 etxop->etxo_qstats_update(etp, stat); 742 } 743 #endif 744 745 746 #if EFSYS_OPT_SIENA 747 748 static __checkReturn efx_rc_t 749 siena_tx_init( 750 __in efx_nic_t *enp) 751 { 752 efx_oword_t oword; 753 754 /* 755 * Disable the timer-based TX DMA backoff and allow TX DMA to be 756 * controlled by the RX FIFO fill level (although always allow a 757 * minimal trickle). 758 */ 759 EFX_BAR_READO(enp, FR_AZ_TX_RESERVED_REG, &oword); 760 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER, 0xfe); 761 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_RX_SPACER_EN, 1); 762 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_ONE_PKT_PER_Q, 1); 763 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PUSH_EN, 0); 764 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DIS_NON_IP_EV, 1); 765 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_THRESHOLD, 2); 766 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_PREF_WD_TMR, 0x3fffff); 767 768 /* 769 * Filter all packets less than 14 bytes to avoid parsing 770 * errors. 771 */ 772 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_FLUSH_MIN_LEN_EN, 1); 773 EFX_BAR_WRITEO(enp, FR_AZ_TX_RESERVED_REG, &oword); 774 775 /* 776 * Do not set TX_NO_EOP_DISC_EN, since it limits packets to 16 777 * descriptors (which is bad). 778 */ 779 EFX_BAR_READO(enp, FR_AZ_TX_CFG_REG, &oword); 780 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_NO_EOP_DISC_EN, 0); 781 EFX_BAR_WRITEO(enp, FR_AZ_TX_CFG_REG, &oword); 782 783 return (0); 784 } 785 786 #define EFX_TX_DESC(_etp, _addr, _size, _eop, _added) \ 787 do { \ 788 unsigned int id; \ 789 size_t offset; \ 790 efx_qword_t qword; \ 791 \ 792 id = (_added)++ & (_etp)->et_mask; \ 793 offset = id * sizeof (efx_qword_t); \ 794 \ 795 EFSYS_PROBE5(tx_post, unsigned int, (_etp)->et_index, \ 796 unsigned int, id, efsys_dma_addr_t, (_addr), \ 797 size_t, (_size), boolean_t, (_eop)); \ 798 \ 799 EFX_POPULATE_QWORD_4(qword, \ 800 FSF_AZ_TX_KER_CONT, (_eop) ? 0 : 1, \ 801 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)(_size), \ 802 FSF_AZ_TX_KER_BUF_ADDR_DW0, \ 803 (uint32_t)((_addr) & 0xffffffff), \ 804 FSF_AZ_TX_KER_BUF_ADDR_DW1, \ 805 (uint32_t)((_addr) >> 32)); \ 806 EFSYS_MEM_WRITEQ((_etp)->et_esmp, offset, &qword); \ 807 \ 808 _NOTE(CONSTANTCONDITION) \ 809 } while (B_FALSE) 810 811 static __checkReturn efx_rc_t 812 siena_tx_qpost( 813 __in efx_txq_t *etp, 814 __in_ecount(ndescs) efx_buffer_t *eb, 815 __in unsigned int ndescs, 816 __in unsigned int completed, 817 __inout unsigned int *addedp) 818 { 819 unsigned int added = *addedp; 820 unsigned int i; 821 822 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) 823 return (ENOSPC); 824 825 for (i = 0; i < ndescs; i++) { 826 efx_buffer_t *ebp = &eb[i]; 827 efsys_dma_addr_t start = ebp->eb_addr; 828 size_t size = ebp->eb_size; 829 efsys_dma_addr_t end = start + size; 830 831 /* 832 * Fragments must not span 4k boundaries. 833 * Here it is a stricter requirement than the maximum length. 834 */ 835 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, start + 1, 836 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= end); 837 838 EFX_TX_DESC(etp, start, size, ebp->eb_eop, added); 839 } 840 841 EFX_TX_QSTAT_INCR(etp, TX_POST); 842 843 *addedp = added; 844 return (0); 845 } 846 847 static void 848 siena_tx_qpush( 849 __in efx_txq_t *etp, 850 __in unsigned int added, 851 __in unsigned int pushed) 852 { 853 efx_nic_t *enp = etp->et_enp; 854 uint32_t wptr; 855 efx_dword_t dword; 856 efx_oword_t oword; 857 858 /* Push the populated descriptors out */ 859 wptr = added & etp->et_mask; 860 861 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_DESC_WPTR, wptr); 862 863 /* Only write the third DWORD */ 864 EFX_POPULATE_DWORD_1(dword, 865 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 866 867 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 868 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(etp->et_esmp, etp->et_mask + 1, 869 SIENA_TXQ_DESC_SIZE, wptr, pushed & etp->et_mask); 870 EFSYS_PIO_WRITE_BARRIER(); 871 EFX_BAR_TBL_WRITED3(enp, FR_BZ_TX_DESC_UPD_REGP0, 872 etp->et_index, &dword, B_FALSE); 873 } 874 875 #define EFX_MAX_PACE_VALUE 20 876 877 static __checkReturn efx_rc_t 878 siena_tx_qpace( 879 __in efx_txq_t *etp, 880 __in unsigned int ns) 881 { 882 efx_nic_t *enp = etp->et_enp; 883 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 884 efx_oword_t oword; 885 unsigned int pace_val; 886 unsigned int timer_period; 887 efx_rc_t rc; 888 889 if (ns == 0) { 890 pace_val = 0; 891 } else { 892 /* 893 * The pace_val to write into the table is s.t 894 * ns <= timer_period * (2 ^ pace_val) 895 */ 896 timer_period = 104 / encp->enc_clk_mult; 897 for (pace_val = 1; pace_val <= EFX_MAX_PACE_VALUE; pace_val++) { 898 if ((timer_period << pace_val) >= ns) 899 break; 900 } 901 } 902 if (pace_val > EFX_MAX_PACE_VALUE) { 903 rc = EINVAL; 904 goto fail1; 905 } 906 907 /* Update the pacing table */ 908 EFX_POPULATE_OWORD_1(oword, FRF_AZ_TX_PACE, pace_val); 909 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_PACE_TBL, etp->et_index, 910 &oword, B_TRUE); 911 912 return (0); 913 914 fail1: 915 EFSYS_PROBE1(fail1, efx_rc_t, rc); 916 917 return (rc); 918 } 919 920 static __checkReturn efx_rc_t 921 siena_tx_qflush( 922 __in efx_txq_t *etp) 923 { 924 efx_nic_t *enp = etp->et_enp; 925 efx_oword_t oword; 926 uint32_t label; 927 928 efx_tx_qpace(etp, 0); 929 930 label = etp->et_index; 931 932 /* Flush the queue */ 933 EFX_POPULATE_OWORD_2(oword, FRF_AZ_TX_FLUSH_DESCQ_CMD, 1, 934 FRF_AZ_TX_FLUSH_DESCQ, label); 935 EFX_BAR_WRITEO(enp, FR_AZ_TX_FLUSH_DESCQ_REG, &oword); 936 937 return (0); 938 } 939 940 static void 941 siena_tx_qenable( 942 __in efx_txq_t *etp) 943 { 944 efx_nic_t *enp = etp->et_enp; 945 efx_oword_t oword; 946 947 EFX_BAR_TBL_READO(enp, FR_AZ_TX_DESC_PTR_TBL, 948 etp->et_index, &oword, B_TRUE); 949 950 EFSYS_PROBE5(tx_descq_ptr, unsigned int, etp->et_index, 951 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_3), 952 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_2), 953 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_1), 954 uint32_t, EFX_OWORD_FIELD(oword, EFX_DWORD_0)); 955 956 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DC_HW_RPTR, 0); 957 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_HW_RPTR, 0); 958 EFX_SET_OWORD_FIELD(oword, FRF_AZ_TX_DESCQ_EN, 1); 959 960 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 961 etp->et_index, &oword, B_TRUE); 962 } 963 964 static __checkReturn efx_rc_t 965 siena_tx_qcreate( 966 __in efx_nic_t *enp, 967 __in unsigned int index, 968 __in unsigned int label, 969 __in efsys_mem_t *esmp, 970 __in size_t ndescs, 971 __in uint32_t id, 972 __in uint16_t flags, 973 __in efx_evq_t *eep, 974 __in efx_txq_t *etp, 975 __out unsigned int *addedp) 976 { 977 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 978 efx_oword_t oword; 979 uint32_t size; 980 uint16_t inner_csum; 981 efx_rc_t rc; 982 983 _NOTE(ARGUNUSED(esmp)) 984 985 EFX_STATIC_ASSERT(EFX_EV_TX_NLABELS == 986 (1 << FRF_AZ_TX_DESCQ_LABEL_WIDTH)); 987 EFSYS_ASSERT3U(label, <, EFX_EV_TX_NLABELS); 988 989 if (index >= encp->enc_txq_limit) { 990 rc = EINVAL; 991 goto fail1; 992 } 993 for (size = 0; 994 (1U << size) <= encp->enc_txq_max_ndescs / encp->enc_txq_min_ndescs; 995 size++) 996 if ((1U << size) == (uint32_t)ndescs / encp->enc_txq_min_ndescs) 997 break; 998 if (id + (1 << size) >= encp->enc_buftbl_limit) { 999 rc = EINVAL; 1000 goto fail2; 1001 } 1002 1003 inner_csum = EFX_TXQ_CKSUM_INNER_IPV4 | EFX_TXQ_CKSUM_INNER_TCPUDP; 1004 if ((flags & inner_csum) != 0) { 1005 rc = EINVAL; 1006 goto fail3; 1007 } 1008 1009 /* Set up the new descriptor queue */ 1010 *addedp = 0; 1011 1012 EFX_POPULATE_OWORD_6(oword, 1013 FRF_AZ_TX_DESCQ_BUF_BASE_ID, id, 1014 FRF_AZ_TX_DESCQ_EVQ_ID, eep->ee_index, 1015 FRF_AZ_TX_DESCQ_OWNER_ID, 0, 1016 FRF_AZ_TX_DESCQ_LABEL, label, 1017 FRF_AZ_TX_DESCQ_SIZE, size, 1018 FRF_AZ_TX_DESCQ_TYPE, 0); 1019 1020 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_NON_IP_DROP_DIS, 1); 1021 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_IP_CHKSM_DIS, 1022 (flags & EFX_TXQ_CKSUM_IPV4) ? 0 : 1); 1023 EFX_SET_OWORD_FIELD(oword, FRF_BZ_TX_TCP_CHKSM_DIS, 1024 (flags & EFX_TXQ_CKSUM_TCPUDP) ? 0 : 1); 1025 1026 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1027 etp->et_index, &oword, B_TRUE); 1028 1029 return (0); 1030 1031 fail3: 1032 EFSYS_PROBE(fail3); 1033 fail2: 1034 EFSYS_PROBE(fail2); 1035 fail1: 1036 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1037 1038 return (rc); 1039 } 1040 1041 __checkReturn efx_rc_t 1042 siena_tx_qdesc_post( 1043 __in efx_txq_t *etp, 1044 __in_ecount(ndescs) efx_desc_t *ed, 1045 __in unsigned int ndescs, 1046 __in unsigned int completed, 1047 __inout unsigned int *addedp) 1048 { 1049 unsigned int added = *addedp; 1050 unsigned int i; 1051 efx_rc_t rc; 1052 1053 if (added - completed + ndescs > EFX_TXQ_LIMIT(etp->et_mask + 1)) { 1054 rc = ENOSPC; 1055 goto fail1; 1056 } 1057 1058 for (i = 0; i < ndescs; i++) { 1059 efx_desc_t *edp = &ed[i]; 1060 unsigned int id; 1061 size_t offset; 1062 1063 id = added++ & etp->et_mask; 1064 offset = id * sizeof (efx_desc_t); 1065 1066 EFSYS_MEM_WRITEQ(etp->et_esmp, offset, &edp->ed_eq); 1067 } 1068 1069 EFSYS_PROBE3(tx_desc_post, unsigned int, etp->et_index, 1070 unsigned int, added, unsigned int, ndescs); 1071 1072 EFX_TX_QSTAT_INCR(etp, TX_POST); 1073 1074 *addedp = added; 1075 return (0); 1076 1077 fail1: 1078 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1079 return (rc); 1080 } 1081 1082 void 1083 siena_tx_qdesc_dma_create( 1084 __in efx_txq_t *etp, 1085 __in efsys_dma_addr_t addr, 1086 __in size_t size, 1087 __in boolean_t eop, 1088 __out efx_desc_t *edp) 1089 { 1090 /* 1091 * Fragments must not span 4k boundaries. 1092 * Here it is a stricter requirement than the maximum length. 1093 */ 1094 EFSYS_ASSERT(EFX_P2ROUNDUP(efsys_dma_addr_t, addr + 1, 1095 etp->et_enp->en_nic_cfg.enc_tx_dma_desc_boundary) >= addr + size); 1096 1097 EFSYS_PROBE4(tx_desc_dma_create, unsigned int, etp->et_index, 1098 efsys_dma_addr_t, addr, 1099 size_t, size, boolean_t, eop); 1100 1101 EFX_POPULATE_QWORD_4(edp->ed_eq, 1102 FSF_AZ_TX_KER_CONT, eop ? 0 : 1, 1103 FSF_AZ_TX_KER_BYTE_COUNT, (uint32_t)size, 1104 FSF_AZ_TX_KER_BUF_ADDR_DW0, 1105 (uint32_t)(addr & 0xffffffff), 1106 FSF_AZ_TX_KER_BUF_ADDR_DW1, 1107 (uint32_t)(addr >> 32)); 1108 } 1109 1110 #endif /* EFSYS_OPT_SIENA */ 1111 1112 #if EFSYS_OPT_QSTATS 1113 #if EFSYS_OPT_NAMES 1114 /* START MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock 2866874ecd7a363b */ 1115 static const char * const __efx_tx_qstat_name[] = { 1116 "post", 1117 "post_pio", 1118 }; 1119 /* END MKCONFIG GENERATED EfxTransmitQueueStatNamesBlock */ 1120 1121 const char * 1122 efx_tx_qstat_name( 1123 __in efx_nic_t *enp, 1124 __in unsigned int id) 1125 { 1126 _NOTE(ARGUNUSED(enp)) 1127 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 1128 EFSYS_ASSERT3U(id, <, TX_NQSTATS); 1129 1130 return (__efx_tx_qstat_name[id]); 1131 } 1132 #endif /* EFSYS_OPT_NAMES */ 1133 #endif /* EFSYS_OPT_QSTATS */ 1134 1135 #if EFSYS_OPT_SIENA 1136 1137 #if EFSYS_OPT_QSTATS 1138 static void 1139 siena_tx_qstats_update( 1140 __in efx_txq_t *etp, 1141 __inout_ecount(TX_NQSTATS) efsys_stat_t *stat) 1142 { 1143 unsigned int id; 1144 1145 for (id = 0; id < TX_NQSTATS; id++) { 1146 efsys_stat_t *essp = &stat[id]; 1147 1148 EFSYS_STAT_INCR(essp, etp->et_stat[id]); 1149 etp->et_stat[id] = 0; 1150 } 1151 } 1152 #endif /* EFSYS_OPT_QSTATS */ 1153 1154 static void 1155 siena_tx_qdestroy( 1156 __in efx_txq_t *etp) 1157 { 1158 efx_nic_t *enp = etp->et_enp; 1159 efx_oword_t oword; 1160 1161 /* Purge descriptor queue */ 1162 EFX_ZERO_OWORD(oword); 1163 1164 EFX_BAR_TBL_WRITEO(enp, FR_AZ_TX_DESC_PTR_TBL, 1165 etp->et_index, &oword, B_TRUE); 1166 } 1167 1168 static void 1169 siena_tx_fini( 1170 __in efx_nic_t *enp) 1171 { 1172 _NOTE(ARGUNUSED(enp)) 1173 } 1174 1175 #endif /* EFSYS_OPT_SIENA */ 1176