1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2007-2019 Solarflare Communications Inc. 5 */ 6 7 #include "efx.h" 8 #include "efx_impl.h" 9 10 11 #if EFSYS_OPT_SIENA 12 13 static __checkReturn efx_rc_t 14 siena_rx_init( 15 __in efx_nic_t *enp); 16 17 static void 18 siena_rx_fini( 19 __in efx_nic_t *enp); 20 21 #if EFSYS_OPT_RX_SCATTER 22 static __checkReturn efx_rc_t 23 siena_rx_scatter_enable( 24 __in efx_nic_t *enp, 25 __in unsigned int buf_size); 26 #endif /* EFSYS_OPT_RX_SCATTER */ 27 28 #if EFSYS_OPT_RX_SCALE 29 static __checkReturn efx_rc_t 30 siena_rx_scale_mode_set( 31 __in efx_nic_t *enp, 32 __in uint32_t rss_context, 33 __in efx_rx_hash_alg_t alg, 34 __in efx_rx_hash_type_t type, 35 __in boolean_t insert); 36 37 static __checkReturn efx_rc_t 38 siena_rx_scale_key_set( 39 __in efx_nic_t *enp, 40 __in uint32_t rss_context, 41 __in_ecount(n) uint8_t *key, 42 __in size_t n); 43 44 static __checkReturn efx_rc_t 45 siena_rx_scale_tbl_set( 46 __in efx_nic_t *enp, 47 __in uint32_t rss_context, 48 __in_ecount(nentries) unsigned int *table, 49 __in size_t nentries); 50 51 static __checkReturn uint32_t 52 siena_rx_prefix_hash( 53 __in efx_nic_t *enp, 54 __in efx_rx_hash_alg_t func, 55 __in uint8_t *buffer); 56 57 #endif /* EFSYS_OPT_RX_SCALE */ 58 59 static __checkReturn efx_rc_t 60 siena_rx_prefix_pktlen( 61 __in efx_nic_t *enp, 62 __in uint8_t *buffer, 63 __out uint16_t *lengthp); 64 65 static void 66 siena_rx_qpost( 67 __in efx_rxq_t *erp, 68 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 69 __in size_t size, 70 __in unsigned int ndescs, 71 __in unsigned int completed, 72 __in unsigned int added); 73 74 static void 75 siena_rx_qpush( 76 __in efx_rxq_t *erp, 77 __in unsigned int added, 78 __inout unsigned int *pushedp); 79 80 #if EFSYS_OPT_RX_PACKED_STREAM 81 static void 82 siena_rx_qpush_ps_credits( 83 __in efx_rxq_t *erp); 84 85 static __checkReturn uint8_t * 86 siena_rx_qps_packet_info( 87 __in efx_rxq_t *erp, 88 __in uint8_t *buffer, 89 __in uint32_t buffer_length, 90 __in uint32_t current_offset, 91 __out uint16_t *lengthp, 92 __out uint32_t *next_offsetp, 93 __out uint32_t *timestamp); 94 #endif 95 96 static __checkReturn efx_rc_t 97 siena_rx_qflush( 98 __in efx_rxq_t *erp); 99 100 static void 101 siena_rx_qenable( 102 __in efx_rxq_t *erp); 103 104 static __checkReturn efx_rc_t 105 siena_rx_qcreate( 106 __in efx_nic_t *enp, 107 __in unsigned int index, 108 __in unsigned int label, 109 __in efx_rxq_type_t type, 110 __in_opt const efx_rxq_type_data_t *type_data, 111 __in efsys_mem_t *esmp, 112 __in size_t ndescs, 113 __in uint32_t id, 114 __in unsigned int flags, 115 __in efx_evq_t *eep, 116 __in efx_rxq_t *erp); 117 118 static void 119 siena_rx_qdestroy( 120 __in efx_rxq_t *erp); 121 122 #endif /* EFSYS_OPT_SIENA */ 123 124 125 #if EFSYS_OPT_SIENA 126 static const efx_rx_ops_t __efx_rx_siena_ops = { 127 siena_rx_init, /* erxo_init */ 128 siena_rx_fini, /* erxo_fini */ 129 #if EFSYS_OPT_RX_SCATTER 130 siena_rx_scatter_enable, /* erxo_scatter_enable */ 131 #endif 132 #if EFSYS_OPT_RX_SCALE 133 NULL, /* erxo_scale_context_alloc */ 134 NULL, /* erxo_scale_context_free */ 135 siena_rx_scale_mode_set, /* erxo_scale_mode_set */ 136 siena_rx_scale_key_set, /* erxo_scale_key_set */ 137 siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 138 siena_rx_prefix_hash, /* erxo_prefix_hash */ 139 #endif 140 siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 141 siena_rx_qpost, /* erxo_qpost */ 142 siena_rx_qpush, /* erxo_qpush */ 143 #if EFSYS_OPT_RX_PACKED_STREAM 144 siena_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 145 siena_rx_qps_packet_info, /* erxo_qps_packet_info */ 146 #endif 147 siena_rx_qflush, /* erxo_qflush */ 148 siena_rx_qenable, /* erxo_qenable */ 149 siena_rx_qcreate, /* erxo_qcreate */ 150 siena_rx_qdestroy, /* erxo_qdestroy */ 151 }; 152 #endif /* EFSYS_OPT_SIENA */ 153 154 #if EFX_OPTS_EF10() 155 static const efx_rx_ops_t __efx_rx_ef10_ops = { 156 ef10_rx_init, /* erxo_init */ 157 ef10_rx_fini, /* erxo_fini */ 158 #if EFSYS_OPT_RX_SCATTER 159 ef10_rx_scatter_enable, /* erxo_scatter_enable */ 160 #endif 161 #if EFSYS_OPT_RX_SCALE 162 ef10_rx_scale_context_alloc, /* erxo_scale_context_alloc */ 163 ef10_rx_scale_context_free, /* erxo_scale_context_free */ 164 ef10_rx_scale_mode_set, /* erxo_scale_mode_set */ 165 ef10_rx_scale_key_set, /* erxo_scale_key_set */ 166 ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 167 ef10_rx_prefix_hash, /* erxo_prefix_hash */ 168 #endif 169 ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 170 ef10_rx_qpost, /* erxo_qpost */ 171 ef10_rx_qpush, /* erxo_qpush */ 172 #if EFSYS_OPT_RX_PACKED_STREAM 173 ef10_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 174 ef10_rx_qps_packet_info, /* erxo_qps_packet_info */ 175 #endif 176 ef10_rx_qflush, /* erxo_qflush */ 177 ef10_rx_qenable, /* erxo_qenable */ 178 ef10_rx_qcreate, /* erxo_qcreate */ 179 ef10_rx_qdestroy, /* erxo_qdestroy */ 180 }; 181 #endif /* EFX_OPTS_EF10() */ 182 183 #if EFSYS_OPT_RIVERHEAD 184 static const efx_rx_ops_t __efx_rx_rhead_ops = { 185 rhead_rx_init, /* erxo_init */ 186 rhead_rx_fini, /* erxo_fini */ 187 #if EFSYS_OPT_RX_SCATTER 188 rhead_rx_scatter_enable, /* erxo_scatter_enable */ 189 #endif 190 #if EFSYS_OPT_RX_SCALE 191 rhead_rx_scale_context_alloc, /* erxo_scale_context_alloc */ 192 rhead_rx_scale_context_free, /* erxo_scale_context_free */ 193 rhead_rx_scale_mode_set, /* erxo_scale_mode_set */ 194 rhead_rx_scale_key_set, /* erxo_scale_key_set */ 195 rhead_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 196 rhead_rx_prefix_hash, /* erxo_prefix_hash */ 197 #endif 198 rhead_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 199 rhead_rx_qpost, /* erxo_qpost */ 200 rhead_rx_qpush, /* erxo_qpush */ 201 #if EFSYS_OPT_RX_PACKED_STREAM 202 NULL, /* erxo_qpush_ps_credits */ 203 NULL, /* erxo_qps_packet_info */ 204 #endif 205 rhead_rx_qflush, /* erxo_qflush */ 206 rhead_rx_qenable, /* erxo_qenable */ 207 rhead_rx_qcreate, /* erxo_qcreate */ 208 rhead_rx_qdestroy, /* erxo_qdestroy */ 209 }; 210 #endif /* EFSYS_OPT_RIVERHEAD */ 211 212 213 __checkReturn efx_rc_t 214 efx_rx_init( 215 __inout efx_nic_t *enp) 216 { 217 const efx_rx_ops_t *erxop; 218 efx_rc_t rc; 219 220 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 221 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 222 223 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 224 rc = EINVAL; 225 goto fail1; 226 } 227 228 if (enp->en_mod_flags & EFX_MOD_RX) { 229 rc = EINVAL; 230 goto fail2; 231 } 232 233 switch (enp->en_family) { 234 #if EFSYS_OPT_SIENA 235 case EFX_FAMILY_SIENA: 236 erxop = &__efx_rx_siena_ops; 237 break; 238 #endif /* EFSYS_OPT_SIENA */ 239 240 #if EFSYS_OPT_HUNTINGTON 241 case EFX_FAMILY_HUNTINGTON: 242 erxop = &__efx_rx_ef10_ops; 243 break; 244 #endif /* EFSYS_OPT_HUNTINGTON */ 245 246 #if EFSYS_OPT_MEDFORD 247 case EFX_FAMILY_MEDFORD: 248 erxop = &__efx_rx_ef10_ops; 249 break; 250 #endif /* EFSYS_OPT_MEDFORD */ 251 252 #if EFSYS_OPT_MEDFORD2 253 case EFX_FAMILY_MEDFORD2: 254 erxop = &__efx_rx_ef10_ops; 255 break; 256 #endif /* EFSYS_OPT_MEDFORD2 */ 257 258 #if EFSYS_OPT_RIVERHEAD 259 case EFX_FAMILY_RIVERHEAD: 260 erxop = &__efx_rx_rhead_ops; 261 break; 262 #endif /* EFSYS_OPT_RIVERHEAD */ 263 264 default: 265 EFSYS_ASSERT(0); 266 rc = ENOTSUP; 267 goto fail3; 268 } 269 270 if ((rc = erxop->erxo_init(enp)) != 0) 271 goto fail4; 272 273 enp->en_erxop = erxop; 274 enp->en_mod_flags |= EFX_MOD_RX; 275 return (0); 276 277 fail4: 278 EFSYS_PROBE(fail4); 279 fail3: 280 EFSYS_PROBE(fail3); 281 fail2: 282 EFSYS_PROBE(fail2); 283 fail1: 284 EFSYS_PROBE1(fail1, efx_rc_t, rc); 285 286 enp->en_erxop = NULL; 287 enp->en_mod_flags &= ~EFX_MOD_RX; 288 return (rc); 289 } 290 291 void 292 efx_rx_fini( 293 __in efx_nic_t *enp) 294 { 295 const efx_rx_ops_t *erxop = enp->en_erxop; 296 297 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 298 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 299 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 300 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 301 302 erxop->erxo_fini(enp); 303 304 enp->en_erxop = NULL; 305 enp->en_mod_flags &= ~EFX_MOD_RX; 306 } 307 308 #if EFSYS_OPT_RX_SCATTER 309 __checkReturn efx_rc_t 310 efx_rx_scatter_enable( 311 __in efx_nic_t *enp, 312 __in unsigned int buf_size) 313 { 314 const efx_rx_ops_t *erxop = enp->en_erxop; 315 efx_rc_t rc; 316 317 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 318 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 319 320 if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0) 321 goto fail1; 322 323 return (0); 324 325 fail1: 326 EFSYS_PROBE1(fail1, efx_rc_t, rc); 327 return (rc); 328 } 329 #endif /* EFSYS_OPT_RX_SCATTER */ 330 331 #if EFSYS_OPT_RX_SCALE 332 __checkReturn efx_rc_t 333 efx_rx_scale_hash_flags_get( 334 __in efx_nic_t *enp, 335 __in efx_rx_hash_alg_t hash_alg, 336 __out_ecount_part(max_nflags, *nflagsp) unsigned int *flagsp, 337 __in unsigned int max_nflags, 338 __out unsigned int *nflagsp) 339 { 340 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 341 unsigned int nflags = 0; 342 efx_rc_t rc; 343 344 if (flagsp == NULL || nflagsp == NULL) { 345 rc = EINVAL; 346 goto fail1; 347 } 348 349 if ((encp->enc_rx_scale_hash_alg_mask & (1U << hash_alg)) == 0) { 350 nflags = 0; 351 goto done; 352 } 353 354 /* Helper to add flags word to flags array without buffer overflow */ 355 #define INSERT_FLAGS(_flags) \ 356 do { \ 357 if (nflags >= max_nflags) { \ 358 rc = E2BIG; \ 359 goto fail2; \ 360 } \ 361 *(flagsp + nflags) = (_flags); \ 362 nflags++; \ 363 \ 364 _NOTE(CONSTANTCONDITION) \ 365 } while (B_FALSE) 366 367 if (encp->enc_rx_scale_l4_hash_supported != B_FALSE) { 368 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 4TUPLE)); 369 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 4TUPLE)); 370 } 371 372 if ((encp->enc_rx_scale_l4_hash_supported != B_FALSE) && 373 (encp->enc_rx_scale_additional_modes_supported != B_FALSE)) { 374 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_DST)); 375 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_SRC)); 376 377 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_DST)); 378 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_SRC)); 379 380 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 4TUPLE)); 381 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_DST)); 382 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_SRC)); 383 384 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 4TUPLE)); 385 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_DST)); 386 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_SRC)); 387 } 388 389 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE)); 390 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE)); 391 392 INSERT_FLAGS(EFX_RX_HASH(IPV4, 2TUPLE)); 393 INSERT_FLAGS(EFX_RX_HASH(IPV6, 2TUPLE)); 394 395 if (encp->enc_rx_scale_additional_modes_supported != B_FALSE) { 396 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_DST)); 397 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_SRC)); 398 399 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_DST)); 400 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_SRC)); 401 402 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE)); 403 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_DST)); 404 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_SRC)); 405 406 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE)); 407 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_DST)); 408 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_SRC)); 409 410 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_DST)); 411 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_SRC)); 412 413 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_DST)); 414 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_SRC)); 415 } 416 417 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, DISABLE)); 418 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, DISABLE)); 419 420 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, DISABLE)); 421 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, DISABLE)); 422 423 INSERT_FLAGS(EFX_RX_HASH(IPV4, DISABLE)); 424 INSERT_FLAGS(EFX_RX_HASH(IPV6, DISABLE)); 425 426 #undef INSERT_FLAGS 427 428 done: 429 *nflagsp = nflags; 430 return (0); 431 432 fail2: 433 EFSYS_PROBE(fail2); 434 fail1: 435 EFSYS_PROBE1(fail1, efx_rc_t, rc); 436 437 return (rc); 438 } 439 440 __checkReturn efx_rc_t 441 efx_rx_hash_default_support_get( 442 __in efx_nic_t *enp, 443 __out efx_rx_hash_support_t *supportp) 444 { 445 efx_rc_t rc; 446 447 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 448 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 449 450 if (supportp == NULL) { 451 rc = EINVAL; 452 goto fail1; 453 } 454 455 /* 456 * Report the hashing support the client gets by default if it 457 * does not allocate an RSS context itself. 458 */ 459 *supportp = enp->en_hash_support; 460 461 return (0); 462 463 fail1: 464 EFSYS_PROBE1(fail1, efx_rc_t, rc); 465 466 return (rc); 467 } 468 469 __checkReturn efx_rc_t 470 efx_rx_scale_default_support_get( 471 __in efx_nic_t *enp, 472 __out efx_rx_scale_context_type_t *typep) 473 { 474 efx_rc_t rc; 475 476 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 477 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 478 479 if (typep == NULL) { 480 rc = EINVAL; 481 goto fail1; 482 } 483 484 /* 485 * Report the RSS support the client gets by default if it 486 * does not allocate an RSS context itself. 487 */ 488 *typep = enp->en_rss_context_type; 489 490 return (0); 491 492 fail1: 493 EFSYS_PROBE1(fail1, efx_rc_t, rc); 494 495 return (rc); 496 } 497 #endif /* EFSYS_OPT_RX_SCALE */ 498 499 #if EFSYS_OPT_RX_SCALE 500 __checkReturn efx_rc_t 501 efx_rx_scale_context_alloc( 502 __in efx_nic_t *enp, 503 __in efx_rx_scale_context_type_t type, 504 __in uint32_t num_queues, 505 __out uint32_t *rss_contextp) 506 { 507 uint32_t table_nentries = EFX_RSS_TBL_SIZE; 508 const efx_rx_ops_t *erxop = enp->en_erxop; 509 efx_rc_t rc; 510 511 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 512 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 513 514 if (erxop->erxo_scale_context_alloc == NULL) { 515 rc = ENOTSUP; 516 goto fail1; 517 } 518 519 if (type == EFX_RX_SCALE_EVEN_SPREAD) 520 table_nentries = 0; 521 522 if ((rc = erxop->erxo_scale_context_alloc(enp, type, num_queues, 523 table_nentries, rss_contextp)) != 0) { 524 goto fail2; 525 } 526 527 return (0); 528 529 fail2: 530 EFSYS_PROBE(fail2); 531 fail1: 532 EFSYS_PROBE1(fail1, efx_rc_t, rc); 533 return (rc); 534 } 535 #endif /* EFSYS_OPT_RX_SCALE */ 536 537 #if EFSYS_OPT_RX_SCALE 538 __checkReturn efx_rc_t 539 efx_rx_scale_context_alloc_v2( 540 __in efx_nic_t *enp, 541 __in efx_rx_scale_context_type_t type, 542 __in uint32_t num_queues, 543 __in uint32_t table_nentries, 544 __out uint32_t *rss_contextp) 545 { 546 const efx_rx_ops_t *erxop = enp->en_erxop; 547 efx_rc_t rc; 548 549 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 550 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 551 552 if (erxop->erxo_scale_context_alloc == NULL) { 553 rc = ENOTSUP; 554 goto fail1; 555 } 556 557 if ((rc = erxop->erxo_scale_context_alloc(enp, type, 558 num_queues, table_nentries, rss_contextp)) != 0) { 559 goto fail2; 560 } 561 562 return (0); 563 564 fail2: 565 EFSYS_PROBE(fail2); 566 fail1: 567 EFSYS_PROBE1(fail1, efx_rc_t, rc); 568 return (rc); 569 } 570 #endif /* EFSYS_OPT_RX_SCALE */ 571 572 #if EFSYS_OPT_RX_SCALE 573 __checkReturn efx_rc_t 574 efx_rx_scale_context_free( 575 __in efx_nic_t *enp, 576 __in uint32_t rss_context) 577 { 578 const efx_rx_ops_t *erxop = enp->en_erxop; 579 efx_rc_t rc; 580 581 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 582 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 583 584 if (erxop->erxo_scale_context_free == NULL) { 585 rc = ENOTSUP; 586 goto fail1; 587 } 588 if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0) 589 goto fail2; 590 591 return (0); 592 593 fail2: 594 EFSYS_PROBE(fail2); 595 fail1: 596 EFSYS_PROBE1(fail1, efx_rc_t, rc); 597 return (rc); 598 } 599 #endif /* EFSYS_OPT_RX_SCALE */ 600 601 #if EFSYS_OPT_RX_SCALE 602 __checkReturn efx_rc_t 603 efx_rx_scale_mode_set( 604 __in efx_nic_t *enp, 605 __in uint32_t rss_context, 606 __in efx_rx_hash_alg_t alg, 607 __in efx_rx_hash_type_t type, 608 __in boolean_t insert) 609 { 610 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 611 const efx_rx_ops_t *erxop = enp->en_erxop; 612 efx_rx_hash_type_t type_check; 613 unsigned int i; 614 efx_rc_t rc; 615 616 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 617 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 618 619 /* 620 * Legacy flags and modern bits cannot be 621 * used at the same time in the hash type. 622 */ 623 if ((type & EFX_RX_HASH_LEGACY_MASK) && 624 (type & ~EFX_RX_HASH_LEGACY_MASK)) { 625 rc = EINVAL; 626 goto fail1; 627 } 628 629 /* 630 * If RSS hash type is represented by additional bits 631 * in the value, the latter need to be verified since 632 * not all bit combinations are valid RSS modes. Also, 633 * depending on the firmware, some valid combinations 634 * may be unsupported. Discern additional bits in the 635 * type value and try to recognise valid combinations. 636 * If some bits remain unrecognised, report the error. 637 */ 638 type_check = type & ~EFX_RX_HASH_LEGACY_MASK; 639 if (type_check != 0) { 640 unsigned int type_flags[EFX_RX_HASH_NFLAGS]; 641 unsigned int type_nflags; 642 643 rc = efx_rx_scale_hash_flags_get(enp, alg, type_flags, 644 EFX_ARRAY_SIZE(type_flags), &type_nflags); 645 if (rc != 0) 646 goto fail2; 647 648 for (i = 0; i < type_nflags; ++i) { 649 if ((type_check & type_flags[i]) == type_flags[i]) 650 type_check &= ~(type_flags[i]); 651 } 652 653 if (type_check != 0) { 654 rc = EINVAL; 655 goto fail3; 656 } 657 } 658 659 /* 660 * Translate EFX_RX_HASH() flags to their legacy counterparts 661 * provided that the FW claims no support for additional modes. 662 */ 663 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE) { 664 efx_rx_hash_type_t t_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE) | 665 EFX_RX_HASH(IPV4_TCP, 2TUPLE); 666 efx_rx_hash_type_t t_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE) | 667 EFX_RX_HASH(IPV6_TCP, 2TUPLE); 668 efx_rx_hash_type_t t_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE); 669 efx_rx_hash_type_t t_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE); 670 671 if ((type & t_ipv4) == t_ipv4) 672 type |= EFX_RX_HASH_IPV4; 673 if ((type & t_ipv6) == t_ipv6) 674 type |= EFX_RX_HASH_IPV6; 675 676 if (encp->enc_rx_scale_l4_hash_supported == B_TRUE) { 677 if ((type & t_ipv4_tcp) == t_ipv4_tcp) 678 type |= EFX_RX_HASH_TCPIPV4; 679 if ((type & t_ipv6_tcp) == t_ipv6_tcp) 680 type |= EFX_RX_HASH_TCPIPV6; 681 } 682 683 type &= EFX_RX_HASH_LEGACY_MASK; 684 } 685 686 if (erxop->erxo_scale_mode_set != NULL) { 687 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg, 688 type, insert)) != 0) 689 goto fail4; 690 } 691 692 return (0); 693 694 fail4: 695 EFSYS_PROBE(fail4); 696 fail3: 697 EFSYS_PROBE(fail3); 698 fail2: 699 EFSYS_PROBE(fail2); 700 fail1: 701 EFSYS_PROBE1(fail1, efx_rc_t, rc); 702 return (rc); 703 } 704 #endif /* EFSYS_OPT_RX_SCALE */ 705 706 #if EFSYS_OPT_RX_SCALE 707 __checkReturn efx_rc_t 708 efx_rx_scale_key_set( 709 __in efx_nic_t *enp, 710 __in uint32_t rss_context, 711 __in_ecount(n) uint8_t *key, 712 __in size_t n) 713 { 714 const efx_rx_ops_t *erxop = enp->en_erxop; 715 efx_rc_t rc; 716 717 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 718 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 719 720 if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0) 721 goto fail1; 722 723 return (0); 724 725 fail1: 726 EFSYS_PROBE1(fail1, efx_rc_t, rc); 727 728 return (rc); 729 } 730 #endif /* EFSYS_OPT_RX_SCALE */ 731 732 #if EFSYS_OPT_RX_SCALE 733 __checkReturn efx_rc_t 734 efx_rx_scale_tbl_set( 735 __in efx_nic_t *enp, 736 __in uint32_t rss_context, 737 __in_ecount(nentries) unsigned int *table, 738 __in size_t nentries) 739 { 740 const efx_rx_ops_t *erxop = enp->en_erxop; 741 efx_rc_t rc; 742 743 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 744 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 745 746 if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, 747 nentries)) != 0) 748 goto fail1; 749 750 return (0); 751 752 fail1: 753 EFSYS_PROBE1(fail1, efx_rc_t, rc); 754 755 return (rc); 756 } 757 #endif /* EFSYS_OPT_RX_SCALE */ 758 759 void 760 efx_rx_qpost( 761 __in efx_rxq_t *erp, 762 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 763 __in size_t size, 764 __in unsigned int ndescs, 765 __in unsigned int completed, 766 __in unsigned int added) 767 { 768 efx_nic_t *enp = erp->er_enp; 769 const efx_rx_ops_t *erxop = enp->en_erxop; 770 771 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 772 EFSYS_ASSERT(erp->er_buf_size == 0 || size == erp->er_buf_size); 773 774 erxop->erxo_qpost(erp, addrp, size, ndescs, completed, added); 775 } 776 777 #if EFSYS_OPT_RX_PACKED_STREAM 778 779 void 780 efx_rx_qpush_ps_credits( 781 __in efx_rxq_t *erp) 782 { 783 efx_nic_t *enp = erp->er_enp; 784 const efx_rx_ops_t *erxop = enp->en_erxop; 785 786 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 787 788 erxop->erxo_qpush_ps_credits(erp); 789 } 790 791 __checkReturn uint8_t * 792 efx_rx_qps_packet_info( 793 __in efx_rxq_t *erp, 794 __in uint8_t *buffer, 795 __in uint32_t buffer_length, 796 __in uint32_t current_offset, 797 __out uint16_t *lengthp, 798 __out uint32_t *next_offsetp, 799 __out uint32_t *timestamp) 800 { 801 efx_nic_t *enp = erp->er_enp; 802 const efx_rx_ops_t *erxop = enp->en_erxop; 803 804 return (erxop->erxo_qps_packet_info(erp, buffer, 805 buffer_length, current_offset, lengthp, 806 next_offsetp, timestamp)); 807 } 808 809 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 810 811 void 812 efx_rx_qpush( 813 __in efx_rxq_t *erp, 814 __in unsigned int added, 815 __inout unsigned int *pushedp) 816 { 817 efx_nic_t *enp = erp->er_enp; 818 const efx_rx_ops_t *erxop = enp->en_erxop; 819 820 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 821 822 erxop->erxo_qpush(erp, added, pushedp); 823 } 824 825 __checkReturn efx_rc_t 826 efx_rx_qflush( 827 __in efx_rxq_t *erp) 828 { 829 efx_nic_t *enp = erp->er_enp; 830 const efx_rx_ops_t *erxop = enp->en_erxop; 831 efx_rc_t rc; 832 833 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 834 835 if ((rc = erxop->erxo_qflush(erp)) != 0) 836 goto fail1; 837 838 return (0); 839 840 fail1: 841 EFSYS_PROBE1(fail1, efx_rc_t, rc); 842 843 return (rc); 844 } 845 846 __checkReturn size_t 847 efx_rxq_size( 848 __in const efx_nic_t *enp, 849 __in unsigned int ndescs) 850 { 851 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 852 853 return (ndescs * encp->enc_rx_desc_size); 854 } 855 856 __checkReturn unsigned int 857 efx_rxq_nbufs( 858 __in const efx_nic_t *enp, 859 __in unsigned int ndescs) 860 { 861 return (EFX_DIV_ROUND_UP(efx_rxq_size(enp, ndescs), EFX_BUF_SIZE)); 862 } 863 864 void 865 efx_rx_qenable( 866 __in efx_rxq_t *erp) 867 { 868 efx_nic_t *enp = erp->er_enp; 869 const efx_rx_ops_t *erxop = enp->en_erxop; 870 871 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 872 873 erxop->erxo_qenable(erp); 874 } 875 876 static __checkReturn efx_rc_t 877 efx_rx_qcreate_internal( 878 __in efx_nic_t *enp, 879 __in unsigned int index, 880 __in unsigned int label, 881 __in efx_rxq_type_t type, 882 __in_opt const efx_rxq_type_data_t *type_data, 883 __in efsys_mem_t *esmp, 884 __in size_t ndescs, 885 __in uint32_t id, 886 __in unsigned int flags, 887 __in efx_evq_t *eep, 888 __deref_out efx_rxq_t **erpp) 889 { 890 const efx_rx_ops_t *erxop = enp->en_erxop; 891 efx_rxq_t *erp; 892 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 893 efx_rc_t rc; 894 895 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 896 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 897 898 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 899 900 EFSYS_ASSERT(ISP2(encp->enc_rxq_max_ndescs)); 901 EFSYS_ASSERT(ISP2(encp->enc_rxq_min_ndescs)); 902 903 if (index >= encp->enc_rxq_limit) { 904 rc = EINVAL; 905 goto fail1; 906 } 907 908 if (!ISP2(ndescs) || 909 ndescs < encp->enc_rxq_min_ndescs || 910 ndescs > encp->enc_rxq_max_ndescs) { 911 rc = EINVAL; 912 goto fail2; 913 } 914 915 /* Allocate an RXQ object */ 916 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 917 918 if (erp == NULL) { 919 rc = ENOMEM; 920 goto fail3; 921 } 922 923 erp->er_magic = EFX_RXQ_MAGIC; 924 erp->er_enp = enp; 925 erp->er_index = index; 926 erp->er_mask = ndescs - 1; 927 erp->er_esmp = esmp; 928 929 if ((rc = erxop->erxo_qcreate(enp, index, label, type, type_data, esmp, 930 ndescs, id, flags, eep, erp)) != 0) 931 goto fail4; 932 933 /* Sanity check queue creation result */ 934 if (flags & EFX_RXQ_FLAG_RSS_HASH) { 935 const efx_rx_prefix_layout_t *erplp = &erp->er_prefix_layout; 936 const efx_rx_prefix_field_info_t *rss_hash_field; 937 938 rss_hash_field = 939 &erplp->erpl_fields[EFX_RX_PREFIX_FIELD_RSS_HASH]; 940 if (rss_hash_field->erpfi_width_bits == 0) { 941 rc = ENOTSUP; 942 goto fail5; 943 } 944 } 945 946 if (flags & EFX_RXQ_FLAG_VLAN_STRIPPED_TCI) { 947 const efx_rx_prefix_layout_t *erplp = &erp->er_prefix_layout; 948 const efx_rx_prefix_field_info_t *vlan_tci_field; 949 950 vlan_tci_field = 951 &erplp->erpl_fields[EFX_RX_PREFIX_FIELD_VLAN_STRIP_TCI]; 952 if (vlan_tci_field->erpfi_width_bits == 0) { 953 rc = ENOTSUP; 954 goto fail6; 955 } 956 } 957 958 enp->en_rx_qcount++; 959 *erpp = erp; 960 961 return (0); 962 963 fail6: 964 EFSYS_PROBE(fail6); 965 fail5: 966 EFSYS_PROBE(fail5); 967 968 erxop->erxo_qdestroy(erp); 969 fail4: 970 EFSYS_PROBE(fail4); 971 972 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 973 fail3: 974 EFSYS_PROBE(fail3); 975 fail2: 976 EFSYS_PROBE(fail2); 977 fail1: 978 EFSYS_PROBE1(fail1, efx_rc_t, rc); 979 980 return (rc); 981 } 982 983 __checkReturn efx_rc_t 984 efx_rx_qcreate( 985 __in efx_nic_t *enp, 986 __in unsigned int index, 987 __in unsigned int label, 988 __in efx_rxq_type_t type, 989 __in size_t buf_size, 990 __in efsys_mem_t *esmp, 991 __in size_t ndescs, 992 __in uint32_t id, 993 __in unsigned int flags, 994 __in efx_evq_t *eep, 995 __deref_out efx_rxq_t **erpp) 996 { 997 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 998 efx_rxq_type_data_t type_data; 999 efx_rc_t rc; 1000 1001 if (buf_size > encp->enc_rx_dma_desc_size_max) { 1002 rc = EINVAL; 1003 goto fail1; 1004 } 1005 1006 memset(&type_data, 0, sizeof (type_data)); 1007 1008 type_data.ertd_default.ed_buf_size = buf_size; 1009 1010 return efx_rx_qcreate_internal(enp, index, label, type, &type_data, 1011 esmp, ndescs, id, flags, eep, erpp); 1012 1013 fail1: 1014 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1015 1016 return (rc); 1017 } 1018 1019 #if EFSYS_OPT_RX_PACKED_STREAM 1020 1021 __checkReturn efx_rc_t 1022 efx_rx_qcreate_packed_stream( 1023 __in efx_nic_t *enp, 1024 __in unsigned int index, 1025 __in unsigned int label, 1026 __in uint32_t ps_buf_size, 1027 __in efsys_mem_t *esmp, 1028 __in size_t ndescs, 1029 __in efx_evq_t *eep, 1030 __deref_out efx_rxq_t **erpp) 1031 { 1032 efx_rxq_type_data_t type_data; 1033 1034 memset(&type_data, 0, sizeof (type_data)); 1035 1036 type_data.ertd_packed_stream.eps_buf_size = ps_buf_size; 1037 1038 return efx_rx_qcreate_internal(enp, index, label, 1039 EFX_RXQ_TYPE_PACKED_STREAM, &type_data, esmp, ndescs, 1040 0 /* id unused on EF10 */, EFX_RXQ_FLAG_NONE, eep, erpp); 1041 } 1042 1043 #endif 1044 1045 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 1046 1047 __checkReturn efx_rc_t 1048 efx_rx_qcreate_es_super_buffer( 1049 __in efx_nic_t *enp, 1050 __in unsigned int index, 1051 __in unsigned int label, 1052 __in uint32_t n_bufs_per_desc, 1053 __in uint32_t max_dma_len, 1054 __in uint32_t buf_stride, 1055 __in uint32_t hol_block_timeout, 1056 __in efsys_mem_t *esmp, 1057 __in size_t ndescs, 1058 __in unsigned int flags, 1059 __in efx_evq_t *eep, 1060 __deref_out efx_rxq_t **erpp) 1061 { 1062 efx_rc_t rc; 1063 efx_rxq_type_data_t type_data; 1064 1065 if (hol_block_timeout > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) { 1066 rc = EINVAL; 1067 goto fail1; 1068 } 1069 1070 memset(&type_data, 0, sizeof (type_data)); 1071 1072 type_data.ertd_es_super_buffer.eessb_bufs_per_desc = n_bufs_per_desc; 1073 type_data.ertd_es_super_buffer.eessb_max_dma_len = max_dma_len; 1074 type_data.ertd_es_super_buffer.eessb_buf_stride = buf_stride; 1075 type_data.ertd_es_super_buffer.eessb_hol_block_timeout = 1076 hol_block_timeout; 1077 1078 rc = efx_rx_qcreate_internal(enp, index, label, 1079 EFX_RXQ_TYPE_ES_SUPER_BUFFER, &type_data, esmp, ndescs, 1080 0 /* id unused on EF10 */, flags, eep, erpp); 1081 if (rc != 0) 1082 goto fail2; 1083 1084 return (0); 1085 1086 fail2: 1087 EFSYS_PROBE(fail2); 1088 fail1: 1089 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1090 1091 return (rc); 1092 } 1093 1094 #endif 1095 1096 1097 void 1098 efx_rx_qdestroy( 1099 __in efx_rxq_t *erp) 1100 { 1101 efx_nic_t *enp = erp->er_enp; 1102 const efx_rx_ops_t *erxop = enp->en_erxop; 1103 1104 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1105 1106 EFSYS_ASSERT(enp->en_rx_qcount != 0); 1107 --enp->en_rx_qcount; 1108 1109 erxop->erxo_qdestroy(erp); 1110 1111 /* Free the RXQ object */ 1112 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1113 } 1114 1115 __checkReturn efx_rc_t 1116 efx_pseudo_hdr_pkt_length_get( 1117 __in efx_rxq_t *erp, 1118 __in uint8_t *buffer, 1119 __out uint16_t *lengthp) 1120 { 1121 efx_nic_t *enp = erp->er_enp; 1122 const efx_rx_ops_t *erxop = enp->en_erxop; 1123 1124 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1125 1126 return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp)); 1127 } 1128 1129 #if EFSYS_OPT_RX_SCALE 1130 __checkReturn uint32_t 1131 efx_pseudo_hdr_hash_get( 1132 __in efx_rxq_t *erp, 1133 __in efx_rx_hash_alg_t func, 1134 __in uint8_t *buffer) 1135 { 1136 efx_nic_t *enp = erp->er_enp; 1137 const efx_rx_ops_t *erxop = enp->en_erxop; 1138 1139 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1140 1141 EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE); 1142 return (erxop->erxo_prefix_hash(enp, func, buffer)); 1143 } 1144 #endif /* EFSYS_OPT_RX_SCALE */ 1145 1146 __checkReturn efx_rc_t 1147 efx_rx_prefix_get_layout( 1148 __in const efx_rxq_t *erp, 1149 __out efx_rx_prefix_layout_t *erplp) 1150 { 1151 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1152 1153 *erplp = erp->er_prefix_layout; 1154 1155 return (0); 1156 } 1157 1158 #if EFSYS_OPT_SIENA 1159 1160 static __checkReturn efx_rc_t 1161 siena_rx_init( 1162 __in efx_nic_t *enp) 1163 { 1164 efx_oword_t oword; 1165 unsigned int index; 1166 1167 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 1168 1169 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 1170 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 1171 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 1172 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 1173 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 1174 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 1175 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 1176 1177 /* Zero the RSS table */ 1178 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 1179 index++) { 1180 EFX_ZERO_OWORD(oword); 1181 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1182 index, &oword, B_TRUE); 1183 } 1184 1185 #if EFSYS_OPT_RX_SCALE 1186 /* The RSS key and indirection table are writable. */ 1187 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 1188 1189 /* Hardware can insert RX hash with/without RSS */ 1190 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 1191 #endif /* EFSYS_OPT_RX_SCALE */ 1192 1193 return (0); 1194 } 1195 1196 #if EFSYS_OPT_RX_SCATTER 1197 static __checkReturn efx_rc_t 1198 siena_rx_scatter_enable( 1199 __in efx_nic_t *enp, 1200 __in unsigned int buf_size) 1201 { 1202 unsigned int nbuf32; 1203 efx_oword_t oword; 1204 efx_rc_t rc; 1205 1206 nbuf32 = buf_size / 32; 1207 if ((nbuf32 == 0) || 1208 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 1209 ((buf_size % 32) != 0)) { 1210 rc = EINVAL; 1211 goto fail1; 1212 } 1213 1214 if (enp->en_rx_qcount > 0) { 1215 rc = EBUSY; 1216 goto fail2; 1217 } 1218 1219 /* Set scatter buffer size */ 1220 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 1221 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 1222 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 1223 1224 /* Enable scatter for packets not matching a filter */ 1225 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 1226 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 1227 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 1228 1229 return (0); 1230 1231 fail2: 1232 EFSYS_PROBE(fail2); 1233 fail1: 1234 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1235 1236 return (rc); 1237 } 1238 #endif /* EFSYS_OPT_RX_SCATTER */ 1239 1240 1241 #define EFX_RX_LFSR_HASH(_enp, _insert) \ 1242 do { \ 1243 efx_oword_t oword; \ 1244 \ 1245 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1246 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 1247 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 1248 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 1249 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 1250 (_insert) ? 1 : 0); \ 1251 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1252 \ 1253 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 1254 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 1255 &oword); \ 1256 EFX_SET_OWORD_FIELD(oword, \ 1257 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 1258 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 1259 &oword); \ 1260 } \ 1261 \ 1262 _NOTE(CONSTANTCONDITION) \ 1263 } while (B_FALSE) 1264 1265 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 1266 do { \ 1267 efx_oword_t oword; \ 1268 \ 1269 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1270 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 1271 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 1272 (_ip) ? 1 : 0); \ 1273 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 1274 (_tcp) ? 0 : 1); \ 1275 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 1276 (_insert) ? 1 : 0); \ 1277 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1278 \ 1279 _NOTE(CONSTANTCONDITION) \ 1280 } while (B_FALSE) 1281 1282 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 1283 do { \ 1284 efx_oword_t oword; \ 1285 \ 1286 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 1287 EFX_SET_OWORD_FIELD(oword, \ 1288 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 1289 EFX_SET_OWORD_FIELD(oword, \ 1290 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 1291 EFX_SET_OWORD_FIELD(oword, \ 1292 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 1293 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 1294 \ 1295 (_rc) = 0; \ 1296 \ 1297 _NOTE(CONSTANTCONDITION) \ 1298 } while (B_FALSE) 1299 1300 1301 #if EFSYS_OPT_RX_SCALE 1302 1303 static __checkReturn efx_rc_t 1304 siena_rx_scale_mode_set( 1305 __in efx_nic_t *enp, 1306 __in uint32_t rss_context, 1307 __in efx_rx_hash_alg_t alg, 1308 __in efx_rx_hash_type_t type, 1309 __in boolean_t insert) 1310 { 1311 efx_rc_t rc; 1312 1313 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1314 rc = EINVAL; 1315 goto fail1; 1316 } 1317 1318 switch (alg) { 1319 case EFX_RX_HASHALG_LFSR: 1320 EFX_RX_LFSR_HASH(enp, insert); 1321 break; 1322 1323 case EFX_RX_HASHALG_TOEPLITZ: 1324 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 1325 (type & EFX_RX_HASH_IPV4) ? B_TRUE : B_FALSE, 1326 (type & EFX_RX_HASH_TCPIPV4) ? B_TRUE : B_FALSE); 1327 1328 EFX_RX_TOEPLITZ_IPV6_HASH(enp, 1329 (type & EFX_RX_HASH_IPV6) ? B_TRUE : B_FALSE, 1330 (type & EFX_RX_HASH_TCPIPV6) ? B_TRUE : B_FALSE, 1331 rc); 1332 if (rc != 0) 1333 goto fail2; 1334 1335 break; 1336 1337 default: 1338 rc = EINVAL; 1339 goto fail3; 1340 } 1341 1342 return (0); 1343 1344 fail3: 1345 EFSYS_PROBE(fail3); 1346 fail2: 1347 EFSYS_PROBE(fail2); 1348 fail1: 1349 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1350 1351 EFX_RX_LFSR_HASH(enp, B_FALSE); 1352 1353 return (rc); 1354 } 1355 #endif 1356 1357 #if EFSYS_OPT_RX_SCALE 1358 static __checkReturn efx_rc_t 1359 siena_rx_scale_key_set( 1360 __in efx_nic_t *enp, 1361 __in uint32_t rss_context, 1362 __in_ecount(n) uint8_t *key, 1363 __in size_t n) 1364 { 1365 efx_oword_t oword; 1366 unsigned int byte; 1367 unsigned int offset; 1368 efx_rc_t rc; 1369 1370 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1371 rc = EINVAL; 1372 goto fail1; 1373 } 1374 1375 byte = 0; 1376 1377 /* Write Toeplitz IPv4 hash key */ 1378 EFX_ZERO_OWORD(oword); 1379 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 1380 offset > 0 && byte < n; 1381 --offset) 1382 oword.eo_u8[offset - 1] = key[byte++]; 1383 1384 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 1385 1386 byte = 0; 1387 1388 /* Verify Toeplitz IPv4 hash key */ 1389 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 1390 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 1391 offset > 0 && byte < n; 1392 --offset) { 1393 if (oword.eo_u8[offset - 1] != key[byte++]) { 1394 rc = EFAULT; 1395 goto fail2; 1396 } 1397 } 1398 1399 if ((enp->en_features & EFX_FEATURE_IPV6) == 0) 1400 goto done; 1401 1402 byte = 0; 1403 1404 /* Write Toeplitz IPv6 hash key 3 */ 1405 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1406 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1407 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1408 offset > 0 && byte < n; 1409 --offset) 1410 oword.eo_u8[offset - 1] = key[byte++]; 1411 1412 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1413 1414 /* Write Toeplitz IPv6 hash key 2 */ 1415 EFX_ZERO_OWORD(oword); 1416 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1417 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1418 offset > 0 && byte < n; 1419 --offset) 1420 oword.eo_u8[offset - 1] = key[byte++]; 1421 1422 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1423 1424 /* Write Toeplitz IPv6 hash key 1 */ 1425 EFX_ZERO_OWORD(oword); 1426 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1427 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1428 offset > 0 && byte < n; 1429 --offset) 1430 oword.eo_u8[offset - 1] = key[byte++]; 1431 1432 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1433 1434 byte = 0; 1435 1436 /* Verify Toeplitz IPv6 hash key 3 */ 1437 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1438 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1439 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1440 offset > 0 && byte < n; 1441 --offset) { 1442 if (oword.eo_u8[offset - 1] != key[byte++]) { 1443 rc = EFAULT; 1444 goto fail3; 1445 } 1446 } 1447 1448 /* Verify Toeplitz IPv6 hash key 2 */ 1449 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1450 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1451 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1452 offset > 0 && byte < n; 1453 --offset) { 1454 if (oword.eo_u8[offset - 1] != key[byte++]) { 1455 rc = EFAULT; 1456 goto fail4; 1457 } 1458 } 1459 1460 /* Verify Toeplitz IPv6 hash key 1 */ 1461 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1462 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1463 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1464 offset > 0 && byte < n; 1465 --offset) { 1466 if (oword.eo_u8[offset - 1] != key[byte++]) { 1467 rc = EFAULT; 1468 goto fail5; 1469 } 1470 } 1471 1472 done: 1473 return (0); 1474 1475 fail5: 1476 EFSYS_PROBE(fail5); 1477 fail4: 1478 EFSYS_PROBE(fail4); 1479 fail3: 1480 EFSYS_PROBE(fail3); 1481 fail2: 1482 EFSYS_PROBE(fail2); 1483 fail1: 1484 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1485 1486 return (rc); 1487 } 1488 #endif 1489 1490 #if EFSYS_OPT_RX_SCALE 1491 static __checkReturn efx_rc_t 1492 siena_rx_scale_tbl_set( 1493 __in efx_nic_t *enp, 1494 __in uint32_t rss_context, 1495 __in_ecount(nentries) unsigned int *table, 1496 __in size_t nentries) 1497 { 1498 efx_oword_t oword; 1499 int index; 1500 efx_rc_t rc; 1501 1502 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 1503 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 1504 1505 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1506 rc = EINVAL; 1507 goto fail1; 1508 } 1509 1510 if (nentries > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 1511 rc = EINVAL; 1512 goto fail2; 1513 } 1514 1515 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 1516 uint32_t byte; 1517 1518 /* Calculate the entry to place in the table */ 1519 byte = (nentries > 0) ? (uint32_t)table[index % nentries] : 0; 1520 1521 EFSYS_PROBE2(table, int, index, uint32_t, byte); 1522 1523 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 1524 1525 /* Write the table */ 1526 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1527 index, &oword, B_TRUE); 1528 } 1529 1530 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 1531 uint32_t byte; 1532 1533 /* Determine if we're starting a new batch */ 1534 byte = (nentries > 0) ? (uint32_t)table[index % nentries] : 0; 1535 1536 /* Read the table */ 1537 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 1538 index, &oword, B_TRUE); 1539 1540 /* Verify the entry */ 1541 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 1542 rc = EFAULT; 1543 goto fail3; 1544 } 1545 } 1546 1547 return (0); 1548 1549 fail3: 1550 EFSYS_PROBE(fail3); 1551 fail2: 1552 EFSYS_PROBE(fail2); 1553 fail1: 1554 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1555 1556 return (rc); 1557 } 1558 #endif 1559 1560 /* 1561 * Falcon/Siena pseudo-header 1562 * -------------------------- 1563 * 1564 * Receive packets are prefixed by an optional 16 byte pseudo-header. 1565 * The pseudo-header is a byte array of one of the forms: 1566 * 1567 * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 1568 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT 1569 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL 1570 * 1571 * where: 1572 * TT.TT.TT.TT Toeplitz hash (32-bit big-endian) 1573 * LL.LL LFSR hash (16-bit big-endian) 1574 */ 1575 1576 /* 1577 * Provide Rx prefix layout with Toeplitz hash only since LSFR is 1578 * used by no supported drivers. 1579 * 1580 * Siena does not support Rx prefix choice via MC_CMD_GET_RX_PREFIX_ID 1581 * and query its layout using MC_CMD_QUERY_RX_PREFIX_ID. 1582 */ 1583 static const efx_rx_prefix_layout_t siena_toeplitz_rx_prefix_layout = { 1584 .erpl_id = 0, 1585 .erpl_length = 16, 1586 .erpl_fields = { 1587 [EFX_RX_PREFIX_FIELD_RSS_HASH] = { 12 * 8, 32, B_TRUE }, 1588 } 1589 }; 1590 1591 #if EFSYS_OPT_RX_SCALE 1592 static __checkReturn uint32_t 1593 siena_rx_prefix_hash( 1594 __in efx_nic_t *enp, 1595 __in efx_rx_hash_alg_t func, 1596 __in uint8_t *buffer) 1597 { 1598 _NOTE(ARGUNUSED(enp)) 1599 1600 switch (func) { 1601 case EFX_RX_HASHALG_TOEPLITZ: 1602 return ((buffer[12] << 24) | 1603 (buffer[13] << 16) | 1604 (buffer[14] << 8) | 1605 buffer[15]); 1606 1607 case EFX_RX_HASHALG_LFSR: 1608 return ((buffer[14] << 8) | buffer[15]); 1609 1610 default: 1611 EFSYS_ASSERT(0); 1612 return (0); 1613 } 1614 } 1615 #endif /* EFSYS_OPT_RX_SCALE */ 1616 1617 static __checkReturn efx_rc_t 1618 siena_rx_prefix_pktlen( 1619 __in efx_nic_t *enp, 1620 __in uint8_t *buffer, 1621 __out uint16_t *lengthp) 1622 { 1623 _NOTE(ARGUNUSED(enp, buffer, lengthp)) 1624 1625 /* Not supported by Falcon/Siena hardware */ 1626 EFSYS_ASSERT(0); 1627 return (ENOTSUP); 1628 } 1629 1630 1631 static void 1632 siena_rx_qpost( 1633 __in efx_rxq_t *erp, 1634 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 1635 __in size_t size, 1636 __in unsigned int ndescs, 1637 __in unsigned int completed, 1638 __in unsigned int added) 1639 { 1640 efx_qword_t qword; 1641 unsigned int i; 1642 unsigned int offset; 1643 unsigned int id; 1644 1645 /* The client driver must not overfill the queue */ 1646 EFSYS_ASSERT3U(added - completed + ndescs, <=, 1647 EFX_RXQ_LIMIT(erp->er_mask + 1)); 1648 1649 id = added & (erp->er_mask); 1650 for (i = 0; i < ndescs; i++) { 1651 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 1652 unsigned int, id, efsys_dma_addr_t, addrp[i], 1653 size_t, size); 1654 1655 EFX_POPULATE_QWORD_3(qword, 1656 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 1657 FSF_AZ_RX_KER_BUF_ADDR_DW0, 1658 (uint32_t)(addrp[i] & 0xffffffff), 1659 FSF_AZ_RX_KER_BUF_ADDR_DW1, 1660 (uint32_t)(addrp[i] >> 32)); 1661 1662 offset = id * sizeof (efx_qword_t); 1663 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 1664 1665 id = (id + 1) & (erp->er_mask); 1666 } 1667 } 1668 1669 static void 1670 siena_rx_qpush( 1671 __in efx_rxq_t *erp, 1672 __in unsigned int added, 1673 __inout unsigned int *pushedp) 1674 { 1675 efx_nic_t *enp = erp->er_enp; 1676 unsigned int pushed = *pushedp; 1677 uint32_t wptr; 1678 efx_oword_t oword; 1679 efx_dword_t dword; 1680 1681 /* All descriptors are pushed */ 1682 *pushedp = added; 1683 1684 /* Push the populated descriptors out */ 1685 wptr = added & erp->er_mask; 1686 1687 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 1688 1689 /* Only write the third DWORD */ 1690 EFX_POPULATE_DWORD_1(dword, 1691 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 1692 1693 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 1694 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 1695 SIENA_RXQ_DESC_SIZE, wptr, pushed & erp->er_mask); 1696 EFSYS_PIO_WRITE_BARRIER(); 1697 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 1698 erp->er_index, &dword, B_FALSE); 1699 } 1700 1701 #if EFSYS_OPT_RX_PACKED_STREAM 1702 static void 1703 siena_rx_qpush_ps_credits( 1704 __in efx_rxq_t *erp) 1705 { 1706 /* Not supported by Siena hardware */ 1707 EFSYS_ASSERT(0); 1708 } 1709 1710 static uint8_t * 1711 siena_rx_qps_packet_info( 1712 __in efx_rxq_t *erp, 1713 __in uint8_t *buffer, 1714 __in uint32_t buffer_length, 1715 __in uint32_t current_offset, 1716 __out uint16_t *lengthp, 1717 __out uint32_t *next_offsetp, 1718 __out uint32_t *timestamp) 1719 { 1720 /* Not supported by Siena hardware */ 1721 EFSYS_ASSERT(0); 1722 1723 return (NULL); 1724 } 1725 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1726 1727 static __checkReturn efx_rc_t 1728 siena_rx_qflush( 1729 __in efx_rxq_t *erp) 1730 { 1731 efx_nic_t *enp = erp->er_enp; 1732 efx_oword_t oword; 1733 uint32_t label; 1734 1735 label = erp->er_index; 1736 1737 /* Flush the queue */ 1738 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 1739 FRF_AZ_RX_FLUSH_DESCQ, label); 1740 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 1741 1742 return (0); 1743 } 1744 1745 static void 1746 siena_rx_qenable( 1747 __in efx_rxq_t *erp) 1748 { 1749 efx_nic_t *enp = erp->er_enp; 1750 efx_oword_t oword; 1751 1752 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1753 1754 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 1755 erp->er_index, &oword, B_TRUE); 1756 1757 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 1758 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 1759 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 1760 1761 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1762 erp->er_index, &oword, B_TRUE); 1763 } 1764 1765 static __checkReturn efx_rc_t 1766 siena_rx_qcreate( 1767 __in efx_nic_t *enp, 1768 __in unsigned int index, 1769 __in unsigned int label, 1770 __in efx_rxq_type_t type, 1771 __in_opt const efx_rxq_type_data_t *type_data, 1772 __in efsys_mem_t *esmp, 1773 __in size_t ndescs, 1774 __in uint32_t id, 1775 __in unsigned int flags, 1776 __in efx_evq_t *eep, 1777 __in efx_rxq_t *erp) 1778 { 1779 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1780 efx_oword_t oword; 1781 uint32_t size; 1782 boolean_t jumbo = B_FALSE; 1783 efx_rc_t rc; 1784 1785 _NOTE(ARGUNUSED(esmp)) 1786 1787 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == 1788 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 1789 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1790 1791 for (size = 0; 1792 (1U << size) <= encp->enc_rxq_max_ndescs / encp->enc_rxq_min_ndescs; 1793 size++) 1794 if ((1U << size) == (uint32_t)ndescs / encp->enc_rxq_min_ndescs) 1795 break; 1796 if (id + (1 << size) >= encp->enc_buftbl_limit) { 1797 rc = EINVAL; 1798 goto fail1; 1799 } 1800 1801 switch (type) { 1802 case EFX_RXQ_TYPE_DEFAULT: 1803 erp->er_buf_size = type_data->ertd_default.ed_buf_size; 1804 /* 1805 * Ignore EFX_RXQ_FLAG_RSS_HASH since if RSS hash is calculated 1806 * it is always delivered from HW in the pseudo-header. 1807 */ 1808 break; 1809 1810 default: 1811 rc = EINVAL; 1812 goto fail2; 1813 } 1814 1815 #if EFSYS_OPT_RX_SCATTER 1816 #define SUPPORTED_RXQ_FLAGS EFX_RXQ_FLAG_SCATTER 1817 #else 1818 #define SUPPORTED_RXQ_FLAGS EFX_RXQ_FLAG_NONE 1819 #endif 1820 /* Reject flags for unsupported queue features */ 1821 if ((flags & ~SUPPORTED_RXQ_FLAGS) != 0) { 1822 rc = EINVAL; 1823 goto fail3; 1824 } 1825 #undef SUPPORTED_RXQ_FLAGS 1826 1827 if (flags & EFX_RXQ_FLAG_SCATTER) 1828 jumbo = B_TRUE; 1829 1830 /* Set up the new descriptor queue */ 1831 EFX_POPULATE_OWORD_7(oword, 1832 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 1833 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 1834 FRF_AZ_RX_DESCQ_OWNER_ID, 0, 1835 FRF_AZ_RX_DESCQ_LABEL, label, 1836 FRF_AZ_RX_DESCQ_SIZE, size, 1837 FRF_AZ_RX_DESCQ_TYPE, 0, 1838 FRF_AZ_RX_DESCQ_JUMBO, jumbo); 1839 1840 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1841 erp->er_index, &oword, B_TRUE); 1842 1843 erp->er_prefix_layout = siena_toeplitz_rx_prefix_layout; 1844 1845 return (0); 1846 1847 fail3: 1848 EFSYS_PROBE(fail3); 1849 fail2: 1850 EFSYS_PROBE(fail2); 1851 fail1: 1852 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1853 1854 return (rc); 1855 } 1856 1857 static void 1858 siena_rx_qdestroy( 1859 __in efx_rxq_t *erp) 1860 { 1861 efx_nic_t *enp = erp->er_enp; 1862 efx_oword_t oword; 1863 1864 /* Purge descriptor queue */ 1865 EFX_ZERO_OWORD(oword); 1866 1867 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1868 erp->er_index, &oword, B_TRUE); 1869 } 1870 1871 static void 1872 siena_rx_fini( 1873 __in efx_nic_t *enp) 1874 { 1875 _NOTE(ARGUNUSED(enp)) 1876 } 1877 1878 #endif /* EFSYS_OPT_SIENA */ 1879 1880 static __checkReturn boolean_t 1881 efx_rx_prefix_layout_fields_match( 1882 __in const efx_rx_prefix_field_info_t *erpfip1, 1883 __in const efx_rx_prefix_field_info_t *erpfip2) 1884 { 1885 if (erpfip1->erpfi_offset_bits != erpfip2->erpfi_offset_bits) 1886 return (B_FALSE); 1887 1888 if (erpfip1->erpfi_width_bits != erpfip2->erpfi_width_bits) 1889 return (B_FALSE); 1890 1891 if (erpfip1->erpfi_big_endian != erpfip2->erpfi_big_endian) 1892 return (B_FALSE); 1893 1894 return (B_TRUE); 1895 } 1896 1897 __checkReturn uint32_t 1898 efx_rx_prefix_layout_check( 1899 __in const efx_rx_prefix_layout_t *available, 1900 __in const efx_rx_prefix_layout_t *wanted) 1901 { 1902 uint32_t result = 0; 1903 unsigned int i; 1904 1905 EFX_STATIC_ASSERT(EFX_RX_PREFIX_NFIELDS < sizeof (result) * 8); 1906 for (i = 0; i < EFX_RX_PREFIX_NFIELDS; ++i) { 1907 /* Skip the field if driver does not want to use it */ 1908 if (wanted->erpl_fields[i].erpfi_width_bits == 0) 1909 continue; 1910 1911 if (efx_rx_prefix_layout_fields_match( 1912 &available->erpl_fields[i], 1913 &wanted->erpl_fields[i]) == B_FALSE) 1914 result |= (1U << i); 1915 } 1916 1917 return (result); 1918 } 1919