1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2020 Xilinx, Inc. 4 * Copyright(c) 2007-2019 Solarflare Communications Inc. 5 */ 6 7 #include "efx.h" 8 #include "efx_impl.h" 9 10 11 #if EFSYS_OPT_SIENA 12 13 static __checkReturn efx_rc_t 14 siena_rx_init( 15 __in efx_nic_t *enp); 16 17 static void 18 siena_rx_fini( 19 __in efx_nic_t *enp); 20 21 #if EFSYS_OPT_RX_SCATTER 22 static __checkReturn efx_rc_t 23 siena_rx_scatter_enable( 24 __in efx_nic_t *enp, 25 __in unsigned int buf_size); 26 #endif /* EFSYS_OPT_RX_SCATTER */ 27 28 #if EFSYS_OPT_RX_SCALE 29 static __checkReturn efx_rc_t 30 siena_rx_scale_mode_set( 31 __in efx_nic_t *enp, 32 __in uint32_t rss_context, 33 __in efx_rx_hash_alg_t alg, 34 __in efx_rx_hash_type_t type, 35 __in boolean_t insert); 36 37 static __checkReturn efx_rc_t 38 siena_rx_scale_key_set( 39 __in efx_nic_t *enp, 40 __in uint32_t rss_context, 41 __in_ecount(n) uint8_t *key, 42 __in size_t n); 43 44 static __checkReturn efx_rc_t 45 siena_rx_scale_tbl_set( 46 __in efx_nic_t *enp, 47 __in uint32_t rss_context, 48 __in_ecount(n) unsigned int *table, 49 __in size_t n); 50 51 static __checkReturn uint32_t 52 siena_rx_prefix_hash( 53 __in efx_nic_t *enp, 54 __in efx_rx_hash_alg_t func, 55 __in uint8_t *buffer); 56 57 #endif /* EFSYS_OPT_RX_SCALE */ 58 59 static __checkReturn efx_rc_t 60 siena_rx_prefix_pktlen( 61 __in efx_nic_t *enp, 62 __in uint8_t *buffer, 63 __out uint16_t *lengthp); 64 65 static void 66 siena_rx_qpost( 67 __in efx_rxq_t *erp, 68 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 69 __in size_t size, 70 __in unsigned int ndescs, 71 __in unsigned int completed, 72 __in unsigned int added); 73 74 static void 75 siena_rx_qpush( 76 __in efx_rxq_t *erp, 77 __in unsigned int added, 78 __inout unsigned int *pushedp); 79 80 #if EFSYS_OPT_RX_PACKED_STREAM 81 static void 82 siena_rx_qpush_ps_credits( 83 __in efx_rxq_t *erp); 84 85 static __checkReturn uint8_t * 86 siena_rx_qps_packet_info( 87 __in efx_rxq_t *erp, 88 __in uint8_t *buffer, 89 __in uint32_t buffer_length, 90 __in uint32_t current_offset, 91 __out uint16_t *lengthp, 92 __out uint32_t *next_offsetp, 93 __out uint32_t *timestamp); 94 #endif 95 96 static __checkReturn efx_rc_t 97 siena_rx_qflush( 98 __in efx_rxq_t *erp); 99 100 static void 101 siena_rx_qenable( 102 __in efx_rxq_t *erp); 103 104 static __checkReturn efx_rc_t 105 siena_rx_qcreate( 106 __in efx_nic_t *enp, 107 __in unsigned int index, 108 __in unsigned int label, 109 __in efx_rxq_type_t type, 110 __in_opt const efx_rxq_type_data_t *type_data, 111 __in efsys_mem_t *esmp, 112 __in size_t ndescs, 113 __in uint32_t id, 114 __in unsigned int flags, 115 __in efx_evq_t *eep, 116 __in efx_rxq_t *erp); 117 118 static void 119 siena_rx_qdestroy( 120 __in efx_rxq_t *erp); 121 122 #endif /* EFSYS_OPT_SIENA */ 123 124 125 #if EFSYS_OPT_SIENA 126 static const efx_rx_ops_t __efx_rx_siena_ops = { 127 siena_rx_init, /* erxo_init */ 128 siena_rx_fini, /* erxo_fini */ 129 #if EFSYS_OPT_RX_SCATTER 130 siena_rx_scatter_enable, /* erxo_scatter_enable */ 131 #endif 132 #if EFSYS_OPT_RX_SCALE 133 NULL, /* erxo_scale_context_alloc */ 134 NULL, /* erxo_scale_context_free */ 135 siena_rx_scale_mode_set, /* erxo_scale_mode_set */ 136 siena_rx_scale_key_set, /* erxo_scale_key_set */ 137 siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 138 siena_rx_prefix_hash, /* erxo_prefix_hash */ 139 #endif 140 siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 141 siena_rx_qpost, /* erxo_qpost */ 142 siena_rx_qpush, /* erxo_qpush */ 143 #if EFSYS_OPT_RX_PACKED_STREAM 144 siena_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 145 siena_rx_qps_packet_info, /* erxo_qps_packet_info */ 146 #endif 147 siena_rx_qflush, /* erxo_qflush */ 148 siena_rx_qenable, /* erxo_qenable */ 149 siena_rx_qcreate, /* erxo_qcreate */ 150 siena_rx_qdestroy, /* erxo_qdestroy */ 151 }; 152 #endif /* EFSYS_OPT_SIENA */ 153 154 #if EFX_OPTS_EF10() 155 static const efx_rx_ops_t __efx_rx_ef10_ops = { 156 ef10_rx_init, /* erxo_init */ 157 ef10_rx_fini, /* erxo_fini */ 158 #if EFSYS_OPT_RX_SCATTER 159 ef10_rx_scatter_enable, /* erxo_scatter_enable */ 160 #endif 161 #if EFSYS_OPT_RX_SCALE 162 ef10_rx_scale_context_alloc, /* erxo_scale_context_alloc */ 163 ef10_rx_scale_context_free, /* erxo_scale_context_free */ 164 ef10_rx_scale_mode_set, /* erxo_scale_mode_set */ 165 ef10_rx_scale_key_set, /* erxo_scale_key_set */ 166 ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 167 ef10_rx_prefix_hash, /* erxo_prefix_hash */ 168 #endif 169 ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 170 ef10_rx_qpost, /* erxo_qpost */ 171 ef10_rx_qpush, /* erxo_qpush */ 172 #if EFSYS_OPT_RX_PACKED_STREAM 173 ef10_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 174 ef10_rx_qps_packet_info, /* erxo_qps_packet_info */ 175 #endif 176 ef10_rx_qflush, /* erxo_qflush */ 177 ef10_rx_qenable, /* erxo_qenable */ 178 ef10_rx_qcreate, /* erxo_qcreate */ 179 ef10_rx_qdestroy, /* erxo_qdestroy */ 180 }; 181 #endif /* EFX_OPTS_EF10() */ 182 183 #if EFSYS_OPT_RIVERHEAD 184 static const efx_rx_ops_t __efx_rx_rhead_ops = { 185 rhead_rx_init, /* erxo_init */ 186 rhead_rx_fini, /* erxo_fini */ 187 #if EFSYS_OPT_RX_SCATTER 188 rhead_rx_scatter_enable, /* erxo_scatter_enable */ 189 #endif 190 #if EFSYS_OPT_RX_SCALE 191 rhead_rx_scale_context_alloc, /* erxo_scale_context_alloc */ 192 rhead_rx_scale_context_free, /* erxo_scale_context_free */ 193 rhead_rx_scale_mode_set, /* erxo_scale_mode_set */ 194 rhead_rx_scale_key_set, /* erxo_scale_key_set */ 195 rhead_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 196 rhead_rx_prefix_hash, /* erxo_prefix_hash */ 197 #endif 198 rhead_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 199 rhead_rx_qpost, /* erxo_qpost */ 200 rhead_rx_qpush, /* erxo_qpush */ 201 #if EFSYS_OPT_RX_PACKED_STREAM 202 NULL, /* erxo_qpush_ps_credits */ 203 NULL, /* erxo_qps_packet_info */ 204 #endif 205 rhead_rx_qflush, /* erxo_qflush */ 206 rhead_rx_qenable, /* erxo_qenable */ 207 rhead_rx_qcreate, /* erxo_qcreate */ 208 rhead_rx_qdestroy, /* erxo_qdestroy */ 209 }; 210 #endif /* EFSYS_OPT_RIVERHEAD */ 211 212 213 __checkReturn efx_rc_t 214 efx_rx_init( 215 __inout efx_nic_t *enp) 216 { 217 const efx_rx_ops_t *erxop; 218 efx_rc_t rc; 219 220 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 221 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 222 223 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 224 rc = EINVAL; 225 goto fail1; 226 } 227 228 if (enp->en_mod_flags & EFX_MOD_RX) { 229 rc = EINVAL; 230 goto fail2; 231 } 232 233 switch (enp->en_family) { 234 #if EFSYS_OPT_SIENA 235 case EFX_FAMILY_SIENA: 236 erxop = &__efx_rx_siena_ops; 237 break; 238 #endif /* EFSYS_OPT_SIENA */ 239 240 #if EFSYS_OPT_HUNTINGTON 241 case EFX_FAMILY_HUNTINGTON: 242 erxop = &__efx_rx_ef10_ops; 243 break; 244 #endif /* EFSYS_OPT_HUNTINGTON */ 245 246 #if EFSYS_OPT_MEDFORD 247 case EFX_FAMILY_MEDFORD: 248 erxop = &__efx_rx_ef10_ops; 249 break; 250 #endif /* EFSYS_OPT_MEDFORD */ 251 252 #if EFSYS_OPT_MEDFORD2 253 case EFX_FAMILY_MEDFORD2: 254 erxop = &__efx_rx_ef10_ops; 255 break; 256 #endif /* EFSYS_OPT_MEDFORD2 */ 257 258 #if EFSYS_OPT_RIVERHEAD 259 case EFX_FAMILY_RIVERHEAD: 260 erxop = &__efx_rx_rhead_ops; 261 break; 262 #endif /* EFSYS_OPT_RIVERHEAD */ 263 264 default: 265 EFSYS_ASSERT(0); 266 rc = ENOTSUP; 267 goto fail3; 268 } 269 270 if ((rc = erxop->erxo_init(enp)) != 0) 271 goto fail4; 272 273 enp->en_erxop = erxop; 274 enp->en_mod_flags |= EFX_MOD_RX; 275 return (0); 276 277 fail4: 278 EFSYS_PROBE(fail4); 279 fail3: 280 EFSYS_PROBE(fail3); 281 fail2: 282 EFSYS_PROBE(fail2); 283 fail1: 284 EFSYS_PROBE1(fail1, efx_rc_t, rc); 285 286 enp->en_erxop = NULL; 287 enp->en_mod_flags &= ~EFX_MOD_RX; 288 return (rc); 289 } 290 291 void 292 efx_rx_fini( 293 __in efx_nic_t *enp) 294 { 295 const efx_rx_ops_t *erxop = enp->en_erxop; 296 297 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 298 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 299 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 300 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 301 302 erxop->erxo_fini(enp); 303 304 enp->en_erxop = NULL; 305 enp->en_mod_flags &= ~EFX_MOD_RX; 306 } 307 308 #if EFSYS_OPT_RX_SCATTER 309 __checkReturn efx_rc_t 310 efx_rx_scatter_enable( 311 __in efx_nic_t *enp, 312 __in unsigned int buf_size) 313 { 314 const efx_rx_ops_t *erxop = enp->en_erxop; 315 efx_rc_t rc; 316 317 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 318 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 319 320 if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0) 321 goto fail1; 322 323 return (0); 324 325 fail1: 326 EFSYS_PROBE1(fail1, efx_rc_t, rc); 327 return (rc); 328 } 329 #endif /* EFSYS_OPT_RX_SCATTER */ 330 331 #if EFSYS_OPT_RX_SCALE 332 __checkReturn efx_rc_t 333 efx_rx_scale_hash_flags_get( 334 __in efx_nic_t *enp, 335 __in efx_rx_hash_alg_t hash_alg, 336 __out_ecount_part(max_nflags, *nflagsp) unsigned int *flagsp, 337 __in unsigned int max_nflags, 338 __out unsigned int *nflagsp) 339 { 340 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 341 unsigned int nflags = 0; 342 efx_rc_t rc; 343 344 if (flagsp == NULL || nflagsp == NULL) { 345 rc = EINVAL; 346 goto fail1; 347 } 348 349 if ((encp->enc_rx_scale_hash_alg_mask & (1U << hash_alg)) == 0) { 350 nflags = 0; 351 goto done; 352 } 353 354 /* Helper to add flags word to flags array without buffer overflow */ 355 #define INSERT_FLAGS(_flags) \ 356 do { \ 357 if (nflags >= max_nflags) { \ 358 rc = E2BIG; \ 359 goto fail2; \ 360 } \ 361 *(flagsp + nflags) = (_flags); \ 362 nflags++; \ 363 \ 364 _NOTE(CONSTANTCONDITION) \ 365 } while (B_FALSE) 366 367 if (encp->enc_rx_scale_l4_hash_supported != B_FALSE) { 368 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 4TUPLE)); 369 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 4TUPLE)); 370 } 371 372 if ((encp->enc_rx_scale_l4_hash_supported != B_FALSE) && 373 (encp->enc_rx_scale_additional_modes_supported != B_FALSE)) { 374 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_DST)); 375 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_SRC)); 376 377 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_DST)); 378 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_SRC)); 379 380 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 4TUPLE)); 381 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_DST)); 382 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_SRC)); 383 384 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 4TUPLE)); 385 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_DST)); 386 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_SRC)); 387 } 388 389 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE)); 390 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE)); 391 392 INSERT_FLAGS(EFX_RX_HASH(IPV4, 2TUPLE)); 393 INSERT_FLAGS(EFX_RX_HASH(IPV6, 2TUPLE)); 394 395 if (encp->enc_rx_scale_additional_modes_supported != B_FALSE) { 396 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_DST)); 397 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_SRC)); 398 399 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_DST)); 400 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_SRC)); 401 402 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE)); 403 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_DST)); 404 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_SRC)); 405 406 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE)); 407 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_DST)); 408 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_SRC)); 409 410 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_DST)); 411 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_SRC)); 412 413 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_DST)); 414 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_SRC)); 415 } 416 417 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, DISABLE)); 418 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, DISABLE)); 419 420 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, DISABLE)); 421 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, DISABLE)); 422 423 INSERT_FLAGS(EFX_RX_HASH(IPV4, DISABLE)); 424 INSERT_FLAGS(EFX_RX_HASH(IPV6, DISABLE)); 425 426 #undef INSERT_FLAGS 427 428 done: 429 *nflagsp = nflags; 430 return (0); 431 432 fail2: 433 EFSYS_PROBE(fail2); 434 fail1: 435 EFSYS_PROBE1(fail1, efx_rc_t, rc); 436 437 return (rc); 438 } 439 440 __checkReturn efx_rc_t 441 efx_rx_hash_default_support_get( 442 __in efx_nic_t *enp, 443 __out efx_rx_hash_support_t *supportp) 444 { 445 efx_rc_t rc; 446 447 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 448 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 449 450 if (supportp == NULL) { 451 rc = EINVAL; 452 goto fail1; 453 } 454 455 /* 456 * Report the hashing support the client gets by default if it 457 * does not allocate an RSS context itself. 458 */ 459 *supportp = enp->en_hash_support; 460 461 return (0); 462 463 fail1: 464 EFSYS_PROBE1(fail1, efx_rc_t, rc); 465 466 return (rc); 467 } 468 469 __checkReturn efx_rc_t 470 efx_rx_scale_default_support_get( 471 __in efx_nic_t *enp, 472 __out efx_rx_scale_context_type_t *typep) 473 { 474 efx_rc_t rc; 475 476 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 477 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 478 479 if (typep == NULL) { 480 rc = EINVAL; 481 goto fail1; 482 } 483 484 /* 485 * Report the RSS support the client gets by default if it 486 * does not allocate an RSS context itself. 487 */ 488 *typep = enp->en_rss_context_type; 489 490 return (0); 491 492 fail1: 493 EFSYS_PROBE1(fail1, efx_rc_t, rc); 494 495 return (rc); 496 } 497 #endif /* EFSYS_OPT_RX_SCALE */ 498 499 #if EFSYS_OPT_RX_SCALE 500 __checkReturn efx_rc_t 501 efx_rx_scale_context_alloc( 502 __in efx_nic_t *enp, 503 __in efx_rx_scale_context_type_t type, 504 __in uint32_t num_queues, 505 __out uint32_t *rss_contextp) 506 { 507 const efx_rx_ops_t *erxop = enp->en_erxop; 508 efx_rc_t rc; 509 510 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 511 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 512 513 if (erxop->erxo_scale_context_alloc == NULL) { 514 rc = ENOTSUP; 515 goto fail1; 516 } 517 if ((rc = erxop->erxo_scale_context_alloc(enp, type, 518 num_queues, rss_contextp)) != 0) { 519 goto fail2; 520 } 521 522 return (0); 523 524 fail2: 525 EFSYS_PROBE(fail2); 526 fail1: 527 EFSYS_PROBE1(fail1, efx_rc_t, rc); 528 return (rc); 529 } 530 #endif /* EFSYS_OPT_RX_SCALE */ 531 532 #if EFSYS_OPT_RX_SCALE 533 __checkReturn efx_rc_t 534 efx_rx_scale_context_free( 535 __in efx_nic_t *enp, 536 __in uint32_t rss_context) 537 { 538 const efx_rx_ops_t *erxop = enp->en_erxop; 539 efx_rc_t rc; 540 541 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 542 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 543 544 if (erxop->erxo_scale_context_free == NULL) { 545 rc = ENOTSUP; 546 goto fail1; 547 } 548 if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0) 549 goto fail2; 550 551 return (0); 552 553 fail2: 554 EFSYS_PROBE(fail2); 555 fail1: 556 EFSYS_PROBE1(fail1, efx_rc_t, rc); 557 return (rc); 558 } 559 #endif /* EFSYS_OPT_RX_SCALE */ 560 561 #if EFSYS_OPT_RX_SCALE 562 __checkReturn efx_rc_t 563 efx_rx_scale_mode_set( 564 __in efx_nic_t *enp, 565 __in uint32_t rss_context, 566 __in efx_rx_hash_alg_t alg, 567 __in efx_rx_hash_type_t type, 568 __in boolean_t insert) 569 { 570 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 571 const efx_rx_ops_t *erxop = enp->en_erxop; 572 efx_rx_hash_type_t type_check; 573 unsigned int i; 574 efx_rc_t rc; 575 576 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 577 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 578 579 /* 580 * Legacy flags and modern bits cannot be 581 * used at the same time in the hash type. 582 */ 583 if ((type & EFX_RX_HASH_LEGACY_MASK) && 584 (type & ~EFX_RX_HASH_LEGACY_MASK)) { 585 rc = EINVAL; 586 goto fail1; 587 } 588 589 /* 590 * If RSS hash type is represented by additional bits 591 * in the value, the latter need to be verified since 592 * not all bit combinations are valid RSS modes. Also, 593 * depending on the firmware, some valid combinations 594 * may be unsupported. Discern additional bits in the 595 * type value and try to recognise valid combinations. 596 * If some bits remain unrecognised, report the error. 597 */ 598 type_check = type & ~EFX_RX_HASH_LEGACY_MASK; 599 if (type_check != 0) { 600 unsigned int type_flags[EFX_RX_HASH_NFLAGS]; 601 unsigned int type_nflags; 602 603 rc = efx_rx_scale_hash_flags_get(enp, alg, type_flags, 604 EFX_ARRAY_SIZE(type_flags), &type_nflags); 605 if (rc != 0) 606 goto fail2; 607 608 for (i = 0; i < type_nflags; ++i) { 609 if ((type_check & type_flags[i]) == type_flags[i]) 610 type_check &= ~(type_flags[i]); 611 } 612 613 if (type_check != 0) { 614 rc = EINVAL; 615 goto fail3; 616 } 617 } 618 619 /* 620 * Translate EFX_RX_HASH() flags to their legacy counterparts 621 * provided that the FW claims no support for additional modes. 622 */ 623 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE) { 624 efx_rx_hash_type_t t_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE) | 625 EFX_RX_HASH(IPV4_TCP, 2TUPLE); 626 efx_rx_hash_type_t t_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE) | 627 EFX_RX_HASH(IPV6_TCP, 2TUPLE); 628 efx_rx_hash_type_t t_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE); 629 efx_rx_hash_type_t t_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE); 630 631 if ((type & t_ipv4) == t_ipv4) 632 type |= EFX_RX_HASH_IPV4; 633 if ((type & t_ipv6) == t_ipv6) 634 type |= EFX_RX_HASH_IPV6; 635 636 if (encp->enc_rx_scale_l4_hash_supported == B_TRUE) { 637 if ((type & t_ipv4_tcp) == t_ipv4_tcp) 638 type |= EFX_RX_HASH_TCPIPV4; 639 if ((type & t_ipv6_tcp) == t_ipv6_tcp) 640 type |= EFX_RX_HASH_TCPIPV6; 641 } 642 643 type &= EFX_RX_HASH_LEGACY_MASK; 644 } 645 646 if (erxop->erxo_scale_mode_set != NULL) { 647 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg, 648 type, insert)) != 0) 649 goto fail4; 650 } 651 652 return (0); 653 654 fail4: 655 EFSYS_PROBE(fail4); 656 fail3: 657 EFSYS_PROBE(fail3); 658 fail2: 659 EFSYS_PROBE(fail2); 660 fail1: 661 EFSYS_PROBE1(fail1, efx_rc_t, rc); 662 return (rc); 663 } 664 #endif /* EFSYS_OPT_RX_SCALE */ 665 666 #if EFSYS_OPT_RX_SCALE 667 __checkReturn efx_rc_t 668 efx_rx_scale_key_set( 669 __in efx_nic_t *enp, 670 __in uint32_t rss_context, 671 __in_ecount(n) uint8_t *key, 672 __in size_t n) 673 { 674 const efx_rx_ops_t *erxop = enp->en_erxop; 675 efx_rc_t rc; 676 677 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 678 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 679 680 if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0) 681 goto fail1; 682 683 return (0); 684 685 fail1: 686 EFSYS_PROBE1(fail1, efx_rc_t, rc); 687 688 return (rc); 689 } 690 #endif /* EFSYS_OPT_RX_SCALE */ 691 692 #if EFSYS_OPT_RX_SCALE 693 __checkReturn efx_rc_t 694 efx_rx_scale_tbl_set( 695 __in efx_nic_t *enp, 696 __in uint32_t rss_context, 697 __in_ecount(n) unsigned int *table, 698 __in size_t n) 699 { 700 const efx_rx_ops_t *erxop = enp->en_erxop; 701 efx_rc_t rc; 702 703 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 704 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 705 706 if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, n)) != 0) 707 goto fail1; 708 709 return (0); 710 711 fail1: 712 EFSYS_PROBE1(fail1, efx_rc_t, rc); 713 714 return (rc); 715 } 716 #endif /* EFSYS_OPT_RX_SCALE */ 717 718 void 719 efx_rx_qpost( 720 __in efx_rxq_t *erp, 721 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 722 __in size_t size, 723 __in unsigned int ndescs, 724 __in unsigned int completed, 725 __in unsigned int added) 726 { 727 efx_nic_t *enp = erp->er_enp; 728 const efx_rx_ops_t *erxop = enp->en_erxop; 729 730 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 731 EFSYS_ASSERT(erp->er_buf_size == 0 || size == erp->er_buf_size); 732 733 erxop->erxo_qpost(erp, addrp, size, ndescs, completed, added); 734 } 735 736 #if EFSYS_OPT_RX_PACKED_STREAM 737 738 void 739 efx_rx_qpush_ps_credits( 740 __in efx_rxq_t *erp) 741 { 742 efx_nic_t *enp = erp->er_enp; 743 const efx_rx_ops_t *erxop = enp->en_erxop; 744 745 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 746 747 erxop->erxo_qpush_ps_credits(erp); 748 } 749 750 __checkReturn uint8_t * 751 efx_rx_qps_packet_info( 752 __in efx_rxq_t *erp, 753 __in uint8_t *buffer, 754 __in uint32_t buffer_length, 755 __in uint32_t current_offset, 756 __out uint16_t *lengthp, 757 __out uint32_t *next_offsetp, 758 __out uint32_t *timestamp) 759 { 760 efx_nic_t *enp = erp->er_enp; 761 const efx_rx_ops_t *erxop = enp->en_erxop; 762 763 return (erxop->erxo_qps_packet_info(erp, buffer, 764 buffer_length, current_offset, lengthp, 765 next_offsetp, timestamp)); 766 } 767 768 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 769 770 void 771 efx_rx_qpush( 772 __in efx_rxq_t *erp, 773 __in unsigned int added, 774 __inout unsigned int *pushedp) 775 { 776 efx_nic_t *enp = erp->er_enp; 777 const efx_rx_ops_t *erxop = enp->en_erxop; 778 779 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 780 781 erxop->erxo_qpush(erp, added, pushedp); 782 } 783 784 __checkReturn efx_rc_t 785 efx_rx_qflush( 786 __in efx_rxq_t *erp) 787 { 788 efx_nic_t *enp = erp->er_enp; 789 const efx_rx_ops_t *erxop = enp->en_erxop; 790 efx_rc_t rc; 791 792 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 793 794 if ((rc = erxop->erxo_qflush(erp)) != 0) 795 goto fail1; 796 797 return (0); 798 799 fail1: 800 EFSYS_PROBE1(fail1, efx_rc_t, rc); 801 802 return (rc); 803 } 804 805 __checkReturn size_t 806 efx_rxq_size( 807 __in const efx_nic_t *enp, 808 __in unsigned int ndescs) 809 { 810 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 811 812 return (ndescs * encp->enc_rx_desc_size); 813 } 814 815 __checkReturn unsigned int 816 efx_rxq_nbufs( 817 __in const efx_nic_t *enp, 818 __in unsigned int ndescs) 819 { 820 return (EFX_DIV_ROUND_UP(efx_rxq_size(enp, ndescs), EFX_BUF_SIZE)); 821 } 822 823 void 824 efx_rx_qenable( 825 __in efx_rxq_t *erp) 826 { 827 efx_nic_t *enp = erp->er_enp; 828 const efx_rx_ops_t *erxop = enp->en_erxop; 829 830 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 831 832 erxop->erxo_qenable(erp); 833 } 834 835 static __checkReturn efx_rc_t 836 efx_rx_qcreate_internal( 837 __in efx_nic_t *enp, 838 __in unsigned int index, 839 __in unsigned int label, 840 __in efx_rxq_type_t type, 841 __in_opt const efx_rxq_type_data_t *type_data, 842 __in efsys_mem_t *esmp, 843 __in size_t ndescs, 844 __in uint32_t id, 845 __in unsigned int flags, 846 __in efx_evq_t *eep, 847 __deref_out efx_rxq_t **erpp) 848 { 849 const efx_rx_ops_t *erxop = enp->en_erxop; 850 efx_rxq_t *erp; 851 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 852 efx_rc_t rc; 853 854 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 855 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 856 857 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 858 859 EFSYS_ASSERT(ISP2(encp->enc_rxq_max_ndescs)); 860 EFSYS_ASSERT(ISP2(encp->enc_rxq_min_ndescs)); 861 862 if (index >= encp->enc_rxq_limit) { 863 rc = EINVAL; 864 goto fail1; 865 } 866 867 if (!ISP2(ndescs) || 868 ndescs < encp->enc_rxq_min_ndescs || 869 ndescs > encp->enc_rxq_max_ndescs) { 870 rc = EINVAL; 871 goto fail2; 872 } 873 874 /* Allocate an RXQ object */ 875 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 876 877 if (erp == NULL) { 878 rc = ENOMEM; 879 goto fail3; 880 } 881 882 erp->er_magic = EFX_RXQ_MAGIC; 883 erp->er_enp = enp; 884 erp->er_index = index; 885 erp->er_mask = ndescs - 1; 886 erp->er_esmp = esmp; 887 888 if ((rc = erxop->erxo_qcreate(enp, index, label, type, type_data, esmp, 889 ndescs, id, flags, eep, erp)) != 0) 890 goto fail4; 891 892 enp->en_rx_qcount++; 893 *erpp = erp; 894 895 return (0); 896 897 fail4: 898 EFSYS_PROBE(fail4); 899 900 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 901 fail3: 902 EFSYS_PROBE(fail3); 903 fail2: 904 EFSYS_PROBE(fail2); 905 fail1: 906 EFSYS_PROBE1(fail1, efx_rc_t, rc); 907 908 return (rc); 909 } 910 911 __checkReturn efx_rc_t 912 efx_rx_qcreate( 913 __in efx_nic_t *enp, 914 __in unsigned int index, 915 __in unsigned int label, 916 __in efx_rxq_type_t type, 917 __in size_t buf_size, 918 __in efsys_mem_t *esmp, 919 __in size_t ndescs, 920 __in uint32_t id, 921 __in unsigned int flags, 922 __in efx_evq_t *eep, 923 __deref_out efx_rxq_t **erpp) 924 { 925 efx_rxq_type_data_t type_data; 926 927 memset(&type_data, 0, sizeof (type_data)); 928 929 type_data.ertd_default.ed_buf_size = buf_size; 930 931 return efx_rx_qcreate_internal(enp, index, label, type, &type_data, 932 esmp, ndescs, id, flags, eep, erpp); 933 } 934 935 #if EFSYS_OPT_RX_PACKED_STREAM 936 937 __checkReturn efx_rc_t 938 efx_rx_qcreate_packed_stream( 939 __in efx_nic_t *enp, 940 __in unsigned int index, 941 __in unsigned int label, 942 __in uint32_t ps_buf_size, 943 __in efsys_mem_t *esmp, 944 __in size_t ndescs, 945 __in efx_evq_t *eep, 946 __deref_out efx_rxq_t **erpp) 947 { 948 efx_rxq_type_data_t type_data; 949 950 memset(&type_data, 0, sizeof (type_data)); 951 952 type_data.ertd_packed_stream.eps_buf_size = ps_buf_size; 953 954 return efx_rx_qcreate_internal(enp, index, label, 955 EFX_RXQ_TYPE_PACKED_STREAM, &type_data, esmp, ndescs, 956 0 /* id unused on EF10 */, EFX_RXQ_FLAG_NONE, eep, erpp); 957 } 958 959 #endif 960 961 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 962 963 __checkReturn efx_rc_t 964 efx_rx_qcreate_es_super_buffer( 965 __in efx_nic_t *enp, 966 __in unsigned int index, 967 __in unsigned int label, 968 __in uint32_t n_bufs_per_desc, 969 __in uint32_t max_dma_len, 970 __in uint32_t buf_stride, 971 __in uint32_t hol_block_timeout, 972 __in efsys_mem_t *esmp, 973 __in size_t ndescs, 974 __in unsigned int flags, 975 __in efx_evq_t *eep, 976 __deref_out efx_rxq_t **erpp) 977 { 978 efx_rc_t rc; 979 efx_rxq_type_data_t type_data; 980 981 if (hol_block_timeout > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) { 982 rc = EINVAL; 983 goto fail1; 984 } 985 986 memset(&type_data, 0, sizeof (type_data)); 987 988 type_data.ertd_es_super_buffer.eessb_bufs_per_desc = n_bufs_per_desc; 989 type_data.ertd_es_super_buffer.eessb_max_dma_len = max_dma_len; 990 type_data.ertd_es_super_buffer.eessb_buf_stride = buf_stride; 991 type_data.ertd_es_super_buffer.eessb_hol_block_timeout = 992 hol_block_timeout; 993 994 rc = efx_rx_qcreate_internal(enp, index, label, 995 EFX_RXQ_TYPE_ES_SUPER_BUFFER, &type_data, esmp, ndescs, 996 0 /* id unused on EF10 */, flags, eep, erpp); 997 if (rc != 0) 998 goto fail2; 999 1000 return (0); 1001 1002 fail2: 1003 EFSYS_PROBE(fail2); 1004 fail1: 1005 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1006 1007 return (rc); 1008 } 1009 1010 #endif 1011 1012 1013 void 1014 efx_rx_qdestroy( 1015 __in efx_rxq_t *erp) 1016 { 1017 efx_nic_t *enp = erp->er_enp; 1018 const efx_rx_ops_t *erxop = enp->en_erxop; 1019 1020 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1021 1022 EFSYS_ASSERT(enp->en_rx_qcount != 0); 1023 --enp->en_rx_qcount; 1024 1025 erxop->erxo_qdestroy(erp); 1026 1027 /* Free the RXQ object */ 1028 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1029 } 1030 1031 __checkReturn efx_rc_t 1032 efx_pseudo_hdr_pkt_length_get( 1033 __in efx_rxq_t *erp, 1034 __in uint8_t *buffer, 1035 __out uint16_t *lengthp) 1036 { 1037 efx_nic_t *enp = erp->er_enp; 1038 const efx_rx_ops_t *erxop = enp->en_erxop; 1039 1040 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1041 1042 return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp)); 1043 } 1044 1045 #if EFSYS_OPT_RX_SCALE 1046 __checkReturn uint32_t 1047 efx_pseudo_hdr_hash_get( 1048 __in efx_rxq_t *erp, 1049 __in efx_rx_hash_alg_t func, 1050 __in uint8_t *buffer) 1051 { 1052 efx_nic_t *enp = erp->er_enp; 1053 const efx_rx_ops_t *erxop = enp->en_erxop; 1054 1055 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1056 1057 EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE); 1058 return (erxop->erxo_prefix_hash(enp, func, buffer)); 1059 } 1060 #endif /* EFSYS_OPT_RX_SCALE */ 1061 1062 __checkReturn efx_rc_t 1063 efx_rx_prefix_get_layout( 1064 __in const efx_rxq_t *erp, 1065 __out efx_rx_prefix_layout_t *erplp) 1066 { 1067 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1068 1069 *erplp = erp->er_prefix_layout; 1070 1071 return (0); 1072 } 1073 1074 #if EFSYS_OPT_SIENA 1075 1076 static __checkReturn efx_rc_t 1077 siena_rx_init( 1078 __in efx_nic_t *enp) 1079 { 1080 efx_oword_t oword; 1081 unsigned int index; 1082 1083 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 1084 1085 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 1086 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 1087 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 1088 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 1089 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 1090 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 1091 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 1092 1093 /* Zero the RSS table */ 1094 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 1095 index++) { 1096 EFX_ZERO_OWORD(oword); 1097 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1098 index, &oword, B_TRUE); 1099 } 1100 1101 #if EFSYS_OPT_RX_SCALE 1102 /* The RSS key and indirection table are writable. */ 1103 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 1104 1105 /* Hardware can insert RX hash with/without RSS */ 1106 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 1107 #endif /* EFSYS_OPT_RX_SCALE */ 1108 1109 return (0); 1110 } 1111 1112 #if EFSYS_OPT_RX_SCATTER 1113 static __checkReturn efx_rc_t 1114 siena_rx_scatter_enable( 1115 __in efx_nic_t *enp, 1116 __in unsigned int buf_size) 1117 { 1118 unsigned int nbuf32; 1119 efx_oword_t oword; 1120 efx_rc_t rc; 1121 1122 nbuf32 = buf_size / 32; 1123 if ((nbuf32 == 0) || 1124 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 1125 ((buf_size % 32) != 0)) { 1126 rc = EINVAL; 1127 goto fail1; 1128 } 1129 1130 if (enp->en_rx_qcount > 0) { 1131 rc = EBUSY; 1132 goto fail2; 1133 } 1134 1135 /* Set scatter buffer size */ 1136 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 1137 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 1138 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 1139 1140 /* Enable scatter for packets not matching a filter */ 1141 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 1142 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 1143 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 1144 1145 return (0); 1146 1147 fail2: 1148 EFSYS_PROBE(fail2); 1149 fail1: 1150 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1151 1152 return (rc); 1153 } 1154 #endif /* EFSYS_OPT_RX_SCATTER */ 1155 1156 1157 #define EFX_RX_LFSR_HASH(_enp, _insert) \ 1158 do { \ 1159 efx_oword_t oword; \ 1160 \ 1161 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1162 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 1163 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 1164 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 1165 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 1166 (_insert) ? 1 : 0); \ 1167 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1168 \ 1169 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 1170 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 1171 &oword); \ 1172 EFX_SET_OWORD_FIELD(oword, \ 1173 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 1174 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 1175 &oword); \ 1176 } \ 1177 \ 1178 _NOTE(CONSTANTCONDITION) \ 1179 } while (B_FALSE) 1180 1181 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 1182 do { \ 1183 efx_oword_t oword; \ 1184 \ 1185 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1186 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 1187 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 1188 (_ip) ? 1 : 0); \ 1189 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 1190 (_tcp) ? 0 : 1); \ 1191 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 1192 (_insert) ? 1 : 0); \ 1193 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1194 \ 1195 _NOTE(CONSTANTCONDITION) \ 1196 } while (B_FALSE) 1197 1198 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 1199 do { \ 1200 efx_oword_t oword; \ 1201 \ 1202 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 1203 EFX_SET_OWORD_FIELD(oword, \ 1204 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 1205 EFX_SET_OWORD_FIELD(oword, \ 1206 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 1207 EFX_SET_OWORD_FIELD(oword, \ 1208 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 1209 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 1210 \ 1211 (_rc) = 0; \ 1212 \ 1213 _NOTE(CONSTANTCONDITION) \ 1214 } while (B_FALSE) 1215 1216 1217 #if EFSYS_OPT_RX_SCALE 1218 1219 static __checkReturn efx_rc_t 1220 siena_rx_scale_mode_set( 1221 __in efx_nic_t *enp, 1222 __in uint32_t rss_context, 1223 __in efx_rx_hash_alg_t alg, 1224 __in efx_rx_hash_type_t type, 1225 __in boolean_t insert) 1226 { 1227 efx_rc_t rc; 1228 1229 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1230 rc = EINVAL; 1231 goto fail1; 1232 } 1233 1234 switch (alg) { 1235 case EFX_RX_HASHALG_LFSR: 1236 EFX_RX_LFSR_HASH(enp, insert); 1237 break; 1238 1239 case EFX_RX_HASHALG_TOEPLITZ: 1240 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 1241 (type & EFX_RX_HASH_IPV4) ? B_TRUE : B_FALSE, 1242 (type & EFX_RX_HASH_TCPIPV4) ? B_TRUE : B_FALSE); 1243 1244 EFX_RX_TOEPLITZ_IPV6_HASH(enp, 1245 (type & EFX_RX_HASH_IPV6) ? B_TRUE : B_FALSE, 1246 (type & EFX_RX_HASH_TCPIPV6) ? B_TRUE : B_FALSE, 1247 rc); 1248 if (rc != 0) 1249 goto fail2; 1250 1251 break; 1252 1253 default: 1254 rc = EINVAL; 1255 goto fail3; 1256 } 1257 1258 return (0); 1259 1260 fail3: 1261 EFSYS_PROBE(fail3); 1262 fail2: 1263 EFSYS_PROBE(fail2); 1264 fail1: 1265 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1266 1267 EFX_RX_LFSR_HASH(enp, B_FALSE); 1268 1269 return (rc); 1270 } 1271 #endif 1272 1273 #if EFSYS_OPT_RX_SCALE 1274 static __checkReturn efx_rc_t 1275 siena_rx_scale_key_set( 1276 __in efx_nic_t *enp, 1277 __in uint32_t rss_context, 1278 __in_ecount(n) uint8_t *key, 1279 __in size_t n) 1280 { 1281 efx_oword_t oword; 1282 unsigned int byte; 1283 unsigned int offset; 1284 efx_rc_t rc; 1285 1286 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1287 rc = EINVAL; 1288 goto fail1; 1289 } 1290 1291 byte = 0; 1292 1293 /* Write Toeplitz IPv4 hash key */ 1294 EFX_ZERO_OWORD(oword); 1295 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 1296 offset > 0 && byte < n; 1297 --offset) 1298 oword.eo_u8[offset - 1] = key[byte++]; 1299 1300 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 1301 1302 byte = 0; 1303 1304 /* Verify Toeplitz IPv4 hash key */ 1305 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 1306 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 1307 offset > 0 && byte < n; 1308 --offset) { 1309 if (oword.eo_u8[offset - 1] != key[byte++]) { 1310 rc = EFAULT; 1311 goto fail2; 1312 } 1313 } 1314 1315 if ((enp->en_features & EFX_FEATURE_IPV6) == 0) 1316 goto done; 1317 1318 byte = 0; 1319 1320 /* Write Toeplitz IPv6 hash key 3 */ 1321 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1322 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1323 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1324 offset > 0 && byte < n; 1325 --offset) 1326 oword.eo_u8[offset - 1] = key[byte++]; 1327 1328 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1329 1330 /* Write Toeplitz IPv6 hash key 2 */ 1331 EFX_ZERO_OWORD(oword); 1332 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1333 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1334 offset > 0 && byte < n; 1335 --offset) 1336 oword.eo_u8[offset - 1] = key[byte++]; 1337 1338 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1339 1340 /* Write Toeplitz IPv6 hash key 1 */ 1341 EFX_ZERO_OWORD(oword); 1342 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1343 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1344 offset > 0 && byte < n; 1345 --offset) 1346 oword.eo_u8[offset - 1] = key[byte++]; 1347 1348 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1349 1350 byte = 0; 1351 1352 /* Verify Toeplitz IPv6 hash key 3 */ 1353 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1354 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1355 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1356 offset > 0 && byte < n; 1357 --offset) { 1358 if (oword.eo_u8[offset - 1] != key[byte++]) { 1359 rc = EFAULT; 1360 goto fail3; 1361 } 1362 } 1363 1364 /* Verify Toeplitz IPv6 hash key 2 */ 1365 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1366 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1367 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1368 offset > 0 && byte < n; 1369 --offset) { 1370 if (oword.eo_u8[offset - 1] != key[byte++]) { 1371 rc = EFAULT; 1372 goto fail4; 1373 } 1374 } 1375 1376 /* Verify Toeplitz IPv6 hash key 1 */ 1377 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1378 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1379 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1380 offset > 0 && byte < n; 1381 --offset) { 1382 if (oword.eo_u8[offset - 1] != key[byte++]) { 1383 rc = EFAULT; 1384 goto fail5; 1385 } 1386 } 1387 1388 done: 1389 return (0); 1390 1391 fail5: 1392 EFSYS_PROBE(fail5); 1393 fail4: 1394 EFSYS_PROBE(fail4); 1395 fail3: 1396 EFSYS_PROBE(fail3); 1397 fail2: 1398 EFSYS_PROBE(fail2); 1399 fail1: 1400 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1401 1402 return (rc); 1403 } 1404 #endif 1405 1406 #if EFSYS_OPT_RX_SCALE 1407 static __checkReturn efx_rc_t 1408 siena_rx_scale_tbl_set( 1409 __in efx_nic_t *enp, 1410 __in uint32_t rss_context, 1411 __in_ecount(n) unsigned int *table, 1412 __in size_t n) 1413 { 1414 efx_oword_t oword; 1415 int index; 1416 efx_rc_t rc; 1417 1418 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 1419 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 1420 1421 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1422 rc = EINVAL; 1423 goto fail1; 1424 } 1425 1426 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 1427 rc = EINVAL; 1428 goto fail2; 1429 } 1430 1431 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 1432 uint32_t byte; 1433 1434 /* Calculate the entry to place in the table */ 1435 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1436 1437 EFSYS_PROBE2(table, int, index, uint32_t, byte); 1438 1439 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 1440 1441 /* Write the table */ 1442 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1443 index, &oword, B_TRUE); 1444 } 1445 1446 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 1447 uint32_t byte; 1448 1449 /* Determine if we're starting a new batch */ 1450 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1451 1452 /* Read the table */ 1453 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 1454 index, &oword, B_TRUE); 1455 1456 /* Verify the entry */ 1457 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 1458 rc = EFAULT; 1459 goto fail3; 1460 } 1461 } 1462 1463 return (0); 1464 1465 fail3: 1466 EFSYS_PROBE(fail3); 1467 fail2: 1468 EFSYS_PROBE(fail2); 1469 fail1: 1470 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1471 1472 return (rc); 1473 } 1474 #endif 1475 1476 /* 1477 * Falcon/Siena pseudo-header 1478 * -------------------------- 1479 * 1480 * Receive packets are prefixed by an optional 16 byte pseudo-header. 1481 * The pseudo-header is a byte array of one of the forms: 1482 * 1483 * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 1484 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT 1485 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL 1486 * 1487 * where: 1488 * TT.TT.TT.TT Toeplitz hash (32-bit big-endian) 1489 * LL.LL LFSR hash (16-bit big-endian) 1490 */ 1491 1492 /* 1493 * Provide Rx prefix layout with Toeplitz hash only since LSFR is 1494 * used by no supported drivers. 1495 * 1496 * Siena does not support Rx prefix choice via MC_CMD_GET_RX_PREFIX_ID 1497 * and query its layout using MC_CMD_QUERY_RX_PREFIX_ID. 1498 */ 1499 static const efx_rx_prefix_layout_t siena_toeplitz_rx_prefix_layout = { 1500 .erpl_id = 0, 1501 .erpl_length = 16, 1502 .erpl_fields = { 1503 [EFX_RX_PREFIX_FIELD_RSS_HASH] = { 12 * 8, 32, B_TRUE }, 1504 } 1505 }; 1506 1507 #if EFSYS_OPT_RX_SCALE 1508 static __checkReturn uint32_t 1509 siena_rx_prefix_hash( 1510 __in efx_nic_t *enp, 1511 __in efx_rx_hash_alg_t func, 1512 __in uint8_t *buffer) 1513 { 1514 _NOTE(ARGUNUSED(enp)) 1515 1516 switch (func) { 1517 case EFX_RX_HASHALG_TOEPLITZ: 1518 return ((buffer[12] << 24) | 1519 (buffer[13] << 16) | 1520 (buffer[14] << 8) | 1521 buffer[15]); 1522 1523 case EFX_RX_HASHALG_LFSR: 1524 return ((buffer[14] << 8) | buffer[15]); 1525 1526 default: 1527 EFSYS_ASSERT(0); 1528 return (0); 1529 } 1530 } 1531 #endif /* EFSYS_OPT_RX_SCALE */ 1532 1533 static __checkReturn efx_rc_t 1534 siena_rx_prefix_pktlen( 1535 __in efx_nic_t *enp, 1536 __in uint8_t *buffer, 1537 __out uint16_t *lengthp) 1538 { 1539 _NOTE(ARGUNUSED(enp, buffer, lengthp)) 1540 1541 /* Not supported by Falcon/Siena hardware */ 1542 EFSYS_ASSERT(0); 1543 return (ENOTSUP); 1544 } 1545 1546 1547 static void 1548 siena_rx_qpost( 1549 __in efx_rxq_t *erp, 1550 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 1551 __in size_t size, 1552 __in unsigned int ndescs, 1553 __in unsigned int completed, 1554 __in unsigned int added) 1555 { 1556 efx_qword_t qword; 1557 unsigned int i; 1558 unsigned int offset; 1559 unsigned int id; 1560 1561 /* The client driver must not overfill the queue */ 1562 EFSYS_ASSERT3U(added - completed + ndescs, <=, 1563 EFX_RXQ_LIMIT(erp->er_mask + 1)); 1564 1565 id = added & (erp->er_mask); 1566 for (i = 0; i < ndescs; i++) { 1567 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 1568 unsigned int, id, efsys_dma_addr_t, addrp[i], 1569 size_t, size); 1570 1571 EFX_POPULATE_QWORD_3(qword, 1572 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 1573 FSF_AZ_RX_KER_BUF_ADDR_DW0, 1574 (uint32_t)(addrp[i] & 0xffffffff), 1575 FSF_AZ_RX_KER_BUF_ADDR_DW1, 1576 (uint32_t)(addrp[i] >> 32)); 1577 1578 offset = id * sizeof (efx_qword_t); 1579 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 1580 1581 id = (id + 1) & (erp->er_mask); 1582 } 1583 } 1584 1585 static void 1586 siena_rx_qpush( 1587 __in efx_rxq_t *erp, 1588 __in unsigned int added, 1589 __inout unsigned int *pushedp) 1590 { 1591 efx_nic_t *enp = erp->er_enp; 1592 unsigned int pushed = *pushedp; 1593 uint32_t wptr; 1594 efx_oword_t oword; 1595 efx_dword_t dword; 1596 1597 /* All descriptors are pushed */ 1598 *pushedp = added; 1599 1600 /* Push the populated descriptors out */ 1601 wptr = added & erp->er_mask; 1602 1603 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 1604 1605 /* Only write the third DWORD */ 1606 EFX_POPULATE_DWORD_1(dword, 1607 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 1608 1609 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 1610 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 1611 SIENA_RXQ_DESC_SIZE, wptr, pushed & erp->er_mask); 1612 EFSYS_PIO_WRITE_BARRIER(); 1613 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 1614 erp->er_index, &dword, B_FALSE); 1615 } 1616 1617 #if EFSYS_OPT_RX_PACKED_STREAM 1618 static void 1619 siena_rx_qpush_ps_credits( 1620 __in efx_rxq_t *erp) 1621 { 1622 /* Not supported by Siena hardware */ 1623 EFSYS_ASSERT(0); 1624 } 1625 1626 static uint8_t * 1627 siena_rx_qps_packet_info( 1628 __in efx_rxq_t *erp, 1629 __in uint8_t *buffer, 1630 __in uint32_t buffer_length, 1631 __in uint32_t current_offset, 1632 __out uint16_t *lengthp, 1633 __out uint32_t *next_offsetp, 1634 __out uint32_t *timestamp) 1635 { 1636 /* Not supported by Siena hardware */ 1637 EFSYS_ASSERT(0); 1638 1639 return (NULL); 1640 } 1641 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1642 1643 static __checkReturn efx_rc_t 1644 siena_rx_qflush( 1645 __in efx_rxq_t *erp) 1646 { 1647 efx_nic_t *enp = erp->er_enp; 1648 efx_oword_t oword; 1649 uint32_t label; 1650 1651 label = erp->er_index; 1652 1653 /* Flush the queue */ 1654 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 1655 FRF_AZ_RX_FLUSH_DESCQ, label); 1656 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 1657 1658 return (0); 1659 } 1660 1661 static void 1662 siena_rx_qenable( 1663 __in efx_rxq_t *erp) 1664 { 1665 efx_nic_t *enp = erp->er_enp; 1666 efx_oword_t oword; 1667 1668 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1669 1670 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 1671 erp->er_index, &oword, B_TRUE); 1672 1673 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 1674 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 1675 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 1676 1677 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1678 erp->er_index, &oword, B_TRUE); 1679 } 1680 1681 static __checkReturn efx_rc_t 1682 siena_rx_qcreate( 1683 __in efx_nic_t *enp, 1684 __in unsigned int index, 1685 __in unsigned int label, 1686 __in efx_rxq_type_t type, 1687 __in_opt const efx_rxq_type_data_t *type_data, 1688 __in efsys_mem_t *esmp, 1689 __in size_t ndescs, 1690 __in uint32_t id, 1691 __in unsigned int flags, 1692 __in efx_evq_t *eep, 1693 __in efx_rxq_t *erp) 1694 { 1695 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1696 efx_oword_t oword; 1697 uint32_t size; 1698 boolean_t jumbo = B_FALSE; 1699 efx_rc_t rc; 1700 1701 _NOTE(ARGUNUSED(esmp)) 1702 1703 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == 1704 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 1705 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1706 1707 for (size = 0; 1708 (1U << size) <= encp->enc_rxq_max_ndescs / encp->enc_rxq_min_ndescs; 1709 size++) 1710 if ((1U << size) == (uint32_t)ndescs / encp->enc_rxq_min_ndescs) 1711 break; 1712 if (id + (1 << size) >= encp->enc_buftbl_limit) { 1713 rc = EINVAL; 1714 goto fail1; 1715 } 1716 1717 switch (type) { 1718 case EFX_RXQ_TYPE_DEFAULT: 1719 erp->er_buf_size = type_data->ertd_default.ed_buf_size; 1720 break; 1721 1722 default: 1723 rc = EINVAL; 1724 goto fail2; 1725 } 1726 1727 if (flags & EFX_RXQ_FLAG_SCATTER) { 1728 #if EFSYS_OPT_RX_SCATTER 1729 jumbo = B_TRUE; 1730 #else 1731 rc = EINVAL; 1732 goto fail3; 1733 #endif /* EFSYS_OPT_RX_SCATTER */ 1734 } 1735 1736 /* Set up the new descriptor queue */ 1737 EFX_POPULATE_OWORD_7(oword, 1738 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 1739 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 1740 FRF_AZ_RX_DESCQ_OWNER_ID, 0, 1741 FRF_AZ_RX_DESCQ_LABEL, label, 1742 FRF_AZ_RX_DESCQ_SIZE, size, 1743 FRF_AZ_RX_DESCQ_TYPE, 0, 1744 FRF_AZ_RX_DESCQ_JUMBO, jumbo); 1745 1746 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1747 erp->er_index, &oword, B_TRUE); 1748 1749 erp->er_prefix_layout = siena_toeplitz_rx_prefix_layout; 1750 1751 return (0); 1752 1753 #if !EFSYS_OPT_RX_SCATTER 1754 fail3: 1755 EFSYS_PROBE(fail3); 1756 #endif 1757 fail2: 1758 EFSYS_PROBE(fail2); 1759 fail1: 1760 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1761 1762 return (rc); 1763 } 1764 1765 static void 1766 siena_rx_qdestroy( 1767 __in efx_rxq_t *erp) 1768 { 1769 efx_nic_t *enp = erp->er_enp; 1770 efx_oword_t oword; 1771 1772 /* Purge descriptor queue */ 1773 EFX_ZERO_OWORD(oword); 1774 1775 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1776 erp->er_index, &oword, B_TRUE); 1777 } 1778 1779 static void 1780 siena_rx_fini( 1781 __in efx_nic_t *enp) 1782 { 1783 _NOTE(ARGUNUSED(enp)) 1784 } 1785 1786 #endif /* EFSYS_OPT_SIENA */ 1787