1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2020 Xilinx, Inc. 4 * Copyright(c) 2007-2019 Solarflare Communications Inc. 5 */ 6 7 #include "efx.h" 8 #include "efx_impl.h" 9 10 11 #if EFSYS_OPT_SIENA 12 13 static __checkReturn efx_rc_t 14 siena_rx_init( 15 __in efx_nic_t *enp); 16 17 static void 18 siena_rx_fini( 19 __in efx_nic_t *enp); 20 21 #if EFSYS_OPT_RX_SCATTER 22 static __checkReturn efx_rc_t 23 siena_rx_scatter_enable( 24 __in efx_nic_t *enp, 25 __in unsigned int buf_size); 26 #endif /* EFSYS_OPT_RX_SCATTER */ 27 28 #if EFSYS_OPT_RX_SCALE 29 static __checkReturn efx_rc_t 30 siena_rx_scale_mode_set( 31 __in efx_nic_t *enp, 32 __in uint32_t rss_context, 33 __in efx_rx_hash_alg_t alg, 34 __in efx_rx_hash_type_t type, 35 __in boolean_t insert); 36 37 static __checkReturn efx_rc_t 38 siena_rx_scale_key_set( 39 __in efx_nic_t *enp, 40 __in uint32_t rss_context, 41 __in_ecount(n) uint8_t *key, 42 __in size_t n); 43 44 static __checkReturn efx_rc_t 45 siena_rx_scale_tbl_set( 46 __in efx_nic_t *enp, 47 __in uint32_t rss_context, 48 __in_ecount(n) unsigned int *table, 49 __in size_t n); 50 51 static __checkReturn uint32_t 52 siena_rx_prefix_hash( 53 __in efx_nic_t *enp, 54 __in efx_rx_hash_alg_t func, 55 __in uint8_t *buffer); 56 57 #endif /* EFSYS_OPT_RX_SCALE */ 58 59 static __checkReturn efx_rc_t 60 siena_rx_prefix_pktlen( 61 __in efx_nic_t *enp, 62 __in uint8_t *buffer, 63 __out uint16_t *lengthp); 64 65 static void 66 siena_rx_qpost( 67 __in efx_rxq_t *erp, 68 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 69 __in size_t size, 70 __in unsigned int ndescs, 71 __in unsigned int completed, 72 __in unsigned int added); 73 74 static void 75 siena_rx_qpush( 76 __in efx_rxq_t *erp, 77 __in unsigned int added, 78 __inout unsigned int *pushedp); 79 80 #if EFSYS_OPT_RX_PACKED_STREAM 81 static void 82 siena_rx_qpush_ps_credits( 83 __in efx_rxq_t *erp); 84 85 static __checkReturn uint8_t * 86 siena_rx_qps_packet_info( 87 __in efx_rxq_t *erp, 88 __in uint8_t *buffer, 89 __in uint32_t buffer_length, 90 __in uint32_t current_offset, 91 __out uint16_t *lengthp, 92 __out uint32_t *next_offsetp, 93 __out uint32_t *timestamp); 94 #endif 95 96 static __checkReturn efx_rc_t 97 siena_rx_qflush( 98 __in efx_rxq_t *erp); 99 100 static void 101 siena_rx_qenable( 102 __in efx_rxq_t *erp); 103 104 static __checkReturn efx_rc_t 105 siena_rx_qcreate( 106 __in efx_nic_t *enp, 107 __in unsigned int index, 108 __in unsigned int label, 109 __in efx_rxq_type_t type, 110 __in_opt const efx_rxq_type_data_t *type_data, 111 __in efsys_mem_t *esmp, 112 __in size_t ndescs, 113 __in uint32_t id, 114 __in unsigned int flags, 115 __in efx_evq_t *eep, 116 __in efx_rxq_t *erp); 117 118 static void 119 siena_rx_qdestroy( 120 __in efx_rxq_t *erp); 121 122 #endif /* EFSYS_OPT_SIENA */ 123 124 125 #if EFSYS_OPT_SIENA 126 static const efx_rx_ops_t __efx_rx_siena_ops = { 127 siena_rx_init, /* erxo_init */ 128 siena_rx_fini, /* erxo_fini */ 129 #if EFSYS_OPT_RX_SCATTER 130 siena_rx_scatter_enable, /* erxo_scatter_enable */ 131 #endif 132 #if EFSYS_OPT_RX_SCALE 133 NULL, /* erxo_scale_context_alloc */ 134 NULL, /* erxo_scale_context_free */ 135 siena_rx_scale_mode_set, /* erxo_scale_mode_set */ 136 siena_rx_scale_key_set, /* erxo_scale_key_set */ 137 siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 138 siena_rx_prefix_hash, /* erxo_prefix_hash */ 139 #endif 140 siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 141 siena_rx_qpost, /* erxo_qpost */ 142 siena_rx_qpush, /* erxo_qpush */ 143 #if EFSYS_OPT_RX_PACKED_STREAM 144 siena_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 145 siena_rx_qps_packet_info, /* erxo_qps_packet_info */ 146 #endif 147 siena_rx_qflush, /* erxo_qflush */ 148 siena_rx_qenable, /* erxo_qenable */ 149 siena_rx_qcreate, /* erxo_qcreate */ 150 siena_rx_qdestroy, /* erxo_qdestroy */ 151 }; 152 #endif /* EFSYS_OPT_SIENA */ 153 154 #if EFX_OPTS_EF10() 155 static const efx_rx_ops_t __efx_rx_ef10_ops = { 156 ef10_rx_init, /* erxo_init */ 157 ef10_rx_fini, /* erxo_fini */ 158 #if EFSYS_OPT_RX_SCATTER 159 ef10_rx_scatter_enable, /* erxo_scatter_enable */ 160 #endif 161 #if EFSYS_OPT_RX_SCALE 162 ef10_rx_scale_context_alloc, /* erxo_scale_context_alloc */ 163 ef10_rx_scale_context_free, /* erxo_scale_context_free */ 164 ef10_rx_scale_mode_set, /* erxo_scale_mode_set */ 165 ef10_rx_scale_key_set, /* erxo_scale_key_set */ 166 ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 167 ef10_rx_prefix_hash, /* erxo_prefix_hash */ 168 #endif 169 ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 170 ef10_rx_qpost, /* erxo_qpost */ 171 ef10_rx_qpush, /* erxo_qpush */ 172 #if EFSYS_OPT_RX_PACKED_STREAM 173 ef10_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 174 ef10_rx_qps_packet_info, /* erxo_qps_packet_info */ 175 #endif 176 ef10_rx_qflush, /* erxo_qflush */ 177 ef10_rx_qenable, /* erxo_qenable */ 178 ef10_rx_qcreate, /* erxo_qcreate */ 179 ef10_rx_qdestroy, /* erxo_qdestroy */ 180 }; 181 #endif /* EFX_OPTS_EF10() */ 182 183 #if EFSYS_OPT_RIVERHEAD 184 static const efx_rx_ops_t __efx_rx_rhead_ops = { 185 rhead_rx_init, /* erxo_init */ 186 rhead_rx_fini, /* erxo_fini */ 187 #if EFSYS_OPT_RX_SCATTER 188 rhead_rx_scatter_enable, /* erxo_scatter_enable */ 189 #endif 190 #if EFSYS_OPT_RX_SCALE 191 rhead_rx_scale_context_alloc, /* erxo_scale_context_alloc */ 192 rhead_rx_scale_context_free, /* erxo_scale_context_free */ 193 rhead_rx_scale_mode_set, /* erxo_scale_mode_set */ 194 rhead_rx_scale_key_set, /* erxo_scale_key_set */ 195 rhead_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 196 rhead_rx_prefix_hash, /* erxo_prefix_hash */ 197 #endif 198 rhead_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 199 rhead_rx_qpost, /* erxo_qpost */ 200 rhead_rx_qpush, /* erxo_qpush */ 201 #if EFSYS_OPT_RX_PACKED_STREAM 202 NULL, /* erxo_qpush_ps_credits */ 203 NULL, /* erxo_qps_packet_info */ 204 #endif 205 rhead_rx_qflush, /* erxo_qflush */ 206 rhead_rx_qenable, /* erxo_qenable */ 207 rhead_rx_qcreate, /* erxo_qcreate */ 208 rhead_rx_qdestroy, /* erxo_qdestroy */ 209 }; 210 #endif /* EFSYS_OPT_RIVERHEAD */ 211 212 213 __checkReturn efx_rc_t 214 efx_rx_init( 215 __inout efx_nic_t *enp) 216 { 217 const efx_rx_ops_t *erxop; 218 efx_rc_t rc; 219 220 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 221 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 222 223 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 224 rc = EINVAL; 225 goto fail1; 226 } 227 228 if (enp->en_mod_flags & EFX_MOD_RX) { 229 rc = EINVAL; 230 goto fail2; 231 } 232 233 switch (enp->en_family) { 234 #if EFSYS_OPT_SIENA 235 case EFX_FAMILY_SIENA: 236 erxop = &__efx_rx_siena_ops; 237 break; 238 #endif /* EFSYS_OPT_SIENA */ 239 240 #if EFSYS_OPT_HUNTINGTON 241 case EFX_FAMILY_HUNTINGTON: 242 erxop = &__efx_rx_ef10_ops; 243 break; 244 #endif /* EFSYS_OPT_HUNTINGTON */ 245 246 #if EFSYS_OPT_MEDFORD 247 case EFX_FAMILY_MEDFORD: 248 erxop = &__efx_rx_ef10_ops; 249 break; 250 #endif /* EFSYS_OPT_MEDFORD */ 251 252 #if EFSYS_OPT_MEDFORD2 253 case EFX_FAMILY_MEDFORD2: 254 erxop = &__efx_rx_ef10_ops; 255 break; 256 #endif /* EFSYS_OPT_MEDFORD2 */ 257 258 #if EFSYS_OPT_RIVERHEAD 259 case EFX_FAMILY_RIVERHEAD: 260 erxop = &__efx_rx_rhead_ops; 261 break; 262 #endif /* EFSYS_OPT_RIVERHEAD */ 263 264 default: 265 EFSYS_ASSERT(0); 266 rc = ENOTSUP; 267 goto fail3; 268 } 269 270 if ((rc = erxop->erxo_init(enp)) != 0) 271 goto fail4; 272 273 enp->en_erxop = erxop; 274 enp->en_mod_flags |= EFX_MOD_RX; 275 return (0); 276 277 fail4: 278 EFSYS_PROBE(fail4); 279 fail3: 280 EFSYS_PROBE(fail3); 281 fail2: 282 EFSYS_PROBE(fail2); 283 fail1: 284 EFSYS_PROBE1(fail1, efx_rc_t, rc); 285 286 enp->en_erxop = NULL; 287 enp->en_mod_flags &= ~EFX_MOD_RX; 288 return (rc); 289 } 290 291 void 292 efx_rx_fini( 293 __in efx_nic_t *enp) 294 { 295 const efx_rx_ops_t *erxop = enp->en_erxop; 296 297 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 298 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 299 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 300 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 301 302 erxop->erxo_fini(enp); 303 304 enp->en_erxop = NULL; 305 enp->en_mod_flags &= ~EFX_MOD_RX; 306 } 307 308 #if EFSYS_OPT_RX_SCATTER 309 __checkReturn efx_rc_t 310 efx_rx_scatter_enable( 311 __in efx_nic_t *enp, 312 __in unsigned int buf_size) 313 { 314 const efx_rx_ops_t *erxop = enp->en_erxop; 315 efx_rc_t rc; 316 317 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 318 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 319 320 if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0) 321 goto fail1; 322 323 return (0); 324 325 fail1: 326 EFSYS_PROBE1(fail1, efx_rc_t, rc); 327 return (rc); 328 } 329 #endif /* EFSYS_OPT_RX_SCATTER */ 330 331 #if EFSYS_OPT_RX_SCALE 332 __checkReturn efx_rc_t 333 efx_rx_scale_hash_flags_get( 334 __in efx_nic_t *enp, 335 __in efx_rx_hash_alg_t hash_alg, 336 __out_ecount_part(max_nflags, *nflagsp) unsigned int *flagsp, 337 __in unsigned int max_nflags, 338 __out unsigned int *nflagsp) 339 { 340 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 341 unsigned int nflags = 0; 342 efx_rc_t rc; 343 344 if (flagsp == NULL || nflagsp == NULL) { 345 rc = EINVAL; 346 goto fail1; 347 } 348 349 if ((encp->enc_rx_scale_hash_alg_mask & (1U << hash_alg)) == 0) { 350 nflags = 0; 351 goto done; 352 } 353 354 /* Helper to add flags word to flags array without buffer overflow */ 355 #define INSERT_FLAGS(_flags) \ 356 do { \ 357 if (nflags >= max_nflags) { \ 358 rc = E2BIG; \ 359 goto fail2; \ 360 } \ 361 *(flagsp + nflags) = (_flags); \ 362 nflags++; \ 363 \ 364 _NOTE(CONSTANTCONDITION) \ 365 } while (B_FALSE) 366 367 if (encp->enc_rx_scale_l4_hash_supported != B_FALSE) { 368 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 4TUPLE)); 369 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 4TUPLE)); 370 } 371 372 if ((encp->enc_rx_scale_l4_hash_supported != B_FALSE) && 373 (encp->enc_rx_scale_additional_modes_supported != B_FALSE)) { 374 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_DST)); 375 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_SRC)); 376 377 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_DST)); 378 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_SRC)); 379 380 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 4TUPLE)); 381 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_DST)); 382 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_SRC)); 383 384 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 4TUPLE)); 385 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_DST)); 386 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_SRC)); 387 } 388 389 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE)); 390 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE)); 391 392 INSERT_FLAGS(EFX_RX_HASH(IPV4, 2TUPLE)); 393 INSERT_FLAGS(EFX_RX_HASH(IPV6, 2TUPLE)); 394 395 if (encp->enc_rx_scale_additional_modes_supported != B_FALSE) { 396 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_DST)); 397 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_SRC)); 398 399 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_DST)); 400 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_SRC)); 401 402 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE)); 403 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_DST)); 404 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_SRC)); 405 406 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE)); 407 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_DST)); 408 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_SRC)); 409 410 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_DST)); 411 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_SRC)); 412 413 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_DST)); 414 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_SRC)); 415 } 416 417 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, DISABLE)); 418 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, DISABLE)); 419 420 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, DISABLE)); 421 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, DISABLE)); 422 423 INSERT_FLAGS(EFX_RX_HASH(IPV4, DISABLE)); 424 INSERT_FLAGS(EFX_RX_HASH(IPV6, DISABLE)); 425 426 #undef INSERT_FLAGS 427 428 done: 429 *nflagsp = nflags; 430 return (0); 431 432 fail2: 433 EFSYS_PROBE(fail2); 434 fail1: 435 EFSYS_PROBE1(fail1, efx_rc_t, rc); 436 437 return (rc); 438 } 439 440 __checkReturn efx_rc_t 441 efx_rx_hash_default_support_get( 442 __in efx_nic_t *enp, 443 __out efx_rx_hash_support_t *supportp) 444 { 445 efx_rc_t rc; 446 447 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 448 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 449 450 if (supportp == NULL) { 451 rc = EINVAL; 452 goto fail1; 453 } 454 455 /* 456 * Report the hashing support the client gets by default if it 457 * does not allocate an RSS context itself. 458 */ 459 *supportp = enp->en_hash_support; 460 461 return (0); 462 463 fail1: 464 EFSYS_PROBE1(fail1, efx_rc_t, rc); 465 466 return (rc); 467 } 468 469 __checkReturn efx_rc_t 470 efx_rx_scale_default_support_get( 471 __in efx_nic_t *enp, 472 __out efx_rx_scale_context_type_t *typep) 473 { 474 efx_rc_t rc; 475 476 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 477 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 478 479 if (typep == NULL) { 480 rc = EINVAL; 481 goto fail1; 482 } 483 484 /* 485 * Report the RSS support the client gets by default if it 486 * does not allocate an RSS context itself. 487 */ 488 *typep = enp->en_rss_context_type; 489 490 return (0); 491 492 fail1: 493 EFSYS_PROBE1(fail1, efx_rc_t, rc); 494 495 return (rc); 496 } 497 #endif /* EFSYS_OPT_RX_SCALE */ 498 499 #if EFSYS_OPT_RX_SCALE 500 __checkReturn efx_rc_t 501 efx_rx_scale_context_alloc( 502 __in efx_nic_t *enp, 503 __in efx_rx_scale_context_type_t type, 504 __in uint32_t num_queues, 505 __out uint32_t *rss_contextp) 506 { 507 const efx_rx_ops_t *erxop = enp->en_erxop; 508 efx_rc_t rc; 509 510 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 511 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 512 513 if (erxop->erxo_scale_context_alloc == NULL) { 514 rc = ENOTSUP; 515 goto fail1; 516 } 517 if ((rc = erxop->erxo_scale_context_alloc(enp, type, 518 num_queues, rss_contextp)) != 0) { 519 goto fail2; 520 } 521 522 return (0); 523 524 fail2: 525 EFSYS_PROBE(fail2); 526 fail1: 527 EFSYS_PROBE1(fail1, efx_rc_t, rc); 528 return (rc); 529 } 530 #endif /* EFSYS_OPT_RX_SCALE */ 531 532 #if EFSYS_OPT_RX_SCALE 533 __checkReturn efx_rc_t 534 efx_rx_scale_context_free( 535 __in efx_nic_t *enp, 536 __in uint32_t rss_context) 537 { 538 const efx_rx_ops_t *erxop = enp->en_erxop; 539 efx_rc_t rc; 540 541 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 542 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 543 544 if (erxop->erxo_scale_context_free == NULL) { 545 rc = ENOTSUP; 546 goto fail1; 547 } 548 if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0) 549 goto fail2; 550 551 return (0); 552 553 fail2: 554 EFSYS_PROBE(fail2); 555 fail1: 556 EFSYS_PROBE1(fail1, efx_rc_t, rc); 557 return (rc); 558 } 559 #endif /* EFSYS_OPT_RX_SCALE */ 560 561 #if EFSYS_OPT_RX_SCALE 562 __checkReturn efx_rc_t 563 efx_rx_scale_mode_set( 564 __in efx_nic_t *enp, 565 __in uint32_t rss_context, 566 __in efx_rx_hash_alg_t alg, 567 __in efx_rx_hash_type_t type, 568 __in boolean_t insert) 569 { 570 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 571 const efx_rx_ops_t *erxop = enp->en_erxop; 572 efx_rx_hash_type_t type_check; 573 unsigned int i; 574 efx_rc_t rc; 575 576 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 577 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 578 579 /* 580 * Legacy flags and modern bits cannot be 581 * used at the same time in the hash type. 582 */ 583 if ((type & EFX_RX_HASH_LEGACY_MASK) && 584 (type & ~EFX_RX_HASH_LEGACY_MASK)) { 585 rc = EINVAL; 586 goto fail1; 587 } 588 589 /* 590 * If RSS hash type is represented by additional bits 591 * in the value, the latter need to be verified since 592 * not all bit combinations are valid RSS modes. Also, 593 * depending on the firmware, some valid combinations 594 * may be unsupported. Discern additional bits in the 595 * type value and try to recognise valid combinations. 596 * If some bits remain unrecognised, report the error. 597 */ 598 type_check = type & ~EFX_RX_HASH_LEGACY_MASK; 599 if (type_check != 0) { 600 unsigned int type_flags[EFX_RX_HASH_NFLAGS]; 601 unsigned int type_nflags; 602 603 rc = efx_rx_scale_hash_flags_get(enp, alg, type_flags, 604 EFX_ARRAY_SIZE(type_flags), &type_nflags); 605 if (rc != 0) 606 goto fail2; 607 608 for (i = 0; i < type_nflags; ++i) { 609 if ((type_check & type_flags[i]) == type_flags[i]) 610 type_check &= ~(type_flags[i]); 611 } 612 613 if (type_check != 0) { 614 rc = EINVAL; 615 goto fail3; 616 } 617 } 618 619 /* 620 * Translate EFX_RX_HASH() flags to their legacy counterparts 621 * provided that the FW claims no support for additional modes. 622 */ 623 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE) { 624 efx_rx_hash_type_t t_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE) | 625 EFX_RX_HASH(IPV4_TCP, 2TUPLE); 626 efx_rx_hash_type_t t_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE) | 627 EFX_RX_HASH(IPV6_TCP, 2TUPLE); 628 efx_rx_hash_type_t t_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE); 629 efx_rx_hash_type_t t_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE); 630 631 if ((type & t_ipv4) == t_ipv4) 632 type |= EFX_RX_HASH_IPV4; 633 if ((type & t_ipv6) == t_ipv6) 634 type |= EFX_RX_HASH_IPV6; 635 636 if (encp->enc_rx_scale_l4_hash_supported == B_TRUE) { 637 if ((type & t_ipv4_tcp) == t_ipv4_tcp) 638 type |= EFX_RX_HASH_TCPIPV4; 639 if ((type & t_ipv6_tcp) == t_ipv6_tcp) 640 type |= EFX_RX_HASH_TCPIPV6; 641 } 642 643 type &= EFX_RX_HASH_LEGACY_MASK; 644 } 645 646 if (erxop->erxo_scale_mode_set != NULL) { 647 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg, 648 type, insert)) != 0) 649 goto fail4; 650 } 651 652 return (0); 653 654 fail4: 655 EFSYS_PROBE(fail4); 656 fail3: 657 EFSYS_PROBE(fail3); 658 fail2: 659 EFSYS_PROBE(fail2); 660 fail1: 661 EFSYS_PROBE1(fail1, efx_rc_t, rc); 662 return (rc); 663 } 664 #endif /* EFSYS_OPT_RX_SCALE */ 665 666 #if EFSYS_OPT_RX_SCALE 667 __checkReturn efx_rc_t 668 efx_rx_scale_key_set( 669 __in efx_nic_t *enp, 670 __in uint32_t rss_context, 671 __in_ecount(n) uint8_t *key, 672 __in size_t n) 673 { 674 const efx_rx_ops_t *erxop = enp->en_erxop; 675 efx_rc_t rc; 676 677 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 678 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 679 680 if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0) 681 goto fail1; 682 683 return (0); 684 685 fail1: 686 EFSYS_PROBE1(fail1, efx_rc_t, rc); 687 688 return (rc); 689 } 690 #endif /* EFSYS_OPT_RX_SCALE */ 691 692 #if EFSYS_OPT_RX_SCALE 693 __checkReturn efx_rc_t 694 efx_rx_scale_tbl_set( 695 __in efx_nic_t *enp, 696 __in uint32_t rss_context, 697 __in_ecount(n) unsigned int *table, 698 __in size_t n) 699 { 700 const efx_rx_ops_t *erxop = enp->en_erxop; 701 efx_rc_t rc; 702 703 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 704 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 705 706 if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, n)) != 0) 707 goto fail1; 708 709 return (0); 710 711 fail1: 712 EFSYS_PROBE1(fail1, efx_rc_t, rc); 713 714 return (rc); 715 } 716 #endif /* EFSYS_OPT_RX_SCALE */ 717 718 void 719 efx_rx_qpost( 720 __in efx_rxq_t *erp, 721 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 722 __in size_t size, 723 __in unsigned int ndescs, 724 __in unsigned int completed, 725 __in unsigned int added) 726 { 727 efx_nic_t *enp = erp->er_enp; 728 const efx_rx_ops_t *erxop = enp->en_erxop; 729 730 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 731 EFSYS_ASSERT(erp->er_buf_size == 0 || size == erp->er_buf_size); 732 733 erxop->erxo_qpost(erp, addrp, size, ndescs, completed, added); 734 } 735 736 #if EFSYS_OPT_RX_PACKED_STREAM 737 738 void 739 efx_rx_qpush_ps_credits( 740 __in efx_rxq_t *erp) 741 { 742 efx_nic_t *enp = erp->er_enp; 743 const efx_rx_ops_t *erxop = enp->en_erxop; 744 745 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 746 747 erxop->erxo_qpush_ps_credits(erp); 748 } 749 750 __checkReturn uint8_t * 751 efx_rx_qps_packet_info( 752 __in efx_rxq_t *erp, 753 __in uint8_t *buffer, 754 __in uint32_t buffer_length, 755 __in uint32_t current_offset, 756 __out uint16_t *lengthp, 757 __out uint32_t *next_offsetp, 758 __out uint32_t *timestamp) 759 { 760 efx_nic_t *enp = erp->er_enp; 761 const efx_rx_ops_t *erxop = enp->en_erxop; 762 763 return (erxop->erxo_qps_packet_info(erp, buffer, 764 buffer_length, current_offset, lengthp, 765 next_offsetp, timestamp)); 766 } 767 768 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 769 770 void 771 efx_rx_qpush( 772 __in efx_rxq_t *erp, 773 __in unsigned int added, 774 __inout unsigned int *pushedp) 775 { 776 efx_nic_t *enp = erp->er_enp; 777 const efx_rx_ops_t *erxop = enp->en_erxop; 778 779 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 780 781 erxop->erxo_qpush(erp, added, pushedp); 782 } 783 784 __checkReturn efx_rc_t 785 efx_rx_qflush( 786 __in efx_rxq_t *erp) 787 { 788 efx_nic_t *enp = erp->er_enp; 789 const efx_rx_ops_t *erxop = enp->en_erxop; 790 efx_rc_t rc; 791 792 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 793 794 if ((rc = erxop->erxo_qflush(erp)) != 0) 795 goto fail1; 796 797 return (0); 798 799 fail1: 800 EFSYS_PROBE1(fail1, efx_rc_t, rc); 801 802 return (rc); 803 } 804 805 __checkReturn size_t 806 efx_rxq_size( 807 __in const efx_nic_t *enp, 808 __in unsigned int ndescs) 809 { 810 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 811 812 return (ndescs * encp->enc_rx_desc_size); 813 } 814 815 __checkReturn unsigned int 816 efx_rxq_nbufs( 817 __in const efx_nic_t *enp, 818 __in unsigned int ndescs) 819 { 820 return (EFX_DIV_ROUND_UP(efx_rxq_size(enp, ndescs), EFX_BUF_SIZE)); 821 } 822 823 void 824 efx_rx_qenable( 825 __in efx_rxq_t *erp) 826 { 827 efx_nic_t *enp = erp->er_enp; 828 const efx_rx_ops_t *erxop = enp->en_erxop; 829 830 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 831 832 erxop->erxo_qenable(erp); 833 } 834 835 static __checkReturn efx_rc_t 836 efx_rx_qcreate_internal( 837 __in efx_nic_t *enp, 838 __in unsigned int index, 839 __in unsigned int label, 840 __in efx_rxq_type_t type, 841 __in_opt const efx_rxq_type_data_t *type_data, 842 __in efsys_mem_t *esmp, 843 __in size_t ndescs, 844 __in uint32_t id, 845 __in unsigned int flags, 846 __in efx_evq_t *eep, 847 __deref_out efx_rxq_t **erpp) 848 { 849 const efx_rx_ops_t *erxop = enp->en_erxop; 850 efx_rxq_t *erp; 851 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 852 efx_rc_t rc; 853 854 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 855 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 856 857 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 858 859 EFSYS_ASSERT(ISP2(encp->enc_rxq_max_ndescs)); 860 EFSYS_ASSERT(ISP2(encp->enc_rxq_min_ndescs)); 861 862 if (index >= encp->enc_rxq_limit) { 863 rc = EINVAL; 864 goto fail1; 865 } 866 867 if (!ISP2(ndescs) || 868 ndescs < encp->enc_rxq_min_ndescs || 869 ndescs > encp->enc_rxq_max_ndescs) { 870 rc = EINVAL; 871 goto fail2; 872 } 873 874 /* Allocate an RXQ object */ 875 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 876 877 if (erp == NULL) { 878 rc = ENOMEM; 879 goto fail3; 880 } 881 882 erp->er_magic = EFX_RXQ_MAGIC; 883 erp->er_enp = enp; 884 erp->er_index = index; 885 erp->er_mask = ndescs - 1; 886 erp->er_esmp = esmp; 887 888 if ((rc = erxop->erxo_qcreate(enp, index, label, type, type_data, esmp, 889 ndescs, id, flags, eep, erp)) != 0) 890 goto fail4; 891 892 enp->en_rx_qcount++; 893 *erpp = erp; 894 895 return (0); 896 897 fail4: 898 EFSYS_PROBE(fail4); 899 900 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 901 fail3: 902 EFSYS_PROBE(fail3); 903 fail2: 904 EFSYS_PROBE(fail2); 905 fail1: 906 EFSYS_PROBE1(fail1, efx_rc_t, rc); 907 908 return (rc); 909 } 910 911 __checkReturn efx_rc_t 912 efx_rx_qcreate( 913 __in efx_nic_t *enp, 914 __in unsigned int index, 915 __in unsigned int label, 916 __in efx_rxq_type_t type, 917 __in size_t buf_size, 918 __in efsys_mem_t *esmp, 919 __in size_t ndescs, 920 __in uint32_t id, 921 __in unsigned int flags, 922 __in efx_evq_t *eep, 923 __deref_out efx_rxq_t **erpp) 924 { 925 efx_rxq_type_data_t type_data; 926 927 memset(&type_data, 0, sizeof (type_data)); 928 929 type_data.ertd_default.ed_buf_size = buf_size; 930 931 return efx_rx_qcreate_internal(enp, index, label, type, &type_data, 932 esmp, ndescs, id, flags, eep, erpp); 933 } 934 935 #if EFSYS_OPT_RX_PACKED_STREAM 936 937 __checkReturn efx_rc_t 938 efx_rx_qcreate_packed_stream( 939 __in efx_nic_t *enp, 940 __in unsigned int index, 941 __in unsigned int label, 942 __in uint32_t ps_buf_size, 943 __in efsys_mem_t *esmp, 944 __in size_t ndescs, 945 __in efx_evq_t *eep, 946 __deref_out efx_rxq_t **erpp) 947 { 948 efx_rxq_type_data_t type_data; 949 950 memset(&type_data, 0, sizeof (type_data)); 951 952 type_data.ertd_packed_stream.eps_buf_size = ps_buf_size; 953 954 return efx_rx_qcreate_internal(enp, index, label, 955 EFX_RXQ_TYPE_PACKED_STREAM, &type_data, esmp, ndescs, 956 0 /* id unused on EF10 */, EFX_RXQ_FLAG_NONE, eep, erpp); 957 } 958 959 #endif 960 961 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 962 963 __checkReturn efx_rc_t 964 efx_rx_qcreate_es_super_buffer( 965 __in efx_nic_t *enp, 966 __in unsigned int index, 967 __in unsigned int label, 968 __in uint32_t n_bufs_per_desc, 969 __in uint32_t max_dma_len, 970 __in uint32_t buf_stride, 971 __in uint32_t hol_block_timeout, 972 __in efsys_mem_t *esmp, 973 __in size_t ndescs, 974 __in unsigned int flags, 975 __in efx_evq_t *eep, 976 __deref_out efx_rxq_t **erpp) 977 { 978 efx_rc_t rc; 979 efx_rxq_type_data_t type_data; 980 981 if (hol_block_timeout > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) { 982 rc = EINVAL; 983 goto fail1; 984 } 985 986 memset(&type_data, 0, sizeof (type_data)); 987 988 type_data.ertd_es_super_buffer.eessb_bufs_per_desc = n_bufs_per_desc; 989 type_data.ertd_es_super_buffer.eessb_max_dma_len = max_dma_len; 990 type_data.ertd_es_super_buffer.eessb_buf_stride = buf_stride; 991 type_data.ertd_es_super_buffer.eessb_hol_block_timeout = 992 hol_block_timeout; 993 994 rc = efx_rx_qcreate_internal(enp, index, label, 995 EFX_RXQ_TYPE_ES_SUPER_BUFFER, &type_data, esmp, ndescs, 996 0 /* id unused on EF10 */, flags, eep, erpp); 997 if (rc != 0) 998 goto fail2; 999 1000 return (0); 1001 1002 fail2: 1003 EFSYS_PROBE(fail2); 1004 fail1: 1005 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1006 1007 return (rc); 1008 } 1009 1010 #endif 1011 1012 1013 void 1014 efx_rx_qdestroy( 1015 __in efx_rxq_t *erp) 1016 { 1017 efx_nic_t *enp = erp->er_enp; 1018 const efx_rx_ops_t *erxop = enp->en_erxop; 1019 1020 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1021 1022 EFSYS_ASSERT(enp->en_rx_qcount != 0); 1023 --enp->en_rx_qcount; 1024 1025 erxop->erxo_qdestroy(erp); 1026 1027 /* Free the RXQ object */ 1028 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 1029 } 1030 1031 __checkReturn efx_rc_t 1032 efx_pseudo_hdr_pkt_length_get( 1033 __in efx_rxq_t *erp, 1034 __in uint8_t *buffer, 1035 __out uint16_t *lengthp) 1036 { 1037 efx_nic_t *enp = erp->er_enp; 1038 const efx_rx_ops_t *erxop = enp->en_erxop; 1039 1040 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1041 1042 return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp)); 1043 } 1044 1045 #if EFSYS_OPT_RX_SCALE 1046 __checkReturn uint32_t 1047 efx_pseudo_hdr_hash_get( 1048 __in efx_rxq_t *erp, 1049 __in efx_rx_hash_alg_t func, 1050 __in uint8_t *buffer) 1051 { 1052 efx_nic_t *enp = erp->er_enp; 1053 const efx_rx_ops_t *erxop = enp->en_erxop; 1054 1055 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1056 1057 EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE); 1058 return (erxop->erxo_prefix_hash(enp, func, buffer)); 1059 } 1060 #endif /* EFSYS_OPT_RX_SCALE */ 1061 1062 #if EFSYS_OPT_SIENA 1063 1064 static __checkReturn efx_rc_t 1065 siena_rx_init( 1066 __in efx_nic_t *enp) 1067 { 1068 efx_oword_t oword; 1069 unsigned int index; 1070 1071 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 1072 1073 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 1074 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 1075 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 1076 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 1077 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 1078 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 1079 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 1080 1081 /* Zero the RSS table */ 1082 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 1083 index++) { 1084 EFX_ZERO_OWORD(oword); 1085 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1086 index, &oword, B_TRUE); 1087 } 1088 1089 #if EFSYS_OPT_RX_SCALE 1090 /* The RSS key and indirection table are writable. */ 1091 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 1092 1093 /* Hardware can insert RX hash with/without RSS */ 1094 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 1095 #endif /* EFSYS_OPT_RX_SCALE */ 1096 1097 return (0); 1098 } 1099 1100 #if EFSYS_OPT_RX_SCATTER 1101 static __checkReturn efx_rc_t 1102 siena_rx_scatter_enable( 1103 __in efx_nic_t *enp, 1104 __in unsigned int buf_size) 1105 { 1106 unsigned int nbuf32; 1107 efx_oword_t oword; 1108 efx_rc_t rc; 1109 1110 nbuf32 = buf_size / 32; 1111 if ((nbuf32 == 0) || 1112 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 1113 ((buf_size % 32) != 0)) { 1114 rc = EINVAL; 1115 goto fail1; 1116 } 1117 1118 if (enp->en_rx_qcount > 0) { 1119 rc = EBUSY; 1120 goto fail2; 1121 } 1122 1123 /* Set scatter buffer size */ 1124 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 1125 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 1126 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 1127 1128 /* Enable scatter for packets not matching a filter */ 1129 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 1130 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 1131 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 1132 1133 return (0); 1134 1135 fail2: 1136 EFSYS_PROBE(fail2); 1137 fail1: 1138 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1139 1140 return (rc); 1141 } 1142 #endif /* EFSYS_OPT_RX_SCATTER */ 1143 1144 1145 #define EFX_RX_LFSR_HASH(_enp, _insert) \ 1146 do { \ 1147 efx_oword_t oword; \ 1148 \ 1149 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1150 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 1151 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 1152 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 1153 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 1154 (_insert) ? 1 : 0); \ 1155 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1156 \ 1157 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 1158 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 1159 &oword); \ 1160 EFX_SET_OWORD_FIELD(oword, \ 1161 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 1162 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 1163 &oword); \ 1164 } \ 1165 \ 1166 _NOTE(CONSTANTCONDITION) \ 1167 } while (B_FALSE) 1168 1169 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 1170 do { \ 1171 efx_oword_t oword; \ 1172 \ 1173 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1174 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 1175 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 1176 (_ip) ? 1 : 0); \ 1177 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 1178 (_tcp) ? 0 : 1); \ 1179 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 1180 (_insert) ? 1 : 0); \ 1181 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1182 \ 1183 _NOTE(CONSTANTCONDITION) \ 1184 } while (B_FALSE) 1185 1186 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 1187 do { \ 1188 efx_oword_t oword; \ 1189 \ 1190 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 1191 EFX_SET_OWORD_FIELD(oword, \ 1192 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 1193 EFX_SET_OWORD_FIELD(oword, \ 1194 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 1195 EFX_SET_OWORD_FIELD(oword, \ 1196 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 1197 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 1198 \ 1199 (_rc) = 0; \ 1200 \ 1201 _NOTE(CONSTANTCONDITION) \ 1202 } while (B_FALSE) 1203 1204 1205 #if EFSYS_OPT_RX_SCALE 1206 1207 static __checkReturn efx_rc_t 1208 siena_rx_scale_mode_set( 1209 __in efx_nic_t *enp, 1210 __in uint32_t rss_context, 1211 __in efx_rx_hash_alg_t alg, 1212 __in efx_rx_hash_type_t type, 1213 __in boolean_t insert) 1214 { 1215 efx_rc_t rc; 1216 1217 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1218 rc = EINVAL; 1219 goto fail1; 1220 } 1221 1222 switch (alg) { 1223 case EFX_RX_HASHALG_LFSR: 1224 EFX_RX_LFSR_HASH(enp, insert); 1225 break; 1226 1227 case EFX_RX_HASHALG_TOEPLITZ: 1228 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 1229 (type & EFX_RX_HASH_IPV4) ? B_TRUE : B_FALSE, 1230 (type & EFX_RX_HASH_TCPIPV4) ? B_TRUE : B_FALSE); 1231 1232 EFX_RX_TOEPLITZ_IPV6_HASH(enp, 1233 (type & EFX_RX_HASH_IPV6) ? B_TRUE : B_FALSE, 1234 (type & EFX_RX_HASH_TCPIPV6) ? B_TRUE : B_FALSE, 1235 rc); 1236 if (rc != 0) 1237 goto fail2; 1238 1239 break; 1240 1241 default: 1242 rc = EINVAL; 1243 goto fail3; 1244 } 1245 1246 return (0); 1247 1248 fail3: 1249 EFSYS_PROBE(fail3); 1250 fail2: 1251 EFSYS_PROBE(fail2); 1252 fail1: 1253 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1254 1255 EFX_RX_LFSR_HASH(enp, B_FALSE); 1256 1257 return (rc); 1258 } 1259 #endif 1260 1261 #if EFSYS_OPT_RX_SCALE 1262 static __checkReturn efx_rc_t 1263 siena_rx_scale_key_set( 1264 __in efx_nic_t *enp, 1265 __in uint32_t rss_context, 1266 __in_ecount(n) uint8_t *key, 1267 __in size_t n) 1268 { 1269 efx_oword_t oword; 1270 unsigned int byte; 1271 unsigned int offset; 1272 efx_rc_t rc; 1273 1274 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1275 rc = EINVAL; 1276 goto fail1; 1277 } 1278 1279 byte = 0; 1280 1281 /* Write Toeplitz IPv4 hash key */ 1282 EFX_ZERO_OWORD(oword); 1283 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 1284 offset > 0 && byte < n; 1285 --offset) 1286 oword.eo_u8[offset - 1] = key[byte++]; 1287 1288 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 1289 1290 byte = 0; 1291 1292 /* Verify Toeplitz IPv4 hash key */ 1293 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 1294 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 1295 offset > 0 && byte < n; 1296 --offset) { 1297 if (oword.eo_u8[offset - 1] != key[byte++]) { 1298 rc = EFAULT; 1299 goto fail2; 1300 } 1301 } 1302 1303 if ((enp->en_features & EFX_FEATURE_IPV6) == 0) 1304 goto done; 1305 1306 byte = 0; 1307 1308 /* Write Toeplitz IPv6 hash key 3 */ 1309 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1310 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1311 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1312 offset > 0 && byte < n; 1313 --offset) 1314 oword.eo_u8[offset - 1] = key[byte++]; 1315 1316 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1317 1318 /* Write Toeplitz IPv6 hash key 2 */ 1319 EFX_ZERO_OWORD(oword); 1320 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1321 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1322 offset > 0 && byte < n; 1323 --offset) 1324 oword.eo_u8[offset - 1] = key[byte++]; 1325 1326 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1327 1328 /* Write Toeplitz IPv6 hash key 1 */ 1329 EFX_ZERO_OWORD(oword); 1330 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1331 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1332 offset > 0 && byte < n; 1333 --offset) 1334 oword.eo_u8[offset - 1] = key[byte++]; 1335 1336 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1337 1338 byte = 0; 1339 1340 /* Verify Toeplitz IPv6 hash key 3 */ 1341 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1342 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1343 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1344 offset > 0 && byte < n; 1345 --offset) { 1346 if (oword.eo_u8[offset - 1] != key[byte++]) { 1347 rc = EFAULT; 1348 goto fail3; 1349 } 1350 } 1351 1352 /* Verify Toeplitz IPv6 hash key 2 */ 1353 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1354 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1355 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1356 offset > 0 && byte < n; 1357 --offset) { 1358 if (oword.eo_u8[offset - 1] != key[byte++]) { 1359 rc = EFAULT; 1360 goto fail4; 1361 } 1362 } 1363 1364 /* Verify Toeplitz IPv6 hash key 1 */ 1365 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1366 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1367 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1368 offset > 0 && byte < n; 1369 --offset) { 1370 if (oword.eo_u8[offset - 1] != key[byte++]) { 1371 rc = EFAULT; 1372 goto fail5; 1373 } 1374 } 1375 1376 done: 1377 return (0); 1378 1379 fail5: 1380 EFSYS_PROBE(fail5); 1381 fail4: 1382 EFSYS_PROBE(fail4); 1383 fail3: 1384 EFSYS_PROBE(fail3); 1385 fail2: 1386 EFSYS_PROBE(fail2); 1387 fail1: 1388 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1389 1390 return (rc); 1391 } 1392 #endif 1393 1394 #if EFSYS_OPT_RX_SCALE 1395 static __checkReturn efx_rc_t 1396 siena_rx_scale_tbl_set( 1397 __in efx_nic_t *enp, 1398 __in uint32_t rss_context, 1399 __in_ecount(n) unsigned int *table, 1400 __in size_t n) 1401 { 1402 efx_oword_t oword; 1403 int index; 1404 efx_rc_t rc; 1405 1406 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 1407 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 1408 1409 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1410 rc = EINVAL; 1411 goto fail1; 1412 } 1413 1414 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 1415 rc = EINVAL; 1416 goto fail2; 1417 } 1418 1419 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 1420 uint32_t byte; 1421 1422 /* Calculate the entry to place in the table */ 1423 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1424 1425 EFSYS_PROBE2(table, int, index, uint32_t, byte); 1426 1427 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 1428 1429 /* Write the table */ 1430 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1431 index, &oword, B_TRUE); 1432 } 1433 1434 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 1435 uint32_t byte; 1436 1437 /* Determine if we're starting a new batch */ 1438 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1439 1440 /* Read the table */ 1441 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 1442 index, &oword, B_TRUE); 1443 1444 /* Verify the entry */ 1445 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 1446 rc = EFAULT; 1447 goto fail3; 1448 } 1449 } 1450 1451 return (0); 1452 1453 fail3: 1454 EFSYS_PROBE(fail3); 1455 fail2: 1456 EFSYS_PROBE(fail2); 1457 fail1: 1458 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1459 1460 return (rc); 1461 } 1462 #endif 1463 1464 /* 1465 * Falcon/Siena pseudo-header 1466 * -------------------------- 1467 * 1468 * Receive packets are prefixed by an optional 16 byte pseudo-header. 1469 * The pseudo-header is a byte array of one of the forms: 1470 * 1471 * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 1472 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT 1473 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL 1474 * 1475 * where: 1476 * TT.TT.TT.TT Toeplitz hash (32-bit big-endian) 1477 * LL.LL LFSR hash (16-bit big-endian) 1478 */ 1479 1480 #if EFSYS_OPT_RX_SCALE 1481 static __checkReturn uint32_t 1482 siena_rx_prefix_hash( 1483 __in efx_nic_t *enp, 1484 __in efx_rx_hash_alg_t func, 1485 __in uint8_t *buffer) 1486 { 1487 _NOTE(ARGUNUSED(enp)) 1488 1489 switch (func) { 1490 case EFX_RX_HASHALG_TOEPLITZ: 1491 return ((buffer[12] << 24) | 1492 (buffer[13] << 16) | 1493 (buffer[14] << 8) | 1494 buffer[15]); 1495 1496 case EFX_RX_HASHALG_LFSR: 1497 return ((buffer[14] << 8) | buffer[15]); 1498 1499 default: 1500 EFSYS_ASSERT(0); 1501 return (0); 1502 } 1503 } 1504 #endif /* EFSYS_OPT_RX_SCALE */ 1505 1506 static __checkReturn efx_rc_t 1507 siena_rx_prefix_pktlen( 1508 __in efx_nic_t *enp, 1509 __in uint8_t *buffer, 1510 __out uint16_t *lengthp) 1511 { 1512 _NOTE(ARGUNUSED(enp, buffer, lengthp)) 1513 1514 /* Not supported by Falcon/Siena hardware */ 1515 EFSYS_ASSERT(0); 1516 return (ENOTSUP); 1517 } 1518 1519 1520 static void 1521 siena_rx_qpost( 1522 __in efx_rxq_t *erp, 1523 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 1524 __in size_t size, 1525 __in unsigned int ndescs, 1526 __in unsigned int completed, 1527 __in unsigned int added) 1528 { 1529 efx_qword_t qword; 1530 unsigned int i; 1531 unsigned int offset; 1532 unsigned int id; 1533 1534 /* The client driver must not overfill the queue */ 1535 EFSYS_ASSERT3U(added - completed + ndescs, <=, 1536 EFX_RXQ_LIMIT(erp->er_mask + 1)); 1537 1538 id = added & (erp->er_mask); 1539 for (i = 0; i < ndescs; i++) { 1540 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 1541 unsigned int, id, efsys_dma_addr_t, addrp[i], 1542 size_t, size); 1543 1544 EFX_POPULATE_QWORD_3(qword, 1545 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 1546 FSF_AZ_RX_KER_BUF_ADDR_DW0, 1547 (uint32_t)(addrp[i] & 0xffffffff), 1548 FSF_AZ_RX_KER_BUF_ADDR_DW1, 1549 (uint32_t)(addrp[i] >> 32)); 1550 1551 offset = id * sizeof (efx_qword_t); 1552 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 1553 1554 id = (id + 1) & (erp->er_mask); 1555 } 1556 } 1557 1558 static void 1559 siena_rx_qpush( 1560 __in efx_rxq_t *erp, 1561 __in unsigned int added, 1562 __inout unsigned int *pushedp) 1563 { 1564 efx_nic_t *enp = erp->er_enp; 1565 unsigned int pushed = *pushedp; 1566 uint32_t wptr; 1567 efx_oword_t oword; 1568 efx_dword_t dword; 1569 1570 /* All descriptors are pushed */ 1571 *pushedp = added; 1572 1573 /* Push the populated descriptors out */ 1574 wptr = added & erp->er_mask; 1575 1576 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 1577 1578 /* Only write the third DWORD */ 1579 EFX_POPULATE_DWORD_1(dword, 1580 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 1581 1582 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 1583 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 1584 wptr, pushed & erp->er_mask); 1585 EFSYS_PIO_WRITE_BARRIER(); 1586 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 1587 erp->er_index, &dword, B_FALSE); 1588 } 1589 1590 #if EFSYS_OPT_RX_PACKED_STREAM 1591 static void 1592 siena_rx_qpush_ps_credits( 1593 __in efx_rxq_t *erp) 1594 { 1595 /* Not supported by Siena hardware */ 1596 EFSYS_ASSERT(0); 1597 } 1598 1599 static uint8_t * 1600 siena_rx_qps_packet_info( 1601 __in efx_rxq_t *erp, 1602 __in uint8_t *buffer, 1603 __in uint32_t buffer_length, 1604 __in uint32_t current_offset, 1605 __out uint16_t *lengthp, 1606 __out uint32_t *next_offsetp, 1607 __out uint32_t *timestamp) 1608 { 1609 /* Not supported by Siena hardware */ 1610 EFSYS_ASSERT(0); 1611 1612 return (NULL); 1613 } 1614 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1615 1616 static __checkReturn efx_rc_t 1617 siena_rx_qflush( 1618 __in efx_rxq_t *erp) 1619 { 1620 efx_nic_t *enp = erp->er_enp; 1621 efx_oword_t oword; 1622 uint32_t label; 1623 1624 label = erp->er_index; 1625 1626 /* Flush the queue */ 1627 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 1628 FRF_AZ_RX_FLUSH_DESCQ, label); 1629 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 1630 1631 return (0); 1632 } 1633 1634 static void 1635 siena_rx_qenable( 1636 __in efx_rxq_t *erp) 1637 { 1638 efx_nic_t *enp = erp->er_enp; 1639 efx_oword_t oword; 1640 1641 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1642 1643 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 1644 erp->er_index, &oword, B_TRUE); 1645 1646 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 1647 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 1648 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 1649 1650 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1651 erp->er_index, &oword, B_TRUE); 1652 } 1653 1654 static __checkReturn efx_rc_t 1655 siena_rx_qcreate( 1656 __in efx_nic_t *enp, 1657 __in unsigned int index, 1658 __in unsigned int label, 1659 __in efx_rxq_type_t type, 1660 __in_opt const efx_rxq_type_data_t *type_data, 1661 __in efsys_mem_t *esmp, 1662 __in size_t ndescs, 1663 __in uint32_t id, 1664 __in unsigned int flags, 1665 __in efx_evq_t *eep, 1666 __in efx_rxq_t *erp) 1667 { 1668 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1669 efx_oword_t oword; 1670 uint32_t size; 1671 boolean_t jumbo = B_FALSE; 1672 efx_rc_t rc; 1673 1674 _NOTE(ARGUNUSED(esmp)) 1675 1676 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == 1677 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 1678 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1679 1680 for (size = 0; 1681 (1U << size) <= encp->enc_rxq_max_ndescs / encp->enc_rxq_min_ndescs; 1682 size++) 1683 if ((1U << size) == (uint32_t)ndescs / encp->enc_rxq_min_ndescs) 1684 break; 1685 if (id + (1 << size) >= encp->enc_buftbl_limit) { 1686 rc = EINVAL; 1687 goto fail1; 1688 } 1689 1690 switch (type) { 1691 case EFX_RXQ_TYPE_DEFAULT: 1692 erp->er_buf_size = type_data->ertd_default.ed_buf_size; 1693 break; 1694 1695 default: 1696 rc = EINVAL; 1697 goto fail2; 1698 } 1699 1700 if (flags & EFX_RXQ_FLAG_SCATTER) { 1701 #if EFSYS_OPT_RX_SCATTER 1702 jumbo = B_TRUE; 1703 #else 1704 rc = EINVAL; 1705 goto fail3; 1706 #endif /* EFSYS_OPT_RX_SCATTER */ 1707 } 1708 1709 /* Set up the new descriptor queue */ 1710 EFX_POPULATE_OWORD_7(oword, 1711 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 1712 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 1713 FRF_AZ_RX_DESCQ_OWNER_ID, 0, 1714 FRF_AZ_RX_DESCQ_LABEL, label, 1715 FRF_AZ_RX_DESCQ_SIZE, size, 1716 FRF_AZ_RX_DESCQ_TYPE, 0, 1717 FRF_AZ_RX_DESCQ_JUMBO, jumbo); 1718 1719 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1720 erp->er_index, &oword, B_TRUE); 1721 1722 return (0); 1723 1724 #if !EFSYS_OPT_RX_SCATTER 1725 fail3: 1726 EFSYS_PROBE(fail3); 1727 #endif 1728 fail2: 1729 EFSYS_PROBE(fail2); 1730 fail1: 1731 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1732 1733 return (rc); 1734 } 1735 1736 static void 1737 siena_rx_qdestroy( 1738 __in efx_rxq_t *erp) 1739 { 1740 efx_nic_t *enp = erp->er_enp; 1741 efx_oword_t oword; 1742 1743 /* Purge descriptor queue */ 1744 EFX_ZERO_OWORD(oword); 1745 1746 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1747 erp->er_index, &oword, B_TRUE); 1748 } 1749 1750 static void 1751 siena_rx_fini( 1752 __in efx_nic_t *enp) 1753 { 1754 _NOTE(ARGUNUSED(enp)) 1755 } 1756 1757 #endif /* EFSYS_OPT_SIENA */ 1758