1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2020 Xilinx, Inc. 4 * Copyright(c) 2007-2019 Solarflare Communications Inc. 5 */ 6 7 #include "efx.h" 8 #include "efx_impl.h" 9 10 11 #if EFSYS_OPT_SIENA 12 13 static __checkReturn efx_rc_t 14 siena_rx_init( 15 __in efx_nic_t *enp); 16 17 static void 18 siena_rx_fini( 19 __in efx_nic_t *enp); 20 21 #if EFSYS_OPT_RX_SCATTER 22 static __checkReturn efx_rc_t 23 siena_rx_scatter_enable( 24 __in efx_nic_t *enp, 25 __in unsigned int buf_size); 26 #endif /* EFSYS_OPT_RX_SCATTER */ 27 28 #if EFSYS_OPT_RX_SCALE 29 static __checkReturn efx_rc_t 30 siena_rx_scale_mode_set( 31 __in efx_nic_t *enp, 32 __in uint32_t rss_context, 33 __in efx_rx_hash_alg_t alg, 34 __in efx_rx_hash_type_t type, 35 __in boolean_t insert); 36 37 static __checkReturn efx_rc_t 38 siena_rx_scale_key_set( 39 __in efx_nic_t *enp, 40 __in uint32_t rss_context, 41 __in_ecount(n) uint8_t *key, 42 __in size_t n); 43 44 static __checkReturn efx_rc_t 45 siena_rx_scale_tbl_set( 46 __in efx_nic_t *enp, 47 __in uint32_t rss_context, 48 __in_ecount(n) unsigned int *table, 49 __in size_t n); 50 51 static __checkReturn uint32_t 52 siena_rx_prefix_hash( 53 __in efx_nic_t *enp, 54 __in efx_rx_hash_alg_t func, 55 __in uint8_t *buffer); 56 57 #endif /* EFSYS_OPT_RX_SCALE */ 58 59 static __checkReturn efx_rc_t 60 siena_rx_prefix_pktlen( 61 __in efx_nic_t *enp, 62 __in uint8_t *buffer, 63 __out uint16_t *lengthp); 64 65 static void 66 siena_rx_qpost( 67 __in efx_rxq_t *erp, 68 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 69 __in size_t size, 70 __in unsigned int ndescs, 71 __in unsigned int completed, 72 __in unsigned int added); 73 74 static void 75 siena_rx_qpush( 76 __in efx_rxq_t *erp, 77 __in unsigned int added, 78 __inout unsigned int *pushedp); 79 80 #if EFSYS_OPT_RX_PACKED_STREAM 81 static void 82 siena_rx_qpush_ps_credits( 83 __in efx_rxq_t *erp); 84 85 static __checkReturn uint8_t * 86 siena_rx_qps_packet_info( 87 __in efx_rxq_t *erp, 88 __in uint8_t *buffer, 89 __in uint32_t buffer_length, 90 __in uint32_t current_offset, 91 __out uint16_t *lengthp, 92 __out uint32_t *next_offsetp, 93 __out uint32_t *timestamp); 94 #endif 95 96 static __checkReturn efx_rc_t 97 siena_rx_qflush( 98 __in efx_rxq_t *erp); 99 100 static void 101 siena_rx_qenable( 102 __in efx_rxq_t *erp); 103 104 static __checkReturn efx_rc_t 105 siena_rx_qcreate( 106 __in efx_nic_t *enp, 107 __in unsigned int index, 108 __in unsigned int label, 109 __in efx_rxq_type_t type, 110 __in_opt const efx_rxq_type_data_t *type_data, 111 __in efsys_mem_t *esmp, 112 __in size_t ndescs, 113 __in uint32_t id, 114 __in unsigned int flags, 115 __in efx_evq_t *eep, 116 __in efx_rxq_t *erp); 117 118 static void 119 siena_rx_qdestroy( 120 __in efx_rxq_t *erp); 121 122 #endif /* EFSYS_OPT_SIENA */ 123 124 125 #if EFSYS_OPT_SIENA 126 static const efx_rx_ops_t __efx_rx_siena_ops = { 127 siena_rx_init, /* erxo_init */ 128 siena_rx_fini, /* erxo_fini */ 129 #if EFSYS_OPT_RX_SCATTER 130 siena_rx_scatter_enable, /* erxo_scatter_enable */ 131 #endif 132 #if EFSYS_OPT_RX_SCALE 133 NULL, /* erxo_scale_context_alloc */ 134 NULL, /* erxo_scale_context_free */ 135 siena_rx_scale_mode_set, /* erxo_scale_mode_set */ 136 siena_rx_scale_key_set, /* erxo_scale_key_set */ 137 siena_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 138 siena_rx_prefix_hash, /* erxo_prefix_hash */ 139 #endif 140 siena_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 141 siena_rx_qpost, /* erxo_qpost */ 142 siena_rx_qpush, /* erxo_qpush */ 143 #if EFSYS_OPT_RX_PACKED_STREAM 144 siena_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 145 siena_rx_qps_packet_info, /* erxo_qps_packet_info */ 146 #endif 147 siena_rx_qflush, /* erxo_qflush */ 148 siena_rx_qenable, /* erxo_qenable */ 149 siena_rx_qcreate, /* erxo_qcreate */ 150 siena_rx_qdestroy, /* erxo_qdestroy */ 151 }; 152 #endif /* EFSYS_OPT_SIENA */ 153 154 #if EFX_OPTS_EF10() 155 static const efx_rx_ops_t __efx_rx_ef10_ops = { 156 ef10_rx_init, /* erxo_init */ 157 ef10_rx_fini, /* erxo_fini */ 158 #if EFSYS_OPT_RX_SCATTER 159 ef10_rx_scatter_enable, /* erxo_scatter_enable */ 160 #endif 161 #if EFSYS_OPT_RX_SCALE 162 ef10_rx_scale_context_alloc, /* erxo_scale_context_alloc */ 163 ef10_rx_scale_context_free, /* erxo_scale_context_free */ 164 ef10_rx_scale_mode_set, /* erxo_scale_mode_set */ 165 ef10_rx_scale_key_set, /* erxo_scale_key_set */ 166 ef10_rx_scale_tbl_set, /* erxo_scale_tbl_set */ 167 ef10_rx_prefix_hash, /* erxo_prefix_hash */ 168 #endif 169 ef10_rx_prefix_pktlen, /* erxo_prefix_pktlen */ 170 ef10_rx_qpost, /* erxo_qpost */ 171 ef10_rx_qpush, /* erxo_qpush */ 172 #if EFSYS_OPT_RX_PACKED_STREAM 173 ef10_rx_qpush_ps_credits, /* erxo_qpush_ps_credits */ 174 ef10_rx_qps_packet_info, /* erxo_qps_packet_info */ 175 #endif 176 ef10_rx_qflush, /* erxo_qflush */ 177 ef10_rx_qenable, /* erxo_qenable */ 178 ef10_rx_qcreate, /* erxo_qcreate */ 179 ef10_rx_qdestroy, /* erxo_qdestroy */ 180 }; 181 #endif /* EFX_OPTS_EF10() */ 182 183 184 __checkReturn efx_rc_t 185 efx_rx_init( 186 __inout efx_nic_t *enp) 187 { 188 const efx_rx_ops_t *erxop; 189 efx_rc_t rc; 190 191 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 192 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 193 194 if (!(enp->en_mod_flags & EFX_MOD_EV)) { 195 rc = EINVAL; 196 goto fail1; 197 } 198 199 if (enp->en_mod_flags & EFX_MOD_RX) { 200 rc = EINVAL; 201 goto fail2; 202 } 203 204 switch (enp->en_family) { 205 #if EFSYS_OPT_SIENA 206 case EFX_FAMILY_SIENA: 207 erxop = &__efx_rx_siena_ops; 208 break; 209 #endif /* EFSYS_OPT_SIENA */ 210 211 #if EFSYS_OPT_HUNTINGTON 212 case EFX_FAMILY_HUNTINGTON: 213 erxop = &__efx_rx_ef10_ops; 214 break; 215 #endif /* EFSYS_OPT_HUNTINGTON */ 216 217 #if EFSYS_OPT_MEDFORD 218 case EFX_FAMILY_MEDFORD: 219 erxop = &__efx_rx_ef10_ops; 220 break; 221 #endif /* EFSYS_OPT_MEDFORD */ 222 223 #if EFSYS_OPT_MEDFORD2 224 case EFX_FAMILY_MEDFORD2: 225 erxop = &__efx_rx_ef10_ops; 226 break; 227 #endif /* EFSYS_OPT_MEDFORD2 */ 228 229 default: 230 EFSYS_ASSERT(0); 231 rc = ENOTSUP; 232 goto fail3; 233 } 234 235 if ((rc = erxop->erxo_init(enp)) != 0) 236 goto fail4; 237 238 enp->en_erxop = erxop; 239 enp->en_mod_flags |= EFX_MOD_RX; 240 return (0); 241 242 fail4: 243 EFSYS_PROBE(fail4); 244 fail3: 245 EFSYS_PROBE(fail3); 246 fail2: 247 EFSYS_PROBE(fail2); 248 fail1: 249 EFSYS_PROBE1(fail1, efx_rc_t, rc); 250 251 enp->en_erxop = NULL; 252 enp->en_mod_flags &= ~EFX_MOD_RX; 253 return (rc); 254 } 255 256 void 257 efx_rx_fini( 258 __in efx_nic_t *enp) 259 { 260 const efx_rx_ops_t *erxop = enp->en_erxop; 261 262 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 263 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_NIC); 264 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 265 EFSYS_ASSERT3U(enp->en_rx_qcount, ==, 0); 266 267 erxop->erxo_fini(enp); 268 269 enp->en_erxop = NULL; 270 enp->en_mod_flags &= ~EFX_MOD_RX; 271 } 272 273 #if EFSYS_OPT_RX_SCATTER 274 __checkReturn efx_rc_t 275 efx_rx_scatter_enable( 276 __in efx_nic_t *enp, 277 __in unsigned int buf_size) 278 { 279 const efx_rx_ops_t *erxop = enp->en_erxop; 280 efx_rc_t rc; 281 282 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 283 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 284 285 if ((rc = erxop->erxo_scatter_enable(enp, buf_size)) != 0) 286 goto fail1; 287 288 return (0); 289 290 fail1: 291 EFSYS_PROBE1(fail1, efx_rc_t, rc); 292 return (rc); 293 } 294 #endif /* EFSYS_OPT_RX_SCATTER */ 295 296 #if EFSYS_OPT_RX_SCALE 297 __checkReturn efx_rc_t 298 efx_rx_scale_hash_flags_get( 299 __in efx_nic_t *enp, 300 __in efx_rx_hash_alg_t hash_alg, 301 __out_ecount_part(max_nflags, *nflagsp) unsigned int *flagsp, 302 __in unsigned int max_nflags, 303 __out unsigned int *nflagsp) 304 { 305 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 306 unsigned int nflags = 0; 307 efx_rc_t rc; 308 309 if (flagsp == NULL || nflagsp == NULL) { 310 rc = EINVAL; 311 goto fail1; 312 } 313 314 if ((encp->enc_rx_scale_hash_alg_mask & (1U << hash_alg)) == 0) { 315 nflags = 0; 316 goto done; 317 } 318 319 /* Helper to add flags word to flags array without buffer overflow */ 320 #define INSERT_FLAGS(_flags) \ 321 do { \ 322 if (nflags >= max_nflags) { \ 323 rc = E2BIG; \ 324 goto fail2; \ 325 } \ 326 *(flagsp + nflags) = (_flags); \ 327 nflags++; \ 328 \ 329 _NOTE(CONSTANTCONDITION) \ 330 } while (B_FALSE) 331 332 if (encp->enc_rx_scale_l4_hash_supported != B_FALSE) { 333 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 4TUPLE)); 334 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 4TUPLE)); 335 } 336 337 if ((encp->enc_rx_scale_l4_hash_supported != B_FALSE) && 338 (encp->enc_rx_scale_additional_modes_supported != B_FALSE)) { 339 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_DST)); 340 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE_SRC)); 341 342 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_DST)); 343 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE_SRC)); 344 345 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 4TUPLE)); 346 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_DST)); 347 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE_SRC)); 348 349 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 4TUPLE)); 350 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_DST)); 351 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE_SRC)); 352 } 353 354 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 2TUPLE)); 355 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 2TUPLE)); 356 357 INSERT_FLAGS(EFX_RX_HASH(IPV4, 2TUPLE)); 358 INSERT_FLAGS(EFX_RX_HASH(IPV6, 2TUPLE)); 359 360 if (encp->enc_rx_scale_additional_modes_supported != B_FALSE) { 361 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_DST)); 362 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, 1TUPLE_SRC)); 363 364 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_DST)); 365 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, 1TUPLE_SRC)); 366 367 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 2TUPLE)); 368 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_DST)); 369 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, 1TUPLE_SRC)); 370 371 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 2TUPLE)); 372 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_DST)); 373 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, 1TUPLE_SRC)); 374 375 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_DST)); 376 INSERT_FLAGS(EFX_RX_HASH(IPV4, 1TUPLE_SRC)); 377 378 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_DST)); 379 INSERT_FLAGS(EFX_RX_HASH(IPV6, 1TUPLE_SRC)); 380 } 381 382 INSERT_FLAGS(EFX_RX_HASH(IPV4_TCP, DISABLE)); 383 INSERT_FLAGS(EFX_RX_HASH(IPV6_TCP, DISABLE)); 384 385 INSERT_FLAGS(EFX_RX_HASH(IPV4_UDP, DISABLE)); 386 INSERT_FLAGS(EFX_RX_HASH(IPV6_UDP, DISABLE)); 387 388 INSERT_FLAGS(EFX_RX_HASH(IPV4, DISABLE)); 389 INSERT_FLAGS(EFX_RX_HASH(IPV6, DISABLE)); 390 391 #undef INSERT_FLAGS 392 393 done: 394 *nflagsp = nflags; 395 return (0); 396 397 fail2: 398 EFSYS_PROBE(fail2); 399 fail1: 400 EFSYS_PROBE1(fail1, efx_rc_t, rc); 401 402 return (rc); 403 } 404 405 __checkReturn efx_rc_t 406 efx_rx_hash_default_support_get( 407 __in efx_nic_t *enp, 408 __out efx_rx_hash_support_t *supportp) 409 { 410 efx_rc_t rc; 411 412 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 413 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 414 415 if (supportp == NULL) { 416 rc = EINVAL; 417 goto fail1; 418 } 419 420 /* 421 * Report the hashing support the client gets by default if it 422 * does not allocate an RSS context itself. 423 */ 424 *supportp = enp->en_hash_support; 425 426 return (0); 427 428 fail1: 429 EFSYS_PROBE1(fail1, efx_rc_t, rc); 430 431 return (rc); 432 } 433 434 __checkReturn efx_rc_t 435 efx_rx_scale_default_support_get( 436 __in efx_nic_t *enp, 437 __out efx_rx_scale_context_type_t *typep) 438 { 439 efx_rc_t rc; 440 441 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 442 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 443 444 if (typep == NULL) { 445 rc = EINVAL; 446 goto fail1; 447 } 448 449 /* 450 * Report the RSS support the client gets by default if it 451 * does not allocate an RSS context itself. 452 */ 453 *typep = enp->en_rss_context_type; 454 455 return (0); 456 457 fail1: 458 EFSYS_PROBE1(fail1, efx_rc_t, rc); 459 460 return (rc); 461 } 462 #endif /* EFSYS_OPT_RX_SCALE */ 463 464 #if EFSYS_OPT_RX_SCALE 465 __checkReturn efx_rc_t 466 efx_rx_scale_context_alloc( 467 __in efx_nic_t *enp, 468 __in efx_rx_scale_context_type_t type, 469 __in uint32_t num_queues, 470 __out uint32_t *rss_contextp) 471 { 472 const efx_rx_ops_t *erxop = enp->en_erxop; 473 efx_rc_t rc; 474 475 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 476 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 477 478 if (erxop->erxo_scale_context_alloc == NULL) { 479 rc = ENOTSUP; 480 goto fail1; 481 } 482 if ((rc = erxop->erxo_scale_context_alloc(enp, type, 483 num_queues, rss_contextp)) != 0) { 484 goto fail2; 485 } 486 487 return (0); 488 489 fail2: 490 EFSYS_PROBE(fail2); 491 fail1: 492 EFSYS_PROBE1(fail1, efx_rc_t, rc); 493 return (rc); 494 } 495 #endif /* EFSYS_OPT_RX_SCALE */ 496 497 #if EFSYS_OPT_RX_SCALE 498 __checkReturn efx_rc_t 499 efx_rx_scale_context_free( 500 __in efx_nic_t *enp, 501 __in uint32_t rss_context) 502 { 503 const efx_rx_ops_t *erxop = enp->en_erxop; 504 efx_rc_t rc; 505 506 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 507 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 508 509 if (erxop->erxo_scale_context_free == NULL) { 510 rc = ENOTSUP; 511 goto fail1; 512 } 513 if ((rc = erxop->erxo_scale_context_free(enp, rss_context)) != 0) 514 goto fail2; 515 516 return (0); 517 518 fail2: 519 EFSYS_PROBE(fail2); 520 fail1: 521 EFSYS_PROBE1(fail1, efx_rc_t, rc); 522 return (rc); 523 } 524 #endif /* EFSYS_OPT_RX_SCALE */ 525 526 #if EFSYS_OPT_RX_SCALE 527 __checkReturn efx_rc_t 528 efx_rx_scale_mode_set( 529 __in efx_nic_t *enp, 530 __in uint32_t rss_context, 531 __in efx_rx_hash_alg_t alg, 532 __in efx_rx_hash_type_t type, 533 __in boolean_t insert) 534 { 535 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 536 const efx_rx_ops_t *erxop = enp->en_erxop; 537 efx_rx_hash_type_t type_check; 538 unsigned int i; 539 efx_rc_t rc; 540 541 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 542 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 543 544 /* 545 * Legacy flags and modern bits cannot be 546 * used at the same time in the hash type. 547 */ 548 if ((type & EFX_RX_HASH_LEGACY_MASK) && 549 (type & ~EFX_RX_HASH_LEGACY_MASK)) { 550 rc = EINVAL; 551 goto fail1; 552 } 553 554 /* 555 * If RSS hash type is represented by additional bits 556 * in the value, the latter need to be verified since 557 * not all bit combinations are valid RSS modes. Also, 558 * depending on the firmware, some valid combinations 559 * may be unsupported. Discern additional bits in the 560 * type value and try to recognise valid combinations. 561 * If some bits remain unrecognised, report the error. 562 */ 563 type_check = type & ~EFX_RX_HASH_LEGACY_MASK; 564 if (type_check != 0) { 565 unsigned int type_flags[EFX_RX_HASH_NFLAGS]; 566 unsigned int type_nflags; 567 568 rc = efx_rx_scale_hash_flags_get(enp, alg, type_flags, 569 EFX_ARRAY_SIZE(type_flags), &type_nflags); 570 if (rc != 0) 571 goto fail2; 572 573 for (i = 0; i < type_nflags; ++i) { 574 if ((type_check & type_flags[i]) == type_flags[i]) 575 type_check &= ~(type_flags[i]); 576 } 577 578 if (type_check != 0) { 579 rc = EINVAL; 580 goto fail3; 581 } 582 } 583 584 /* 585 * Translate EFX_RX_HASH() flags to their legacy counterparts 586 * provided that the FW claims no support for additional modes. 587 */ 588 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE) { 589 efx_rx_hash_type_t t_ipv4 = EFX_RX_HASH(IPV4, 2TUPLE) | 590 EFX_RX_HASH(IPV4_TCP, 2TUPLE); 591 efx_rx_hash_type_t t_ipv6 = EFX_RX_HASH(IPV6, 2TUPLE) | 592 EFX_RX_HASH(IPV6_TCP, 2TUPLE); 593 efx_rx_hash_type_t t_ipv4_tcp = EFX_RX_HASH(IPV4_TCP, 4TUPLE); 594 efx_rx_hash_type_t t_ipv6_tcp = EFX_RX_HASH(IPV6_TCP, 4TUPLE); 595 596 if ((type & t_ipv4) == t_ipv4) 597 type |= EFX_RX_HASH_IPV4; 598 if ((type & t_ipv6) == t_ipv6) 599 type |= EFX_RX_HASH_IPV6; 600 601 if (encp->enc_rx_scale_l4_hash_supported == B_TRUE) { 602 if ((type & t_ipv4_tcp) == t_ipv4_tcp) 603 type |= EFX_RX_HASH_TCPIPV4; 604 if ((type & t_ipv6_tcp) == t_ipv6_tcp) 605 type |= EFX_RX_HASH_TCPIPV6; 606 } 607 608 type &= EFX_RX_HASH_LEGACY_MASK; 609 } 610 611 if (erxop->erxo_scale_mode_set != NULL) { 612 if ((rc = erxop->erxo_scale_mode_set(enp, rss_context, alg, 613 type, insert)) != 0) 614 goto fail4; 615 } 616 617 return (0); 618 619 fail4: 620 EFSYS_PROBE(fail4); 621 fail3: 622 EFSYS_PROBE(fail3); 623 fail2: 624 EFSYS_PROBE(fail2); 625 fail1: 626 EFSYS_PROBE1(fail1, efx_rc_t, rc); 627 return (rc); 628 } 629 #endif /* EFSYS_OPT_RX_SCALE */ 630 631 #if EFSYS_OPT_RX_SCALE 632 __checkReturn efx_rc_t 633 efx_rx_scale_key_set( 634 __in efx_nic_t *enp, 635 __in uint32_t rss_context, 636 __in_ecount(n) uint8_t *key, 637 __in size_t n) 638 { 639 const efx_rx_ops_t *erxop = enp->en_erxop; 640 efx_rc_t rc; 641 642 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 643 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 644 645 if ((rc = erxop->erxo_scale_key_set(enp, rss_context, key, n)) != 0) 646 goto fail1; 647 648 return (0); 649 650 fail1: 651 EFSYS_PROBE1(fail1, efx_rc_t, rc); 652 653 return (rc); 654 } 655 #endif /* EFSYS_OPT_RX_SCALE */ 656 657 #if EFSYS_OPT_RX_SCALE 658 __checkReturn efx_rc_t 659 efx_rx_scale_tbl_set( 660 __in efx_nic_t *enp, 661 __in uint32_t rss_context, 662 __in_ecount(n) unsigned int *table, 663 __in size_t n) 664 { 665 const efx_rx_ops_t *erxop = enp->en_erxop; 666 efx_rc_t rc; 667 668 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 669 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 670 671 if ((rc = erxop->erxo_scale_tbl_set(enp, rss_context, table, n)) != 0) 672 goto fail1; 673 674 return (0); 675 676 fail1: 677 EFSYS_PROBE1(fail1, efx_rc_t, rc); 678 679 return (rc); 680 } 681 #endif /* EFSYS_OPT_RX_SCALE */ 682 683 void 684 efx_rx_qpost( 685 __in efx_rxq_t *erp, 686 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 687 __in size_t size, 688 __in unsigned int ndescs, 689 __in unsigned int completed, 690 __in unsigned int added) 691 { 692 efx_nic_t *enp = erp->er_enp; 693 const efx_rx_ops_t *erxop = enp->en_erxop; 694 695 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 696 EFSYS_ASSERT(erp->er_buf_size == 0 || size == erp->er_buf_size); 697 698 erxop->erxo_qpost(erp, addrp, size, ndescs, completed, added); 699 } 700 701 #if EFSYS_OPT_RX_PACKED_STREAM 702 703 void 704 efx_rx_qpush_ps_credits( 705 __in efx_rxq_t *erp) 706 { 707 efx_nic_t *enp = erp->er_enp; 708 const efx_rx_ops_t *erxop = enp->en_erxop; 709 710 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 711 712 erxop->erxo_qpush_ps_credits(erp); 713 } 714 715 __checkReturn uint8_t * 716 efx_rx_qps_packet_info( 717 __in efx_rxq_t *erp, 718 __in uint8_t *buffer, 719 __in uint32_t buffer_length, 720 __in uint32_t current_offset, 721 __out uint16_t *lengthp, 722 __out uint32_t *next_offsetp, 723 __out uint32_t *timestamp) 724 { 725 efx_nic_t *enp = erp->er_enp; 726 const efx_rx_ops_t *erxop = enp->en_erxop; 727 728 return (erxop->erxo_qps_packet_info(erp, buffer, 729 buffer_length, current_offset, lengthp, 730 next_offsetp, timestamp)); 731 } 732 733 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 734 735 void 736 efx_rx_qpush( 737 __in efx_rxq_t *erp, 738 __in unsigned int added, 739 __inout unsigned int *pushedp) 740 { 741 efx_nic_t *enp = erp->er_enp; 742 const efx_rx_ops_t *erxop = enp->en_erxop; 743 744 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 745 746 erxop->erxo_qpush(erp, added, pushedp); 747 } 748 749 __checkReturn efx_rc_t 750 efx_rx_qflush( 751 __in efx_rxq_t *erp) 752 { 753 efx_nic_t *enp = erp->er_enp; 754 const efx_rx_ops_t *erxop = enp->en_erxop; 755 efx_rc_t rc; 756 757 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 758 759 if ((rc = erxop->erxo_qflush(erp)) != 0) 760 goto fail1; 761 762 return (0); 763 764 fail1: 765 EFSYS_PROBE1(fail1, efx_rc_t, rc); 766 767 return (rc); 768 } 769 770 __checkReturn size_t 771 efx_rxq_size( 772 __in const efx_nic_t *enp, 773 __in unsigned int ndescs) 774 { 775 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 776 777 return (ndescs * encp->enc_rx_desc_size); 778 } 779 780 __checkReturn unsigned int 781 efx_rxq_nbufs( 782 __in const efx_nic_t *enp, 783 __in unsigned int ndescs) 784 { 785 return (EFX_DIV_ROUND_UP(efx_rxq_size(enp, ndescs), EFX_BUF_SIZE)); 786 } 787 788 void 789 efx_rx_qenable( 790 __in efx_rxq_t *erp) 791 { 792 efx_nic_t *enp = erp->er_enp; 793 const efx_rx_ops_t *erxop = enp->en_erxop; 794 795 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 796 797 erxop->erxo_qenable(erp); 798 } 799 800 static __checkReturn efx_rc_t 801 efx_rx_qcreate_internal( 802 __in efx_nic_t *enp, 803 __in unsigned int index, 804 __in unsigned int label, 805 __in efx_rxq_type_t type, 806 __in_opt const efx_rxq_type_data_t *type_data, 807 __in efsys_mem_t *esmp, 808 __in size_t ndescs, 809 __in uint32_t id, 810 __in unsigned int flags, 811 __in efx_evq_t *eep, 812 __deref_out efx_rxq_t **erpp) 813 { 814 const efx_rx_ops_t *erxop = enp->en_erxop; 815 efx_rxq_t *erp; 816 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 817 efx_rc_t rc; 818 819 EFSYS_ASSERT3U(enp->en_magic, ==, EFX_NIC_MAGIC); 820 EFSYS_ASSERT3U(enp->en_mod_flags, &, EFX_MOD_RX); 821 822 EFSYS_ASSERT3U(enp->en_rx_qcount + 1, <, encp->enc_rxq_limit); 823 824 EFSYS_ASSERT(ISP2(encp->enc_rxq_max_ndescs)); 825 EFSYS_ASSERT(ISP2(encp->enc_rxq_min_ndescs)); 826 827 if (index >= encp->enc_rxq_limit) { 828 rc = EINVAL; 829 goto fail1; 830 } 831 832 if (!ISP2(ndescs) || 833 ndescs < encp->enc_rxq_min_ndescs || 834 ndescs > encp->enc_rxq_max_ndescs) { 835 rc = EINVAL; 836 goto fail2; 837 } 838 839 /* Allocate an RXQ object */ 840 EFSYS_KMEM_ALLOC(enp->en_esip, sizeof (efx_rxq_t), erp); 841 842 if (erp == NULL) { 843 rc = ENOMEM; 844 goto fail3; 845 } 846 847 erp->er_magic = EFX_RXQ_MAGIC; 848 erp->er_enp = enp; 849 erp->er_index = index; 850 erp->er_mask = ndescs - 1; 851 erp->er_esmp = esmp; 852 853 if ((rc = erxop->erxo_qcreate(enp, index, label, type, type_data, esmp, 854 ndescs, id, flags, eep, erp)) != 0) 855 goto fail4; 856 857 enp->en_rx_qcount++; 858 *erpp = erp; 859 860 return (0); 861 862 fail4: 863 EFSYS_PROBE(fail4); 864 865 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 866 fail3: 867 EFSYS_PROBE(fail3); 868 fail2: 869 EFSYS_PROBE(fail2); 870 fail1: 871 EFSYS_PROBE1(fail1, efx_rc_t, rc); 872 873 return (rc); 874 } 875 876 __checkReturn efx_rc_t 877 efx_rx_qcreate( 878 __in efx_nic_t *enp, 879 __in unsigned int index, 880 __in unsigned int label, 881 __in efx_rxq_type_t type, 882 __in size_t buf_size, 883 __in efsys_mem_t *esmp, 884 __in size_t ndescs, 885 __in uint32_t id, 886 __in unsigned int flags, 887 __in efx_evq_t *eep, 888 __deref_out efx_rxq_t **erpp) 889 { 890 efx_rxq_type_data_t type_data; 891 892 memset(&type_data, 0, sizeof (type_data)); 893 894 type_data.ertd_default.ed_buf_size = buf_size; 895 896 return efx_rx_qcreate_internal(enp, index, label, type, &type_data, 897 esmp, ndescs, id, flags, eep, erpp); 898 } 899 900 #if EFSYS_OPT_RX_PACKED_STREAM 901 902 __checkReturn efx_rc_t 903 efx_rx_qcreate_packed_stream( 904 __in efx_nic_t *enp, 905 __in unsigned int index, 906 __in unsigned int label, 907 __in uint32_t ps_buf_size, 908 __in efsys_mem_t *esmp, 909 __in size_t ndescs, 910 __in efx_evq_t *eep, 911 __deref_out efx_rxq_t **erpp) 912 { 913 efx_rxq_type_data_t type_data; 914 915 memset(&type_data, 0, sizeof (type_data)); 916 917 type_data.ertd_packed_stream.eps_buf_size = ps_buf_size; 918 919 return efx_rx_qcreate_internal(enp, index, label, 920 EFX_RXQ_TYPE_PACKED_STREAM, &type_data, esmp, ndescs, 921 0 /* id unused on EF10 */, EFX_RXQ_FLAG_NONE, eep, erpp); 922 } 923 924 #endif 925 926 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 927 928 __checkReturn efx_rc_t 929 efx_rx_qcreate_es_super_buffer( 930 __in efx_nic_t *enp, 931 __in unsigned int index, 932 __in unsigned int label, 933 __in uint32_t n_bufs_per_desc, 934 __in uint32_t max_dma_len, 935 __in uint32_t buf_stride, 936 __in uint32_t hol_block_timeout, 937 __in efsys_mem_t *esmp, 938 __in size_t ndescs, 939 __in unsigned int flags, 940 __in efx_evq_t *eep, 941 __deref_out efx_rxq_t **erpp) 942 { 943 efx_rc_t rc; 944 efx_rxq_type_data_t type_data; 945 946 if (hol_block_timeout > EFX_RXQ_ES_SUPER_BUFFER_HOL_BLOCK_MAX) { 947 rc = EINVAL; 948 goto fail1; 949 } 950 951 memset(&type_data, 0, sizeof (type_data)); 952 953 type_data.ertd_es_super_buffer.eessb_bufs_per_desc = n_bufs_per_desc; 954 type_data.ertd_es_super_buffer.eessb_max_dma_len = max_dma_len; 955 type_data.ertd_es_super_buffer.eessb_buf_stride = buf_stride; 956 type_data.ertd_es_super_buffer.eessb_hol_block_timeout = 957 hol_block_timeout; 958 959 rc = efx_rx_qcreate_internal(enp, index, label, 960 EFX_RXQ_TYPE_ES_SUPER_BUFFER, &type_data, esmp, ndescs, 961 0 /* id unused on EF10 */, flags, eep, erpp); 962 if (rc != 0) 963 goto fail2; 964 965 return (0); 966 967 fail2: 968 EFSYS_PROBE(fail2); 969 fail1: 970 EFSYS_PROBE1(fail1, efx_rc_t, rc); 971 972 return (rc); 973 } 974 975 #endif 976 977 978 void 979 efx_rx_qdestroy( 980 __in efx_rxq_t *erp) 981 { 982 efx_nic_t *enp = erp->er_enp; 983 const efx_rx_ops_t *erxop = enp->en_erxop; 984 985 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 986 987 EFSYS_ASSERT(enp->en_rx_qcount != 0); 988 --enp->en_rx_qcount; 989 990 erxop->erxo_qdestroy(erp); 991 992 /* Free the RXQ object */ 993 EFSYS_KMEM_FREE(enp->en_esip, sizeof (efx_rxq_t), erp); 994 } 995 996 __checkReturn efx_rc_t 997 efx_pseudo_hdr_pkt_length_get( 998 __in efx_rxq_t *erp, 999 __in uint8_t *buffer, 1000 __out uint16_t *lengthp) 1001 { 1002 efx_nic_t *enp = erp->er_enp; 1003 const efx_rx_ops_t *erxop = enp->en_erxop; 1004 1005 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1006 1007 return (erxop->erxo_prefix_pktlen(enp, buffer, lengthp)); 1008 } 1009 1010 #if EFSYS_OPT_RX_SCALE 1011 __checkReturn uint32_t 1012 efx_pseudo_hdr_hash_get( 1013 __in efx_rxq_t *erp, 1014 __in efx_rx_hash_alg_t func, 1015 __in uint8_t *buffer) 1016 { 1017 efx_nic_t *enp = erp->er_enp; 1018 const efx_rx_ops_t *erxop = enp->en_erxop; 1019 1020 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1021 1022 EFSYS_ASSERT3U(enp->en_hash_support, ==, EFX_RX_HASH_AVAILABLE); 1023 return (erxop->erxo_prefix_hash(enp, func, buffer)); 1024 } 1025 #endif /* EFSYS_OPT_RX_SCALE */ 1026 1027 #if EFSYS_OPT_SIENA 1028 1029 static __checkReturn efx_rc_t 1030 siena_rx_init( 1031 __in efx_nic_t *enp) 1032 { 1033 efx_oword_t oword; 1034 unsigned int index; 1035 1036 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 1037 1038 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_DESC_PUSH_EN, 0); 1039 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); 1040 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); 1041 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); 1042 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, 0); 1043 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, 0x3000 / 32); 1044 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 1045 1046 /* Zero the RSS table */ 1047 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; 1048 index++) { 1049 EFX_ZERO_OWORD(oword); 1050 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1051 index, &oword, B_TRUE); 1052 } 1053 1054 #if EFSYS_OPT_RX_SCALE 1055 /* The RSS key and indirection table are writable. */ 1056 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 1057 1058 /* Hardware can insert RX hash with/without RSS */ 1059 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 1060 #endif /* EFSYS_OPT_RX_SCALE */ 1061 1062 return (0); 1063 } 1064 1065 #if EFSYS_OPT_RX_SCATTER 1066 static __checkReturn efx_rc_t 1067 siena_rx_scatter_enable( 1068 __in efx_nic_t *enp, 1069 __in unsigned int buf_size) 1070 { 1071 unsigned int nbuf32; 1072 efx_oword_t oword; 1073 efx_rc_t rc; 1074 1075 nbuf32 = buf_size / 32; 1076 if ((nbuf32 == 0) || 1077 (nbuf32 >= (1 << FRF_BZ_RX_USR_BUF_SIZE_WIDTH)) || 1078 ((buf_size % 32) != 0)) { 1079 rc = EINVAL; 1080 goto fail1; 1081 } 1082 1083 if (enp->en_rx_qcount > 0) { 1084 rc = EBUSY; 1085 goto fail2; 1086 } 1087 1088 /* Set scatter buffer size */ 1089 EFX_BAR_READO(enp, FR_AZ_RX_CFG_REG, &oword); 1090 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_USR_BUF_SIZE, nbuf32); 1091 EFX_BAR_WRITEO(enp, FR_AZ_RX_CFG_REG, &oword); 1092 1093 /* Enable scatter for packets not matching a filter */ 1094 EFX_BAR_READO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 1095 EFX_SET_OWORD_FIELD(oword, FRF_BZ_SCATTER_ENBL_NO_MATCH_Q, 1); 1096 EFX_BAR_WRITEO(enp, FR_AZ_RX_FILTER_CTL_REG, &oword); 1097 1098 return (0); 1099 1100 fail2: 1101 EFSYS_PROBE(fail2); 1102 fail1: 1103 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1104 1105 return (rc); 1106 } 1107 #endif /* EFSYS_OPT_RX_SCATTER */ 1108 1109 1110 #define EFX_RX_LFSR_HASH(_enp, _insert) \ 1111 do { \ 1112 efx_oword_t oword; \ 1113 \ 1114 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1115 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 0); \ 1116 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, 0); \ 1117 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, 0); \ 1118 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 1119 (_insert) ? 1 : 0); \ 1120 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1121 \ 1122 if ((_enp)->en_family == EFX_FAMILY_SIENA) { \ 1123 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 1124 &oword); \ 1125 EFX_SET_OWORD_FIELD(oword, \ 1126 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 0); \ 1127 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, \ 1128 &oword); \ 1129 } \ 1130 \ 1131 _NOTE(CONSTANTCONDITION) \ 1132 } while (B_FALSE) 1133 1134 #define EFX_RX_TOEPLITZ_IPV4_HASH(_enp, _insert, _ip, _tcp) \ 1135 do { \ 1136 efx_oword_t oword; \ 1137 \ 1138 EFX_BAR_READO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1139 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_ALG, 1); \ 1140 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_IP_HASH, \ 1141 (_ip) ? 1 : 0); \ 1142 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_TCP_SUP, \ 1143 (_tcp) ? 0 : 1); \ 1144 EFX_SET_OWORD_FIELD(oword, FRF_BZ_RX_HASH_INSRT_HDR, \ 1145 (_insert) ? 1 : 0); \ 1146 EFX_BAR_WRITEO((_enp), FR_AZ_RX_CFG_REG, &oword); \ 1147 \ 1148 _NOTE(CONSTANTCONDITION) \ 1149 } while (B_FALSE) 1150 1151 #define EFX_RX_TOEPLITZ_IPV6_HASH(_enp, _ip, _tcp, _rc) \ 1152 do { \ 1153 efx_oword_t oword; \ 1154 \ 1155 EFX_BAR_READO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 1156 EFX_SET_OWORD_FIELD(oword, \ 1157 FRF_CZ_RX_RSS_IPV6_THASH_ENABLE, 1); \ 1158 EFX_SET_OWORD_FIELD(oword, \ 1159 FRF_CZ_RX_RSS_IPV6_IP_THASH_ENABLE, (_ip) ? 1 : 0); \ 1160 EFX_SET_OWORD_FIELD(oword, \ 1161 FRF_CZ_RX_RSS_IPV6_TCP_SUPPRESS, (_tcp) ? 0 : 1); \ 1162 EFX_BAR_WRITEO((_enp), FR_CZ_RX_RSS_IPV6_REG3, &oword); \ 1163 \ 1164 (_rc) = 0; \ 1165 \ 1166 _NOTE(CONSTANTCONDITION) \ 1167 } while (B_FALSE) 1168 1169 1170 #if EFSYS_OPT_RX_SCALE 1171 1172 static __checkReturn efx_rc_t 1173 siena_rx_scale_mode_set( 1174 __in efx_nic_t *enp, 1175 __in uint32_t rss_context, 1176 __in efx_rx_hash_alg_t alg, 1177 __in efx_rx_hash_type_t type, 1178 __in boolean_t insert) 1179 { 1180 efx_rc_t rc; 1181 1182 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1183 rc = EINVAL; 1184 goto fail1; 1185 } 1186 1187 switch (alg) { 1188 case EFX_RX_HASHALG_LFSR: 1189 EFX_RX_LFSR_HASH(enp, insert); 1190 break; 1191 1192 case EFX_RX_HASHALG_TOEPLITZ: 1193 EFX_RX_TOEPLITZ_IPV4_HASH(enp, insert, 1194 (type & EFX_RX_HASH_IPV4) ? B_TRUE : B_FALSE, 1195 (type & EFX_RX_HASH_TCPIPV4) ? B_TRUE : B_FALSE); 1196 1197 EFX_RX_TOEPLITZ_IPV6_HASH(enp, 1198 (type & EFX_RX_HASH_IPV6) ? B_TRUE : B_FALSE, 1199 (type & EFX_RX_HASH_TCPIPV6) ? B_TRUE : B_FALSE, 1200 rc); 1201 if (rc != 0) 1202 goto fail2; 1203 1204 break; 1205 1206 default: 1207 rc = EINVAL; 1208 goto fail3; 1209 } 1210 1211 return (0); 1212 1213 fail3: 1214 EFSYS_PROBE(fail3); 1215 fail2: 1216 EFSYS_PROBE(fail2); 1217 fail1: 1218 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1219 1220 EFX_RX_LFSR_HASH(enp, B_FALSE); 1221 1222 return (rc); 1223 } 1224 #endif 1225 1226 #if EFSYS_OPT_RX_SCALE 1227 static __checkReturn efx_rc_t 1228 siena_rx_scale_key_set( 1229 __in efx_nic_t *enp, 1230 __in uint32_t rss_context, 1231 __in_ecount(n) uint8_t *key, 1232 __in size_t n) 1233 { 1234 efx_oword_t oword; 1235 unsigned int byte; 1236 unsigned int offset; 1237 efx_rc_t rc; 1238 1239 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1240 rc = EINVAL; 1241 goto fail1; 1242 } 1243 1244 byte = 0; 1245 1246 /* Write Toeplitz IPv4 hash key */ 1247 EFX_ZERO_OWORD(oword); 1248 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 1249 offset > 0 && byte < n; 1250 --offset) 1251 oword.eo_u8[offset - 1] = key[byte++]; 1252 1253 EFX_BAR_WRITEO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 1254 1255 byte = 0; 1256 1257 /* Verify Toeplitz IPv4 hash key */ 1258 EFX_BAR_READO(enp, FR_BZ_RX_RSS_TKEY_REG, &oword); 1259 for (offset = (FRF_BZ_RX_RSS_TKEY_LBN + FRF_BZ_RX_RSS_TKEY_WIDTH) / 8; 1260 offset > 0 && byte < n; 1261 --offset) { 1262 if (oword.eo_u8[offset - 1] != key[byte++]) { 1263 rc = EFAULT; 1264 goto fail2; 1265 } 1266 } 1267 1268 if ((enp->en_features & EFX_FEATURE_IPV6) == 0) 1269 goto done; 1270 1271 byte = 0; 1272 1273 /* Write Toeplitz IPv6 hash key 3 */ 1274 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1275 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1276 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1277 offset > 0 && byte < n; 1278 --offset) 1279 oword.eo_u8[offset - 1] = key[byte++]; 1280 1281 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1282 1283 /* Write Toeplitz IPv6 hash key 2 */ 1284 EFX_ZERO_OWORD(oword); 1285 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1286 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1287 offset > 0 && byte < n; 1288 --offset) 1289 oword.eo_u8[offset - 1] = key[byte++]; 1290 1291 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1292 1293 /* Write Toeplitz IPv6 hash key 1 */ 1294 EFX_ZERO_OWORD(oword); 1295 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1296 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1297 offset > 0 && byte < n; 1298 --offset) 1299 oword.eo_u8[offset - 1] = key[byte++]; 1300 1301 EFX_BAR_WRITEO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1302 1303 byte = 0; 1304 1305 /* Verify Toeplitz IPv6 hash key 3 */ 1306 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG3, &oword); 1307 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_HI_LBN + 1308 FRF_CZ_RX_RSS_IPV6_TKEY_HI_WIDTH) / 8; 1309 offset > 0 && byte < n; 1310 --offset) { 1311 if (oword.eo_u8[offset - 1] != key[byte++]) { 1312 rc = EFAULT; 1313 goto fail3; 1314 } 1315 } 1316 1317 /* Verify Toeplitz IPv6 hash key 2 */ 1318 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG2, &oword); 1319 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_MID_LBN + 1320 FRF_CZ_RX_RSS_IPV6_TKEY_MID_WIDTH) / 8; 1321 offset > 0 && byte < n; 1322 --offset) { 1323 if (oword.eo_u8[offset - 1] != key[byte++]) { 1324 rc = EFAULT; 1325 goto fail4; 1326 } 1327 } 1328 1329 /* Verify Toeplitz IPv6 hash key 1 */ 1330 EFX_BAR_READO(enp, FR_CZ_RX_RSS_IPV6_REG1, &oword); 1331 for (offset = (FRF_CZ_RX_RSS_IPV6_TKEY_LO_LBN + 1332 FRF_CZ_RX_RSS_IPV6_TKEY_LO_WIDTH) / 8; 1333 offset > 0 && byte < n; 1334 --offset) { 1335 if (oword.eo_u8[offset - 1] != key[byte++]) { 1336 rc = EFAULT; 1337 goto fail5; 1338 } 1339 } 1340 1341 done: 1342 return (0); 1343 1344 fail5: 1345 EFSYS_PROBE(fail5); 1346 fail4: 1347 EFSYS_PROBE(fail4); 1348 fail3: 1349 EFSYS_PROBE(fail3); 1350 fail2: 1351 EFSYS_PROBE(fail2); 1352 fail1: 1353 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1354 1355 return (rc); 1356 } 1357 #endif 1358 1359 #if EFSYS_OPT_RX_SCALE 1360 static __checkReturn efx_rc_t 1361 siena_rx_scale_tbl_set( 1362 __in efx_nic_t *enp, 1363 __in uint32_t rss_context, 1364 __in_ecount(n) unsigned int *table, 1365 __in size_t n) 1366 { 1367 efx_oword_t oword; 1368 int index; 1369 efx_rc_t rc; 1370 1371 EFX_STATIC_ASSERT(EFX_RSS_TBL_SIZE == FR_BZ_RX_INDIRECTION_TBL_ROWS); 1372 EFX_STATIC_ASSERT(EFX_MAXRSS == (1 << FRF_BZ_IT_QUEUE_WIDTH)); 1373 1374 if (rss_context != EFX_RSS_CONTEXT_DEFAULT) { 1375 rc = EINVAL; 1376 goto fail1; 1377 } 1378 1379 if (n > FR_BZ_RX_INDIRECTION_TBL_ROWS) { 1380 rc = EINVAL; 1381 goto fail2; 1382 } 1383 1384 for (index = 0; index < FR_BZ_RX_INDIRECTION_TBL_ROWS; index++) { 1385 uint32_t byte; 1386 1387 /* Calculate the entry to place in the table */ 1388 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1389 1390 EFSYS_PROBE2(table, int, index, uint32_t, byte); 1391 1392 EFX_POPULATE_OWORD_1(oword, FRF_BZ_IT_QUEUE, byte); 1393 1394 /* Write the table */ 1395 EFX_BAR_TBL_WRITEO(enp, FR_BZ_RX_INDIRECTION_TBL, 1396 index, &oword, B_TRUE); 1397 } 1398 1399 for (index = FR_BZ_RX_INDIRECTION_TBL_ROWS - 1; index >= 0; --index) { 1400 uint32_t byte; 1401 1402 /* Determine if we're starting a new batch */ 1403 byte = (n > 0) ? (uint32_t)table[index % n] : 0; 1404 1405 /* Read the table */ 1406 EFX_BAR_TBL_READO(enp, FR_BZ_RX_INDIRECTION_TBL, 1407 index, &oword, B_TRUE); 1408 1409 /* Verify the entry */ 1410 if (EFX_OWORD_FIELD(oword, FRF_BZ_IT_QUEUE) != byte) { 1411 rc = EFAULT; 1412 goto fail3; 1413 } 1414 } 1415 1416 return (0); 1417 1418 fail3: 1419 EFSYS_PROBE(fail3); 1420 fail2: 1421 EFSYS_PROBE(fail2); 1422 fail1: 1423 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1424 1425 return (rc); 1426 } 1427 #endif 1428 1429 /* 1430 * Falcon/Siena pseudo-header 1431 * -------------------------- 1432 * 1433 * Receive packets are prefixed by an optional 16 byte pseudo-header. 1434 * The pseudo-header is a byte array of one of the forms: 1435 * 1436 * 0 1 2 3 4 5 6 7 8 9 10 11 12 13 14 15 1437 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.TT.TT.TT.TT 1438 * xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.xx.LL.LL 1439 * 1440 * where: 1441 * TT.TT.TT.TT Toeplitz hash (32-bit big-endian) 1442 * LL.LL LFSR hash (16-bit big-endian) 1443 */ 1444 1445 #if EFSYS_OPT_RX_SCALE 1446 static __checkReturn uint32_t 1447 siena_rx_prefix_hash( 1448 __in efx_nic_t *enp, 1449 __in efx_rx_hash_alg_t func, 1450 __in uint8_t *buffer) 1451 { 1452 _NOTE(ARGUNUSED(enp)) 1453 1454 switch (func) { 1455 case EFX_RX_HASHALG_TOEPLITZ: 1456 return ((buffer[12] << 24) | 1457 (buffer[13] << 16) | 1458 (buffer[14] << 8) | 1459 buffer[15]); 1460 1461 case EFX_RX_HASHALG_LFSR: 1462 return ((buffer[14] << 8) | buffer[15]); 1463 1464 default: 1465 EFSYS_ASSERT(0); 1466 return (0); 1467 } 1468 } 1469 #endif /* EFSYS_OPT_RX_SCALE */ 1470 1471 static __checkReturn efx_rc_t 1472 siena_rx_prefix_pktlen( 1473 __in efx_nic_t *enp, 1474 __in uint8_t *buffer, 1475 __out uint16_t *lengthp) 1476 { 1477 _NOTE(ARGUNUSED(enp, buffer, lengthp)) 1478 1479 /* Not supported by Falcon/Siena hardware */ 1480 EFSYS_ASSERT(0); 1481 return (ENOTSUP); 1482 } 1483 1484 1485 static void 1486 siena_rx_qpost( 1487 __in efx_rxq_t *erp, 1488 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 1489 __in size_t size, 1490 __in unsigned int ndescs, 1491 __in unsigned int completed, 1492 __in unsigned int added) 1493 { 1494 efx_qword_t qword; 1495 unsigned int i; 1496 unsigned int offset; 1497 unsigned int id; 1498 1499 /* The client driver must not overfill the queue */ 1500 EFSYS_ASSERT3U(added - completed + ndescs, <=, 1501 EFX_RXQ_LIMIT(erp->er_mask + 1)); 1502 1503 id = added & (erp->er_mask); 1504 for (i = 0; i < ndescs; i++) { 1505 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 1506 unsigned int, id, efsys_dma_addr_t, addrp[i], 1507 size_t, size); 1508 1509 EFX_POPULATE_QWORD_3(qword, 1510 FSF_AZ_RX_KER_BUF_SIZE, (uint32_t)(size), 1511 FSF_AZ_RX_KER_BUF_ADDR_DW0, 1512 (uint32_t)(addrp[i] & 0xffffffff), 1513 FSF_AZ_RX_KER_BUF_ADDR_DW1, 1514 (uint32_t)(addrp[i] >> 32)); 1515 1516 offset = id * sizeof (efx_qword_t); 1517 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 1518 1519 id = (id + 1) & (erp->er_mask); 1520 } 1521 } 1522 1523 static void 1524 siena_rx_qpush( 1525 __in efx_rxq_t *erp, 1526 __in unsigned int added, 1527 __inout unsigned int *pushedp) 1528 { 1529 efx_nic_t *enp = erp->er_enp; 1530 unsigned int pushed = *pushedp; 1531 uint32_t wptr; 1532 efx_oword_t oword; 1533 efx_dword_t dword; 1534 1535 /* All descriptors are pushed */ 1536 *pushedp = added; 1537 1538 /* Push the populated descriptors out */ 1539 wptr = added & erp->er_mask; 1540 1541 EFX_POPULATE_OWORD_1(oword, FRF_AZ_RX_DESC_WPTR, wptr); 1542 1543 /* Only write the third DWORD */ 1544 EFX_POPULATE_DWORD_1(dword, 1545 EFX_DWORD_0, EFX_OWORD_FIELD(oword, EFX_DWORD_3)); 1546 1547 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 1548 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 1549 wptr, pushed & erp->er_mask); 1550 EFSYS_PIO_WRITE_BARRIER(); 1551 EFX_BAR_TBL_WRITED3(enp, FR_BZ_RX_DESC_UPD_REGP0, 1552 erp->er_index, &dword, B_FALSE); 1553 } 1554 1555 #if EFSYS_OPT_RX_PACKED_STREAM 1556 static void 1557 siena_rx_qpush_ps_credits( 1558 __in efx_rxq_t *erp) 1559 { 1560 /* Not supported by Siena hardware */ 1561 EFSYS_ASSERT(0); 1562 } 1563 1564 static uint8_t * 1565 siena_rx_qps_packet_info( 1566 __in efx_rxq_t *erp, 1567 __in uint8_t *buffer, 1568 __in uint32_t buffer_length, 1569 __in uint32_t current_offset, 1570 __out uint16_t *lengthp, 1571 __out uint32_t *next_offsetp, 1572 __out uint32_t *timestamp) 1573 { 1574 /* Not supported by Siena hardware */ 1575 EFSYS_ASSERT(0); 1576 1577 return (NULL); 1578 } 1579 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1580 1581 static __checkReturn efx_rc_t 1582 siena_rx_qflush( 1583 __in efx_rxq_t *erp) 1584 { 1585 efx_nic_t *enp = erp->er_enp; 1586 efx_oword_t oword; 1587 uint32_t label; 1588 1589 label = erp->er_index; 1590 1591 /* Flush the queue */ 1592 EFX_POPULATE_OWORD_2(oword, FRF_AZ_RX_FLUSH_DESCQ_CMD, 1, 1593 FRF_AZ_RX_FLUSH_DESCQ, label); 1594 EFX_BAR_WRITEO(enp, FR_AZ_RX_FLUSH_DESCQ_REG, &oword); 1595 1596 return (0); 1597 } 1598 1599 static void 1600 siena_rx_qenable( 1601 __in efx_rxq_t *erp) 1602 { 1603 efx_nic_t *enp = erp->er_enp; 1604 efx_oword_t oword; 1605 1606 EFSYS_ASSERT3U(erp->er_magic, ==, EFX_RXQ_MAGIC); 1607 1608 EFX_BAR_TBL_READO(enp, FR_AZ_RX_DESC_PTR_TBL, 1609 erp->er_index, &oword, B_TRUE); 1610 1611 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DC_HW_RPTR, 0); 1612 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_HW_RPTR, 0); 1613 EFX_SET_OWORD_FIELD(oword, FRF_AZ_RX_DESCQ_EN, 1); 1614 1615 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1616 erp->er_index, &oword, B_TRUE); 1617 } 1618 1619 static __checkReturn efx_rc_t 1620 siena_rx_qcreate( 1621 __in efx_nic_t *enp, 1622 __in unsigned int index, 1623 __in unsigned int label, 1624 __in efx_rxq_type_t type, 1625 __in_opt const efx_rxq_type_data_t *type_data, 1626 __in efsys_mem_t *esmp, 1627 __in size_t ndescs, 1628 __in uint32_t id, 1629 __in unsigned int flags, 1630 __in efx_evq_t *eep, 1631 __in efx_rxq_t *erp) 1632 { 1633 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1634 efx_oword_t oword; 1635 uint32_t size; 1636 boolean_t jumbo = B_FALSE; 1637 efx_rc_t rc; 1638 1639 _NOTE(ARGUNUSED(esmp)) 1640 1641 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == 1642 (1 << FRF_AZ_RX_DESCQ_LABEL_WIDTH)); 1643 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1644 1645 for (size = 0; 1646 (1U << size) <= encp->enc_rxq_max_ndescs / encp->enc_rxq_min_ndescs; 1647 size++) 1648 if ((1U << size) == (uint32_t)ndescs / encp->enc_rxq_min_ndescs) 1649 break; 1650 if (id + (1 << size) >= encp->enc_buftbl_limit) { 1651 rc = EINVAL; 1652 goto fail1; 1653 } 1654 1655 switch (type) { 1656 case EFX_RXQ_TYPE_DEFAULT: 1657 erp->er_buf_size = type_data->ertd_default.ed_buf_size; 1658 break; 1659 1660 default: 1661 rc = EINVAL; 1662 goto fail2; 1663 } 1664 1665 if (flags & EFX_RXQ_FLAG_SCATTER) { 1666 #if EFSYS_OPT_RX_SCATTER 1667 jumbo = B_TRUE; 1668 #else 1669 rc = EINVAL; 1670 goto fail3; 1671 #endif /* EFSYS_OPT_RX_SCATTER */ 1672 } 1673 1674 /* Set up the new descriptor queue */ 1675 EFX_POPULATE_OWORD_7(oword, 1676 FRF_AZ_RX_DESCQ_BUF_BASE_ID, id, 1677 FRF_AZ_RX_DESCQ_EVQ_ID, eep->ee_index, 1678 FRF_AZ_RX_DESCQ_OWNER_ID, 0, 1679 FRF_AZ_RX_DESCQ_LABEL, label, 1680 FRF_AZ_RX_DESCQ_SIZE, size, 1681 FRF_AZ_RX_DESCQ_TYPE, 0, 1682 FRF_AZ_RX_DESCQ_JUMBO, jumbo); 1683 1684 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1685 erp->er_index, &oword, B_TRUE); 1686 1687 return (0); 1688 1689 #if !EFSYS_OPT_RX_SCATTER 1690 fail3: 1691 EFSYS_PROBE(fail3); 1692 #endif 1693 fail2: 1694 EFSYS_PROBE(fail2); 1695 fail1: 1696 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1697 1698 return (rc); 1699 } 1700 1701 static void 1702 siena_rx_qdestroy( 1703 __in efx_rxq_t *erp) 1704 { 1705 efx_nic_t *enp = erp->er_enp; 1706 efx_oword_t oword; 1707 1708 /* Purge descriptor queue */ 1709 EFX_ZERO_OWORD(oword); 1710 1711 EFX_BAR_TBL_WRITEO(enp, FR_AZ_RX_DESC_PTR_TBL, 1712 erp->er_index, &oword, B_TRUE); 1713 } 1714 1715 static void 1716 siena_rx_fini( 1717 __in efx_nic_t *enp) 1718 { 1719 _NOTE(ARGUNUSED(enp)) 1720 } 1721 1722 #endif /* EFSYS_OPT_SIENA */ 1723