1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2012-2019 Solarflare Communications Inc. 5 */ 6 7 #include "efx.h" 8 #include "efx_impl.h" 9 10 11 #if EFSYS_OPT_RIVERHEAD || EFX_OPTS_EF10() 12 13 #if EFSYS_OPT_RX_SCALE 14 static __checkReturn efx_rc_t 15 efx_mcdi_rss_context_alloc( 16 __in efx_nic_t *enp, 17 __in efx_rx_scale_context_type_t type, 18 __in uint32_t num_queues, 19 __in uint32_t table_nentries, 20 __out uint32_t *rss_contextp) 21 { 22 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 23 efx_mcdi_req_t req; 24 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_V2_IN_LEN, 25 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN); 26 uint32_t table_nentries_min; 27 uint32_t table_nentries_max; 28 uint32_t num_queues_max; 29 uint32_t rss_context; 30 uint32_t context_type; 31 efx_rc_t rc; 32 33 switch (type) { 34 case EFX_RX_SCALE_EXCLUSIVE: 35 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE; 36 num_queues_max = encp->enc_rx_scale_indirection_max_nqueues; 37 table_nentries_min = encp->enc_rx_scale_tbl_min_nentries; 38 table_nentries_max = encp->enc_rx_scale_tbl_max_nentries; 39 break; 40 case EFX_RX_SCALE_SHARED: 41 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; 42 num_queues_max = encp->enc_rx_scale_indirection_max_nqueues; 43 table_nentries_min = encp->enc_rx_scale_tbl_min_nentries; 44 table_nentries_max = encp->enc_rx_scale_tbl_max_nentries; 45 break; 46 case EFX_RX_SCALE_EVEN_SPREAD: 47 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EVEN_SPREADING; 48 num_queues_max = encp->enc_rx_scale_even_spread_max_nqueues; 49 table_nentries_min = 0; 50 table_nentries_max = 0; 51 break; 52 default: 53 rc = EINVAL; 54 goto fail1; 55 } 56 57 if (num_queues == 0 || num_queues > num_queues_max) { 58 rc = EINVAL; 59 goto fail2; 60 } 61 62 if (table_nentries < table_nentries_min || 63 table_nentries > table_nentries_max || 64 (table_nentries != 0 && !ISP2(table_nentries))) { 65 rc = EINVAL; 66 goto fail3; 67 } 68 69 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC; 70 req.emr_in_buf = payload; 71 req.emr_in_length = 72 (encp->enc_rx_scale_tbl_entry_count_is_selectable != B_FALSE) ? 73 MC_CMD_RSS_CONTEXT_ALLOC_V2_IN_LEN : 74 MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN; 75 req.emr_out_buf = payload; 76 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN; 77 78 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, 79 enp->en_vport_id); 80 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type); 81 82 /* 83 * For exclusive contexts, NUM_QUEUES is only used to validate 84 * indirection table offsets. 85 * For shared contexts, the provided context will spread traffic over 86 * NUM_QUEUES many queues. 87 * For the even spread contexts, the provided context will spread 88 * traffic over NUM_QUEUES many queues, but that will not involve 89 * the use of precious indirection table resources in the adapter. 90 */ 91 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues); 92 93 if (encp->enc_rx_scale_tbl_entry_count_is_selectable != B_FALSE) { 94 MCDI_IN_SET_DWORD(req, 95 RSS_CONTEXT_ALLOC_V2_IN_INDIRECTION_TABLE_SIZE, 96 table_nentries); 97 } 98 99 efx_mcdi_execute(enp, &req); 100 101 if (req.emr_rc != 0) { 102 rc = req.emr_rc; 103 goto fail4; 104 } 105 106 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) { 107 rc = EMSGSIZE; 108 goto fail5; 109 } 110 111 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); 112 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 113 rc = ENOENT; 114 goto fail6; 115 } 116 117 *rss_contextp = rss_context; 118 119 return (0); 120 121 fail6: 122 EFSYS_PROBE(fail6); 123 fail5: 124 EFSYS_PROBE(fail5); 125 fail4: 126 EFSYS_PROBE(fail4); 127 fail3: 128 EFSYS_PROBE(fail3); 129 fail2: 130 EFSYS_PROBE(fail2); 131 fail1: 132 EFSYS_PROBE1(fail1, efx_rc_t, rc); 133 134 return (rc); 135 } 136 #endif /* EFSYS_OPT_RX_SCALE */ 137 138 #if EFSYS_OPT_RX_SCALE 139 static efx_rc_t 140 efx_mcdi_rss_context_free( 141 __in efx_nic_t *enp, 142 __in uint32_t rss_context) 143 { 144 efx_mcdi_req_t req; 145 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN, 146 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN); 147 efx_rc_t rc; 148 149 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 150 rc = EINVAL; 151 goto fail1; 152 } 153 154 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE; 155 req.emr_in_buf = payload; 156 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN; 157 req.emr_out_buf = payload; 158 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN; 159 160 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context); 161 162 efx_mcdi_execute_quiet(enp, &req); 163 164 if (req.emr_rc != 0) { 165 rc = req.emr_rc; 166 goto fail2; 167 } 168 169 return (0); 170 171 fail2: 172 EFSYS_PROBE(fail2); 173 fail1: 174 EFSYS_PROBE1(fail1, efx_rc_t, rc); 175 176 return (rc); 177 } 178 #endif /* EFSYS_OPT_RX_SCALE */ 179 180 #if EFSYS_OPT_RX_SCALE 181 static efx_rc_t 182 efx_mcdi_rss_context_set_flags( 183 __in efx_nic_t *enp, 184 __in uint32_t rss_context, 185 __in efx_rx_hash_type_t type) 186 { 187 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 188 efx_mcdi_req_t req; 189 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN, 190 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN); 191 efx_rc_t rc; 192 193 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_LBN == 194 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_LBN); 195 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_WIDTH == 196 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_WIDTH); 197 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_LBN == 198 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_LBN); 199 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_WIDTH == 200 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_WIDTH); 201 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_LBN == 202 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_LBN); 203 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_WIDTH == 204 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_WIDTH); 205 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_LBN == 206 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_LBN); 207 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_WIDTH == 208 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_WIDTH); 209 210 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 211 rc = EINVAL; 212 goto fail1; 213 } 214 215 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS; 216 req.emr_in_buf = payload; 217 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN; 218 req.emr_out_buf = payload; 219 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN; 220 221 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, 222 rss_context); 223 224 /* 225 * If the firmware lacks support for additional modes, RSS_MODE 226 * fields must contain zeros, otherwise the operation will fail. 227 */ 228 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE) 229 type &= EFX_RX_HASH_LEGACY_MASK; 230 231 MCDI_IN_POPULATE_DWORD_10(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, 232 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN, 233 (type & EFX_RX_HASH_IPV4) ? 1 : 0, 234 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN, 235 (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0, 236 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN, 237 (type & EFX_RX_HASH_IPV6) ? 1 : 0, 238 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN, 239 (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0, 240 RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE, 241 (type >> EFX_RX_CLASS_IPV4_TCP_LBN) & 242 EFX_MASK32(EFX_RX_CLASS_IPV4_TCP), 243 RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE, 244 (type >> EFX_RX_CLASS_IPV4_UDP_LBN) & 245 EFX_MASK32(EFX_RX_CLASS_IPV4_UDP), 246 RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE, 247 (type >> EFX_RX_CLASS_IPV4_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV4), 248 RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE, 249 (type >> EFX_RX_CLASS_IPV6_TCP_LBN) & 250 EFX_MASK32(EFX_RX_CLASS_IPV6_TCP), 251 RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE, 252 (type >> EFX_RX_CLASS_IPV6_UDP_LBN) & 253 EFX_MASK32(EFX_RX_CLASS_IPV6_UDP), 254 RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE, 255 (type >> EFX_RX_CLASS_IPV6_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV6)); 256 257 efx_mcdi_execute(enp, &req); 258 259 if (req.emr_rc != 0) { 260 rc = req.emr_rc; 261 goto fail2; 262 } 263 264 return (0); 265 266 fail2: 267 EFSYS_PROBE(fail2); 268 fail1: 269 EFSYS_PROBE1(fail1, efx_rc_t, rc); 270 271 return (rc); 272 } 273 #endif /* EFSYS_OPT_RX_SCALE */ 274 275 #if EFSYS_OPT_RX_SCALE 276 static efx_rc_t 277 efx_mcdi_rss_context_set_key( 278 __in efx_nic_t *enp, 279 __in uint32_t rss_context, 280 __in_ecount(n) uint8_t *key, 281 __in size_t n) 282 { 283 efx_mcdi_req_t req; 284 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN, 285 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN); 286 efx_rc_t rc; 287 288 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 289 rc = EINVAL; 290 goto fail1; 291 } 292 293 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY; 294 req.emr_in_buf = payload; 295 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN; 296 req.emr_out_buf = payload; 297 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN; 298 299 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, 300 rss_context); 301 302 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 303 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) { 304 rc = EINVAL; 305 goto fail2; 306 } 307 308 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), 309 key, n); 310 311 efx_mcdi_execute(enp, &req); 312 313 if (req.emr_rc != 0) { 314 rc = req.emr_rc; 315 goto fail3; 316 } 317 318 return (0); 319 320 fail3: 321 EFSYS_PROBE(fail3); 322 fail2: 323 EFSYS_PROBE(fail2); 324 fail1: 325 EFSYS_PROBE1(fail1, efx_rc_t, rc); 326 327 return (rc); 328 } 329 #endif /* EFSYS_OPT_RX_SCALE */ 330 331 #if EFSYS_OPT_RX_SCALE 332 static efx_rc_t 333 efx_mcdi_rss_context_set_table( 334 __in efx_nic_t *enp, 335 __in uint32_t rss_context, 336 __in_ecount(nentries) unsigned int *table, 337 __in size_t nentries) 338 { 339 efx_mcdi_req_t req; 340 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN, 341 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN); 342 uint8_t *req_table; 343 int i, rc; 344 345 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 346 rc = EINVAL; 347 goto fail1; 348 } 349 350 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE; 351 req.emr_in_buf = payload; 352 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN; 353 req.emr_out_buf = payload; 354 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN; 355 356 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, 357 rss_context); 358 359 req_table = 360 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE); 361 362 for (i = 0; 363 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN; 364 i++) { 365 req_table[i] = (nentries > 0) ? 366 (uint8_t)table[i % nentries] : 0; 367 } 368 369 efx_mcdi_execute(enp, &req); 370 371 if (req.emr_rc != 0) { 372 rc = req.emr_rc; 373 goto fail2; 374 } 375 376 return (0); 377 378 fail2: 379 EFSYS_PROBE(fail2); 380 fail1: 381 EFSYS_PROBE1(fail1, efx_rc_t, rc); 382 383 return (rc); 384 } 385 #endif /* EFSYS_OPT_RX_SCALE */ 386 387 #if EFSYS_OPT_RX_SCALE 388 static __checkReturn efx_rc_t 389 efx_mcdi_rss_context_write_table( 390 __in efx_nic_t *enp, 391 __in uint32_t context, 392 __in unsigned int start_idx, 393 __in_ecount(nentries) unsigned int *table, 394 __in unsigned int nentries) 395 { 396 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 397 efx_mcdi_req_t req; 398 EFX_MCDI_DECLARE_BUF(payload, 399 MC_CMD_RSS_CONTEXT_WRITE_TABLE_IN_LENMAX_MCDI2, 400 MC_CMD_RSS_CONTEXT_WRITE_TABLE_OUT_LEN); 401 unsigned int i; 402 int rc; 403 404 if (nentries > 405 MC_CMD_RSS_CONTEXT_WRITE_TABLE_IN_ENTRIES_MAXNUM_MCDI2) { 406 rc = EINVAL; 407 goto fail1; 408 } 409 410 if (start_idx + nentries > 411 encp->enc_rx_scale_tbl_max_nentries) { 412 rc = EINVAL; 413 goto fail2; 414 } 415 416 req.emr_cmd = MC_CMD_RSS_CONTEXT_WRITE_TABLE; 417 req.emr_in_buf = payload; 418 req.emr_in_length = MC_CMD_RSS_CONTEXT_WRITE_TABLE_IN_LEN(nentries); 419 req.emr_out_buf = payload; 420 req.emr_out_length = MC_CMD_RSS_CONTEXT_WRITE_TABLE_OUT_LEN; 421 422 MCDI_IN_SET_DWORD(req, 423 RSS_CONTEXT_WRITE_TABLE_IN_RSS_CONTEXT_ID, context); 424 425 for (i = 0; i < nentries; ++i) { 426 if (table[i] >= encp->enc_rx_scale_indirection_max_nqueues) { 427 rc = EINVAL; 428 goto fail3; 429 } 430 431 MCDI_IN_POPULATE_INDEXED_DWORD_2(req, 432 RSS_CONTEXT_WRITE_TABLE_IN_ENTRIES, i, 433 RSS_CONTEXT_WRITE_TABLE_ENTRY_INDEX, start_idx + i, 434 RSS_CONTEXT_WRITE_TABLE_ENTRY_VALUE, table[i]); 435 } 436 437 efx_mcdi_execute(enp, &req); 438 if (req.emr_rc != 0) { 439 rc = req.emr_rc; 440 goto fail4; 441 } 442 443 return (0); 444 445 fail4: 446 EFSYS_PROBE(fail4); 447 fail3: 448 EFSYS_PROBE(fail3); 449 fail2: 450 EFSYS_PROBE(fail2); 451 fail1: 452 EFSYS_PROBE1(fail1, efx_rc_t, rc); 453 return (rc); 454 } 455 #endif /* EFSYS_OPT_RX_SCALE */ 456 457 458 __checkReturn efx_rc_t 459 ef10_rx_init( 460 __in efx_nic_t *enp) 461 { 462 #if EFSYS_OPT_RX_SCALE 463 464 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS, 465 EFX_RSS_TBL_SIZE, &enp->en_rss_context) == 0) { 466 /* 467 * Allocated an exclusive RSS context, which allows both the 468 * indirection table and key to be modified. 469 */ 470 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 471 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 472 } else { 473 /* 474 * Failed to allocate an exclusive RSS context. Continue 475 * operation without support for RSS. The pseudo-header in 476 * received packets will not contain a Toeplitz hash value. 477 */ 478 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; 479 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE; 480 } 481 482 #endif /* EFSYS_OPT_RX_SCALE */ 483 484 return (0); 485 } 486 487 #if EFX_OPTS_EF10() 488 489 #if EFSYS_OPT_RX_SCATTER 490 __checkReturn efx_rc_t 491 ef10_rx_scatter_enable( 492 __in efx_nic_t *enp, 493 __in unsigned int buf_size) 494 { 495 _NOTE(ARGUNUSED(enp, buf_size)) 496 return (0); 497 } 498 #endif /* EFSYS_OPT_RX_SCATTER */ 499 500 #endif /* EFX_OPTS_EF10() */ 501 502 #if EFSYS_OPT_RX_SCALE 503 __checkReturn efx_rc_t 504 ef10_rx_scale_context_alloc( 505 __in efx_nic_t *enp, 506 __in efx_rx_scale_context_type_t type, 507 __in uint32_t num_queues, 508 __in uint32_t table_nentries, 509 __out uint32_t *rss_contextp) 510 { 511 efx_rc_t rc; 512 513 rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, table_nentries, 514 rss_contextp); 515 if (rc != 0) 516 goto fail1; 517 518 return (0); 519 520 fail1: 521 EFSYS_PROBE1(fail1, efx_rc_t, rc); 522 return (rc); 523 } 524 #endif /* EFSYS_OPT_RX_SCALE */ 525 526 #if EFSYS_OPT_RX_SCALE 527 __checkReturn efx_rc_t 528 ef10_rx_scale_context_free( 529 __in efx_nic_t *enp, 530 __in uint32_t rss_context) 531 { 532 efx_rc_t rc; 533 534 rc = efx_mcdi_rss_context_free(enp, rss_context); 535 if (rc != 0) 536 goto fail1; 537 538 return (0); 539 540 fail1: 541 EFSYS_PROBE1(fail1, efx_rc_t, rc); 542 return (rc); 543 } 544 #endif /* EFSYS_OPT_RX_SCALE */ 545 546 #if EFSYS_OPT_RX_SCALE 547 __checkReturn efx_rc_t 548 ef10_rx_scale_mode_set( 549 __in efx_nic_t *enp, 550 __in uint32_t rss_context, 551 __in efx_rx_hash_alg_t alg, 552 __in efx_rx_hash_type_t type, 553 __in boolean_t insert) 554 { 555 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 556 efx_rc_t rc; 557 558 EFSYS_ASSERT3U(insert, ==, B_TRUE); 559 560 if ((encp->enc_rx_scale_hash_alg_mask & (1U << alg)) == 0 || 561 insert == B_FALSE) { 562 rc = EINVAL; 563 goto fail1; 564 } 565 566 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 567 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 568 rc = ENOTSUP; 569 goto fail2; 570 } 571 rss_context = enp->en_rss_context; 572 } 573 574 if ((rc = efx_mcdi_rss_context_set_flags(enp, 575 rss_context, type)) != 0) 576 goto fail3; 577 578 return (0); 579 580 fail3: 581 EFSYS_PROBE(fail3); 582 fail2: 583 EFSYS_PROBE(fail2); 584 fail1: 585 EFSYS_PROBE1(fail1, efx_rc_t, rc); 586 587 return (rc); 588 } 589 #endif /* EFSYS_OPT_RX_SCALE */ 590 591 #if EFSYS_OPT_RX_SCALE 592 __checkReturn efx_rc_t 593 ef10_rx_scale_key_set( 594 __in efx_nic_t *enp, 595 __in uint32_t rss_context, 596 __in_ecount(n) uint8_t *key, 597 __in size_t n) 598 { 599 efx_rc_t rc; 600 601 EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE == 602 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 603 604 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 605 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 606 rc = ENOTSUP; 607 goto fail1; 608 } 609 rss_context = enp->en_rss_context; 610 } 611 612 if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0) 613 goto fail2; 614 615 return (0); 616 617 fail2: 618 EFSYS_PROBE(fail2); 619 fail1: 620 EFSYS_PROBE1(fail1, efx_rc_t, rc); 621 622 return (rc); 623 } 624 #endif /* EFSYS_OPT_RX_SCALE */ 625 626 #if EFSYS_OPT_RX_SCALE 627 __checkReturn efx_rc_t 628 ef10_rx_scale_tbl_set( 629 __in efx_nic_t *enp, 630 __in uint32_t rss_context, 631 __in_ecount(nentries) unsigned int *table, 632 __in size_t nentries) 633 { 634 const efx_nic_cfg_t *encp = efx_nic_cfg_get(enp); 635 efx_rc_t rc; 636 637 638 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 639 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 640 rc = ENOTSUP; 641 goto fail1; 642 } 643 rss_context = enp->en_rss_context; 644 } 645 646 if (encp->enc_rx_scale_tbl_entry_count_is_selectable != B_FALSE) { 647 uint32_t index, remain, batch; 648 649 batch = MC_CMD_RSS_CONTEXT_WRITE_TABLE_IN_ENTRIES_MAXNUM_MCDI2; 650 index = 0; 651 652 for (remain = nentries; remain > 0; remain -= batch) { 653 if (batch > remain) 654 batch = remain; 655 656 rc = efx_mcdi_rss_context_write_table(enp, rss_context, 657 index, &table[index], batch); 658 if (rc != 0) 659 goto fail2; 660 661 index += batch; 662 } 663 } else { 664 rc = efx_mcdi_rss_context_set_table(enp, rss_context, table, 665 nentries); 666 if (rc != 0) 667 goto fail3; 668 } 669 670 return (0); 671 672 fail3: 673 EFSYS_PROBE(fail3); 674 fail2: 675 EFSYS_PROBE(fail2); 676 fail1: 677 EFSYS_PROBE1(fail1, efx_rc_t, rc); 678 679 return (rc); 680 } 681 #endif /* EFSYS_OPT_RX_SCALE */ 682 683 #if EFX_OPTS_EF10() 684 685 /* 686 * EF10 RX pseudo-header (aka Rx prefix) 687 * ------------------------------------- 688 * 689 * Receive packets are prefixed by an (optional) 14 byte pseudo-header: 690 * 691 * +00: Toeplitz hash value. 692 * (32bit little-endian) 693 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag. 694 * (16bit big-endian) 695 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag. 696 * (16bit big-endian) 697 * +08: Packet Length. Zero if the RX datapath was in cut-through mode. 698 * (16bit little-endian) 699 * +10: MAC timestamp. Zero if timestamping is not enabled. 700 * (32bit little-endian) 701 * 702 * See "The RX Pseudo-header" in SF-109306-TC. 703 * 704 * EF10 does not support Rx prefix choice using MC_CMD_GET_RX_PREFIX_ID 705 * and query its layout using MC_CMD_QUERY_RX_PREFIX_ID. 706 */ 707 static const efx_rx_prefix_layout_t ef10_default_rx_prefix_layout = { 708 .erpl_id = 0, 709 .erpl_length = 14, 710 .erpl_fields = { 711 [EFX_RX_PREFIX_FIELD_RSS_HASH] = 712 { 0, 32, B_FALSE }, 713 [EFX_RX_PREFIX_FIELD_VLAN_STRIP_TCI] = 714 { 32, 16, B_TRUE }, 715 [EFX_RX_PREFIX_FIELD_INNER_VLAN_STRIP_TCI] = 716 { 48, 16, B_TRUE }, 717 [EFX_RX_PREFIX_FIELD_LENGTH] = 718 { 64, 16, B_FALSE }, 719 [EFX_RX_PREFIX_FIELD_PARTIAL_TSTAMP] = 720 { 80, 32, B_FALSE }, 721 } 722 }; 723 724 #if EFSYS_OPT_RX_PACKED_STREAM 725 726 /* 727 * EF10 packed stream Rx prefix layout. 728 * 729 * See SF-112241-TC Full speed capture for Huntington and Medford section 4.5. 730 */ 731 static const efx_rx_prefix_layout_t ef10_packed_stream_rx_prefix_layout = { 732 .erpl_id = 0, 733 .erpl_length = 8, 734 .erpl_fields = { 735 #define EF10_PS_RX_PREFIX_FIELD(_efx, _ef10) \ 736 EFX_RX_PREFIX_FIELD(_efx, ES_DZ_PS_RX_PREFIX_ ## _ef10, B_FALSE) 737 738 EF10_PS_RX_PREFIX_FIELD(PARTIAL_TSTAMP, TSTAMP), 739 EF10_PS_RX_PREFIX_FIELD(LENGTH, CAP_LEN), 740 EF10_PS_RX_PREFIX_FIELD(ORIG_LENGTH, ORIG_LEN), 741 742 #undef EF10_PS_RX_PREFIX_FIELD 743 } 744 }; 745 746 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 747 748 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 749 750 /* 751 * EF10 equal stride super-buffer Rx prefix layout. 752 * 753 * See SF-119419-TC DPDK Firmware Driver Interface section 3.4. 754 */ 755 static const efx_rx_prefix_layout_t ef10_essb_rx_prefix_layout = { 756 .erpl_id = 0, 757 .erpl_length = ES_EZ_ESSB_RX_PREFIX_LEN, 758 .erpl_fields = { 759 #define EF10_ESSB_RX_PREFIX_FIELD(_efx, _ef10) \ 760 EFX_RX_PREFIX_FIELD(_efx, ES_EZ_ESSB_RX_PREFIX_ ## _ef10, B_FALSE) 761 762 EF10_ESSB_RX_PREFIX_FIELD(LENGTH, DATA_LEN), 763 EF10_ESSB_RX_PREFIX_FIELD(USER_MARK, MARK), 764 EF10_ESSB_RX_PREFIX_FIELD(RSS_HASH_VALID, HASH_VALID), 765 EF10_ESSB_RX_PREFIX_FIELD(USER_MARK_VALID, MARK_VALID), 766 EF10_ESSB_RX_PREFIX_FIELD(USER_FLAG, MATCH_FLAG), 767 EF10_ESSB_RX_PREFIX_FIELD(RSS_HASH, HASH), 768 769 #undef EF10_ESSB_RX_PREFIX_FIELD 770 } 771 }; 772 773 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 774 775 __checkReturn efx_rc_t 776 ef10_rx_prefix_pktlen( 777 __in efx_nic_t *enp, 778 __in uint8_t *buffer, 779 __out uint16_t *lengthp) 780 { 781 _NOTE(ARGUNUSED(enp)) 782 783 /* 784 * The RX pseudo-header contains the packet length, excluding the 785 * pseudo-header. If the hardware receive datapath was operating in 786 * cut-through mode then the length in the RX pseudo-header will be 787 * zero, and the packet length must be obtained from the DMA length 788 * reported in the RX event. 789 */ 790 *lengthp = buffer[8] | (buffer[9] << 8); 791 return (0); 792 } 793 794 #if EFSYS_OPT_RX_SCALE 795 __checkReturn uint32_t 796 ef10_rx_prefix_hash( 797 __in efx_nic_t *enp, 798 __in efx_rx_hash_alg_t func, 799 __in uint8_t *buffer) 800 { 801 _NOTE(ARGUNUSED(enp)) 802 803 switch (func) { 804 case EFX_RX_HASHALG_PACKED_STREAM: 805 case EFX_RX_HASHALG_TOEPLITZ: 806 return (buffer[0] | 807 (buffer[1] << 8) | 808 (buffer[2] << 16) | 809 (buffer[3] << 24)); 810 811 default: 812 EFSYS_ASSERT(0); 813 return (0); 814 } 815 } 816 #endif /* EFSYS_OPT_RX_SCALE */ 817 818 #if EFSYS_OPT_RX_PACKED_STREAM 819 /* 820 * Fake length for RXQ descriptors in packed stream mode 821 * to make hardware happy 822 */ 823 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32 824 #endif 825 826 void 827 ef10_rx_qpost( 828 __in efx_rxq_t *erp, 829 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 830 __in size_t size, 831 __in unsigned int ndescs, 832 __in unsigned int completed, 833 __in unsigned int added) 834 { 835 efx_qword_t qword; 836 unsigned int i; 837 unsigned int offset; 838 unsigned int id; 839 840 _NOTE(ARGUNUSED(completed)) 841 842 #if EFSYS_OPT_RX_PACKED_STREAM 843 /* 844 * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT 845 * and equal to 0 after applying mask. Hardware does not like it. 846 */ 847 if (erp->er_ev_qstate->eers_rx_packed_stream) 848 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE; 849 #endif 850 851 /* The client driver must not overfill the queue */ 852 EFSYS_ASSERT3U(added - completed + ndescs, <=, 853 EFX_RXQ_LIMIT(erp->er_mask + 1)); 854 855 id = added & (erp->er_mask); 856 for (i = 0; i < ndescs; i++) { 857 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 858 unsigned int, id, efsys_dma_addr_t, addrp[i], 859 size_t, size); 860 861 EFX_POPULATE_QWORD_3(qword, 862 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size), 863 ESF_DZ_RX_KER_BUF_ADDR_DW0, 864 (uint32_t)(addrp[i] & 0xffffffff), 865 ESF_DZ_RX_KER_BUF_ADDR_DW1, 866 (uint32_t)(addrp[i] >> 32)); 867 868 offset = id * sizeof (efx_qword_t); 869 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 870 871 id = (id + 1) & (erp->er_mask); 872 } 873 } 874 875 void 876 ef10_rx_qpush( 877 __in efx_rxq_t *erp, 878 __in unsigned int added, 879 __inout unsigned int *pushedp) 880 { 881 efx_nic_t *enp = erp->er_enp; 882 unsigned int pushed = *pushedp; 883 uint32_t wptr; 884 efx_dword_t dword; 885 886 /* Hardware has alignment restriction for WPTR */ 887 wptr = EFX_P2ALIGN(unsigned int, added, EF10_RX_WPTR_ALIGN); 888 if (pushed == wptr) 889 return; 890 891 *pushedp = wptr; 892 893 /* Push the populated descriptors out */ 894 wptr &= erp->er_mask; 895 896 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr); 897 898 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 899 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 900 EF10_RXQ_DESC_SIZE, wptr, pushed & erp->er_mask); 901 EFSYS_PIO_WRITE_BARRIER(); 902 EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 903 erp->er_index, &dword, B_FALSE); 904 } 905 906 #if EFSYS_OPT_RX_PACKED_STREAM 907 908 void 909 ef10_rx_qpush_ps_credits( 910 __in efx_rxq_t *erp) 911 { 912 efx_nic_t *enp = erp->er_enp; 913 efx_dword_t dword; 914 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; 915 uint32_t credits; 916 917 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); 918 919 if (rxq_state->eers_rx_packed_stream_credits == 0) 920 return; 921 922 /* 923 * It is a bug if we think that FW has utilized more 924 * credits than it is allowed to have (maximum). However, 925 * make sure that we do not credit more than maximum anyway. 926 */ 927 credits = MIN(rxq_state->eers_rx_packed_stream_credits, 928 EFX_RX_PACKED_STREAM_MAX_CREDITS); 929 EFX_POPULATE_DWORD_3(dword, 930 ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1, 931 ERF_DZ_RX_DESC_MAGIC_CMD, 932 ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS, 933 ERF_DZ_RX_DESC_MAGIC_DATA, credits); 934 EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 935 erp->er_index, &dword, B_FALSE); 936 937 rxq_state->eers_rx_packed_stream_credits = 0; 938 } 939 940 /* 941 * In accordance with SF-112241-TC the received data has the following layout: 942 * - 8 byte pseudo-header which consist of: 943 * - 4 byte little-endian timestamp 944 * - 2 byte little-endian captured length in bytes 945 * - 2 byte little-endian original packet length in bytes 946 * - captured packet bytes 947 * - optional padding to align to 64 bytes boundary 948 * - 64 bytes scratch space for the host software 949 */ 950 __checkReturn uint8_t * 951 ef10_rx_qps_packet_info( 952 __in efx_rxq_t *erp, 953 __in uint8_t *buffer, 954 __in uint32_t buffer_length, 955 __in uint32_t current_offset, 956 __out uint16_t *lengthp, 957 __out uint32_t *next_offsetp, 958 __out uint32_t *timestamp) 959 { 960 uint16_t buf_len; 961 uint8_t *pkt_start; 962 efx_qword_t *qwordp; 963 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; 964 965 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); 966 967 buffer += current_offset; 968 pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE; 969 970 qwordp = (efx_qword_t *)buffer; 971 *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP); 972 *lengthp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN); 973 buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN); 974 975 buf_len = EFX_P2ROUNDUP(uint16_t, 976 buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE, 977 EFX_RX_PACKED_STREAM_ALIGNMENT); 978 *next_offsetp = 979 current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT; 980 981 EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length); 982 EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp); 983 984 if ((*next_offsetp ^ current_offset) & 985 EFX_RX_PACKED_STREAM_MEM_PER_CREDIT) 986 rxq_state->eers_rx_packed_stream_credits++; 987 988 return (pkt_start); 989 } 990 991 992 #endif 993 994 __checkReturn efx_rc_t 995 ef10_rx_qflush( 996 __in efx_rxq_t *erp) 997 { 998 efx_nic_t *enp = erp->er_enp; 999 efx_rc_t rc; 1000 1001 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0) 1002 goto fail1; 1003 1004 return (0); 1005 1006 fail1: 1007 /* 1008 * EALREADY is not an error, but indicates that the MC has rebooted and 1009 * that the RXQ has already been destroyed. Callers need to know that 1010 * the RXQ flush has completed to avoid waiting until timeout for a 1011 * flush done event that will not be delivered. 1012 */ 1013 if (rc != EALREADY) 1014 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1015 1016 return (rc); 1017 } 1018 1019 void 1020 ef10_rx_qenable( 1021 __in efx_rxq_t *erp) 1022 { 1023 /* FIXME */ 1024 _NOTE(ARGUNUSED(erp)) 1025 /* FIXME */ 1026 } 1027 1028 __checkReturn efx_rc_t 1029 ef10_rx_qcreate( 1030 __in efx_nic_t *enp, 1031 __in unsigned int index, 1032 __in unsigned int label, 1033 __in efx_rxq_type_t type, 1034 __in_opt const efx_rxq_type_data_t *type_data, 1035 __in efsys_mem_t *esmp, 1036 __in size_t ndescs, 1037 __in uint32_t id, 1038 __in unsigned int flags, 1039 __in efx_evq_t *eep, 1040 __in efx_rxq_t *erp) 1041 { 1042 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 1043 efx_mcdi_init_rxq_params_t params; 1044 const efx_rx_prefix_layout_t *erpl; 1045 efx_rc_t rc; 1046 1047 _NOTE(ARGUNUSED(id, erp)) 1048 1049 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH)); 1050 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 1051 1052 memset(¶ms, 0, sizeof (params)); 1053 params.buf_size = erp->er_buf_size; 1054 1055 switch (type) { 1056 case EFX_RXQ_TYPE_DEFAULT: 1057 erpl = &ef10_default_rx_prefix_layout; 1058 if (type_data == NULL) { 1059 rc = EINVAL; 1060 goto fail1; 1061 } 1062 erp->er_buf_size = type_data->ertd_default.ed_buf_size; 1063 if (flags & EFX_RXQ_FLAG_USER_MARK) { 1064 rc = ENOTSUP; 1065 goto fail2; 1066 } 1067 if (flags & EFX_RXQ_FLAG_USER_FLAG) { 1068 rc = ENOTSUP; 1069 goto fail3; 1070 } 1071 /* 1072 * Ignore EFX_RXQ_FLAG_RSS_HASH since if RSS hash is calculated 1073 * it is always delivered from HW in the pseudo-header. 1074 */ 1075 break; 1076 #if EFSYS_OPT_RX_PACKED_STREAM 1077 case EFX_RXQ_TYPE_PACKED_STREAM: 1078 erpl = &ef10_packed_stream_rx_prefix_layout; 1079 if (type_data == NULL) { 1080 rc = EINVAL; 1081 goto fail4; 1082 } 1083 switch (type_data->ertd_packed_stream.eps_buf_size) { 1084 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M: 1085 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M; 1086 break; 1087 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K: 1088 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K; 1089 break; 1090 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K: 1091 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K; 1092 break; 1093 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K: 1094 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K; 1095 break; 1096 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K: 1097 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K; 1098 break; 1099 default: 1100 rc = ENOTSUP; 1101 goto fail5; 1102 } 1103 erp->er_buf_size = type_data->ertd_packed_stream.eps_buf_size; 1104 /* Packed stream pseudo header does not have RSS hash value */ 1105 if (flags & EFX_RXQ_FLAG_RSS_HASH) { 1106 rc = ENOTSUP; 1107 goto fail6; 1108 } 1109 if (flags & EFX_RXQ_FLAG_USER_MARK) { 1110 rc = ENOTSUP; 1111 goto fail7; 1112 } 1113 if (flags & EFX_RXQ_FLAG_USER_FLAG) { 1114 rc = ENOTSUP; 1115 goto fail8; 1116 } 1117 break; 1118 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1119 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 1120 case EFX_RXQ_TYPE_ES_SUPER_BUFFER: 1121 erpl = &ef10_essb_rx_prefix_layout; 1122 if (type_data == NULL) { 1123 rc = EINVAL; 1124 goto fail9; 1125 } 1126 params.es_bufs_per_desc = 1127 type_data->ertd_es_super_buffer.eessb_bufs_per_desc; 1128 params.es_max_dma_len = 1129 type_data->ertd_es_super_buffer.eessb_max_dma_len; 1130 params.es_buf_stride = 1131 type_data->ertd_es_super_buffer.eessb_buf_stride; 1132 params.hol_block_timeout = 1133 type_data->ertd_es_super_buffer.eessb_hol_block_timeout; 1134 /* 1135 * Ignore EFX_RXQ_FLAG_RSS_HASH since if RSS hash is calculated 1136 * it is always delivered from HW in the pseudo-header. 1137 */ 1138 break; 1139 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1140 default: 1141 rc = ENOTSUP; 1142 goto fail10; 1143 } 1144 1145 #if EFSYS_OPT_RX_PACKED_STREAM 1146 if (params.ps_buf_size != 0) { 1147 /* Check if datapath firmware supports packed stream mode */ 1148 if (encp->enc_rx_packed_stream_supported == B_FALSE) { 1149 rc = ENOTSUP; 1150 goto fail11; 1151 } 1152 /* Check if packed stream allows configurable buffer sizes */ 1153 if ((params.ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) && 1154 (encp->enc_rx_var_packed_stream_supported == B_FALSE)) { 1155 rc = ENOTSUP; 1156 goto fail12; 1157 } 1158 } 1159 #else /* EFSYS_OPT_RX_PACKED_STREAM */ 1160 EFSYS_ASSERT(params.ps_buf_size == 0); 1161 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1162 1163 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 1164 if (params.es_bufs_per_desc > 0) { 1165 if (encp->enc_rx_es_super_buffer_supported == B_FALSE) { 1166 rc = ENOTSUP; 1167 goto fail13; 1168 } 1169 if (!EFX_IS_P2ALIGNED(uint32_t, params.es_max_dma_len, 1170 EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) { 1171 rc = EINVAL; 1172 goto fail14; 1173 } 1174 if (!EFX_IS_P2ALIGNED(uint32_t, params.es_buf_stride, 1175 EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) { 1176 rc = EINVAL; 1177 goto fail15; 1178 } 1179 } 1180 #else /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1181 EFSYS_ASSERT(params.es_bufs_per_desc == 0); 1182 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1183 1184 if (flags & EFX_RXQ_FLAG_INGRESS_MPORT) { 1185 rc = ENOTSUP; 1186 goto fail16; 1187 } 1188 1189 /* Scatter can only be disabled if the firmware supports doing so */ 1190 if (flags & EFX_RXQ_FLAG_SCATTER) 1191 params.disable_scatter = B_FALSE; 1192 else 1193 params.disable_scatter = encp->enc_rx_disable_scatter_supported; 1194 1195 if (flags & EFX_RXQ_FLAG_INNER_CLASSES) 1196 params.want_inner_classes = B_TRUE; 1197 else 1198 params.want_inner_classes = B_FALSE; 1199 1200 if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep, label, index, 1201 esmp, ¶ms)) != 0) 1202 goto fail17; 1203 1204 erp->er_eep = eep; 1205 erp->er_label = label; 1206 1207 ef10_ev_rxlabel_init(eep, erp, label, type); 1208 1209 erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label]; 1210 1211 erp->er_prefix_layout = *erpl; 1212 1213 return (0); 1214 1215 fail17: 1216 EFSYS_PROBE(fail15); 1217 fail16: 1218 EFSYS_PROBE(fail14); 1219 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 1220 fail15: 1221 EFSYS_PROBE(fail15); 1222 fail14: 1223 EFSYS_PROBE(fail14); 1224 fail13: 1225 EFSYS_PROBE(fail13); 1226 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1227 #if EFSYS_OPT_RX_PACKED_STREAM 1228 fail12: 1229 EFSYS_PROBE(fail12); 1230 fail11: 1231 EFSYS_PROBE(fail11); 1232 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1233 fail10: 1234 EFSYS_PROBE(fail10); 1235 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 1236 fail9: 1237 EFSYS_PROBE(fail9); 1238 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1239 #if EFSYS_OPT_RX_PACKED_STREAM 1240 fail8: 1241 EFSYS_PROBE(fail8); 1242 fail7: 1243 EFSYS_PROBE(fail7); 1244 fail6: 1245 EFSYS_PROBE(fail6); 1246 fail5: 1247 EFSYS_PROBE(fail5); 1248 fail4: 1249 EFSYS_PROBE(fail4); 1250 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1251 fail3: 1252 EFSYS_PROBE(fail3); 1253 fail2: 1254 EFSYS_PROBE(fail2); 1255 fail1: 1256 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1257 1258 return (rc); 1259 } 1260 1261 void 1262 ef10_rx_qdestroy( 1263 __in efx_rxq_t *erp) 1264 { 1265 efx_evq_t *eep = erp->er_eep; 1266 unsigned int label = erp->er_label; 1267 1268 ef10_ev_rxlabel_fini(eep, label); 1269 } 1270 1271 #endif /* EFX_OPTS_EF10() */ 1272 1273 void 1274 ef10_rx_fini( 1275 __in efx_nic_t *enp) 1276 { 1277 #if EFSYS_OPT_RX_SCALE 1278 if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE) 1279 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context); 1280 enp->en_rss_context = 0; 1281 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; 1282 #else 1283 _NOTE(ARGUNUSED(enp)) 1284 #endif /* EFSYS_OPT_RX_SCALE */ 1285 } 1286 1287 #endif /* EFSYS_OPT_RIVERHEAD || EFX_OPTS_EF10() */ 1288