1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2012-2019 Solarflare Communications Inc. 5 */ 6 7 #include "efx.h" 8 #include "efx_impl.h" 9 10 11 #if EFSYS_OPT_RIVERHEAD || EFX_OPTS_EF10() 12 13 #if EFSYS_OPT_RX_SCALE 14 static __checkReturn efx_rc_t 15 efx_mcdi_rss_context_alloc( 16 __in efx_nic_t *enp, 17 __in efx_rx_scale_context_type_t type, 18 __in uint32_t num_queues, 19 __out uint32_t *rss_contextp) 20 { 21 efx_mcdi_req_t req; 22 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN, 23 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN); 24 uint32_t rss_context; 25 uint32_t context_type; 26 efx_rc_t rc; 27 28 if (num_queues > EFX_MAXRSS) { 29 rc = EINVAL; 30 goto fail1; 31 } 32 33 switch (type) { 34 case EFX_RX_SCALE_EXCLUSIVE: 35 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE; 36 break; 37 case EFX_RX_SCALE_SHARED: 38 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; 39 break; 40 default: 41 rc = EINVAL; 42 goto fail2; 43 } 44 45 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC; 46 req.emr_in_buf = payload; 47 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN; 48 req.emr_out_buf = payload; 49 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN; 50 51 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, 52 enp->en_vport_id); 53 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type); 54 55 /* 56 * For exclusive contexts, NUM_QUEUES is only used to validate 57 * indirection table offsets. 58 * For shared contexts, the provided context will spread traffic over 59 * NUM_QUEUES many queues. 60 */ 61 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues); 62 63 efx_mcdi_execute(enp, &req); 64 65 if (req.emr_rc != 0) { 66 rc = req.emr_rc; 67 goto fail3; 68 } 69 70 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) { 71 rc = EMSGSIZE; 72 goto fail4; 73 } 74 75 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); 76 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 77 rc = ENOENT; 78 goto fail5; 79 } 80 81 *rss_contextp = rss_context; 82 83 return (0); 84 85 fail5: 86 EFSYS_PROBE(fail5); 87 fail4: 88 EFSYS_PROBE(fail4); 89 fail3: 90 EFSYS_PROBE(fail3); 91 fail2: 92 EFSYS_PROBE(fail2); 93 fail1: 94 EFSYS_PROBE1(fail1, efx_rc_t, rc); 95 96 return (rc); 97 } 98 #endif /* EFSYS_OPT_RX_SCALE */ 99 100 #if EFSYS_OPT_RX_SCALE 101 static efx_rc_t 102 efx_mcdi_rss_context_free( 103 __in efx_nic_t *enp, 104 __in uint32_t rss_context) 105 { 106 efx_mcdi_req_t req; 107 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN, 108 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN); 109 efx_rc_t rc; 110 111 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 112 rc = EINVAL; 113 goto fail1; 114 } 115 116 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE; 117 req.emr_in_buf = payload; 118 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN; 119 req.emr_out_buf = payload; 120 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN; 121 122 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context); 123 124 efx_mcdi_execute_quiet(enp, &req); 125 126 if (req.emr_rc != 0) { 127 rc = req.emr_rc; 128 goto fail2; 129 } 130 131 return (0); 132 133 fail2: 134 EFSYS_PROBE(fail2); 135 fail1: 136 EFSYS_PROBE1(fail1, efx_rc_t, rc); 137 138 return (rc); 139 } 140 #endif /* EFSYS_OPT_RX_SCALE */ 141 142 #if EFSYS_OPT_RX_SCALE 143 static efx_rc_t 144 efx_mcdi_rss_context_set_flags( 145 __in efx_nic_t *enp, 146 __in uint32_t rss_context, 147 __in efx_rx_hash_type_t type) 148 { 149 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 150 efx_mcdi_req_t req; 151 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN, 152 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN); 153 efx_rc_t rc; 154 155 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_LBN == 156 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_LBN); 157 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_WIDTH == 158 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_WIDTH); 159 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_LBN == 160 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_LBN); 161 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_WIDTH == 162 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_WIDTH); 163 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_LBN == 164 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_LBN); 165 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_WIDTH == 166 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_WIDTH); 167 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_LBN == 168 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_LBN); 169 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_WIDTH == 170 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_WIDTH); 171 172 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 173 rc = EINVAL; 174 goto fail1; 175 } 176 177 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS; 178 req.emr_in_buf = payload; 179 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN; 180 req.emr_out_buf = payload; 181 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN; 182 183 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, 184 rss_context); 185 186 /* 187 * If the firmware lacks support for additional modes, RSS_MODE 188 * fields must contain zeros, otherwise the operation will fail. 189 */ 190 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE) 191 type &= EFX_RX_HASH_LEGACY_MASK; 192 193 MCDI_IN_POPULATE_DWORD_10(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, 194 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN, 195 (type & EFX_RX_HASH_IPV4) ? 1 : 0, 196 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN, 197 (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0, 198 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN, 199 (type & EFX_RX_HASH_IPV6) ? 1 : 0, 200 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN, 201 (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0, 202 RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE, 203 (type >> EFX_RX_CLASS_IPV4_TCP_LBN) & 204 EFX_MASK32(EFX_RX_CLASS_IPV4_TCP), 205 RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE, 206 (type >> EFX_RX_CLASS_IPV4_UDP_LBN) & 207 EFX_MASK32(EFX_RX_CLASS_IPV4_UDP), 208 RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE, 209 (type >> EFX_RX_CLASS_IPV4_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV4), 210 RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE, 211 (type >> EFX_RX_CLASS_IPV6_TCP_LBN) & 212 EFX_MASK32(EFX_RX_CLASS_IPV6_TCP), 213 RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE, 214 (type >> EFX_RX_CLASS_IPV6_UDP_LBN) & 215 EFX_MASK32(EFX_RX_CLASS_IPV6_UDP), 216 RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE, 217 (type >> EFX_RX_CLASS_IPV6_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV6)); 218 219 efx_mcdi_execute(enp, &req); 220 221 if (req.emr_rc != 0) { 222 rc = req.emr_rc; 223 goto fail2; 224 } 225 226 return (0); 227 228 fail2: 229 EFSYS_PROBE(fail2); 230 fail1: 231 EFSYS_PROBE1(fail1, efx_rc_t, rc); 232 233 return (rc); 234 } 235 #endif /* EFSYS_OPT_RX_SCALE */ 236 237 #if EFSYS_OPT_RX_SCALE 238 static efx_rc_t 239 efx_mcdi_rss_context_set_key( 240 __in efx_nic_t *enp, 241 __in uint32_t rss_context, 242 __in_ecount(n) uint8_t *key, 243 __in size_t n) 244 { 245 efx_mcdi_req_t req; 246 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN, 247 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN); 248 efx_rc_t rc; 249 250 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 251 rc = EINVAL; 252 goto fail1; 253 } 254 255 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY; 256 req.emr_in_buf = payload; 257 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN; 258 req.emr_out_buf = payload; 259 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN; 260 261 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, 262 rss_context); 263 264 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 265 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) { 266 rc = EINVAL; 267 goto fail2; 268 } 269 270 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), 271 key, n); 272 273 efx_mcdi_execute(enp, &req); 274 275 if (req.emr_rc != 0) { 276 rc = req.emr_rc; 277 goto fail3; 278 } 279 280 return (0); 281 282 fail3: 283 EFSYS_PROBE(fail3); 284 fail2: 285 EFSYS_PROBE(fail2); 286 fail1: 287 EFSYS_PROBE1(fail1, efx_rc_t, rc); 288 289 return (rc); 290 } 291 #endif /* EFSYS_OPT_RX_SCALE */ 292 293 #if EFSYS_OPT_RX_SCALE 294 static efx_rc_t 295 efx_mcdi_rss_context_set_table( 296 __in efx_nic_t *enp, 297 __in uint32_t rss_context, 298 __in_ecount(n) unsigned int *table, 299 __in size_t n) 300 { 301 efx_mcdi_req_t req; 302 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN, 303 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN); 304 uint8_t *req_table; 305 int i, rc; 306 307 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 308 rc = EINVAL; 309 goto fail1; 310 } 311 312 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE; 313 req.emr_in_buf = payload; 314 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN; 315 req.emr_out_buf = payload; 316 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN; 317 318 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, 319 rss_context); 320 321 req_table = 322 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE); 323 324 for (i = 0; 325 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN; 326 i++) { 327 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0; 328 } 329 330 efx_mcdi_execute(enp, &req); 331 332 if (req.emr_rc != 0) { 333 rc = req.emr_rc; 334 goto fail2; 335 } 336 337 return (0); 338 339 fail2: 340 EFSYS_PROBE(fail2); 341 fail1: 342 EFSYS_PROBE1(fail1, efx_rc_t, rc); 343 344 return (rc); 345 } 346 #endif /* EFSYS_OPT_RX_SCALE */ 347 348 349 __checkReturn efx_rc_t 350 ef10_rx_init( 351 __in efx_nic_t *enp) 352 { 353 #if EFSYS_OPT_RX_SCALE 354 355 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS, 356 &enp->en_rss_context) == 0) { 357 /* 358 * Allocated an exclusive RSS context, which allows both the 359 * indirection table and key to be modified. 360 */ 361 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 362 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 363 } else { 364 /* 365 * Failed to allocate an exclusive RSS context. Continue 366 * operation without support for RSS. The pseudo-header in 367 * received packets will not contain a Toeplitz hash value. 368 */ 369 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; 370 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE; 371 } 372 373 #endif /* EFSYS_OPT_RX_SCALE */ 374 375 return (0); 376 } 377 378 #if EFX_OPTS_EF10() 379 380 #if EFSYS_OPT_RX_SCATTER 381 __checkReturn efx_rc_t 382 ef10_rx_scatter_enable( 383 __in efx_nic_t *enp, 384 __in unsigned int buf_size) 385 { 386 _NOTE(ARGUNUSED(enp, buf_size)) 387 return (0); 388 } 389 #endif /* EFSYS_OPT_RX_SCATTER */ 390 391 #endif /* EFX_OPTS_EF10() */ 392 393 #if EFSYS_OPT_RX_SCALE 394 __checkReturn efx_rc_t 395 ef10_rx_scale_context_alloc( 396 __in efx_nic_t *enp, 397 __in efx_rx_scale_context_type_t type, 398 __in uint32_t num_queues, 399 __out uint32_t *rss_contextp) 400 { 401 efx_rc_t rc; 402 403 rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp); 404 if (rc != 0) 405 goto fail1; 406 407 return (0); 408 409 fail1: 410 EFSYS_PROBE1(fail1, efx_rc_t, rc); 411 return (rc); 412 } 413 #endif /* EFSYS_OPT_RX_SCALE */ 414 415 #if EFSYS_OPT_RX_SCALE 416 __checkReturn efx_rc_t 417 ef10_rx_scale_context_free( 418 __in efx_nic_t *enp, 419 __in uint32_t rss_context) 420 { 421 efx_rc_t rc; 422 423 rc = efx_mcdi_rss_context_free(enp, rss_context); 424 if (rc != 0) 425 goto fail1; 426 427 return (0); 428 429 fail1: 430 EFSYS_PROBE1(fail1, efx_rc_t, rc); 431 return (rc); 432 } 433 #endif /* EFSYS_OPT_RX_SCALE */ 434 435 #if EFSYS_OPT_RX_SCALE 436 __checkReturn efx_rc_t 437 ef10_rx_scale_mode_set( 438 __in efx_nic_t *enp, 439 __in uint32_t rss_context, 440 __in efx_rx_hash_alg_t alg, 441 __in efx_rx_hash_type_t type, 442 __in boolean_t insert) 443 { 444 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 445 efx_rc_t rc; 446 447 EFSYS_ASSERT3U(insert, ==, B_TRUE); 448 449 if ((encp->enc_rx_scale_hash_alg_mask & (1U << alg)) == 0 || 450 insert == B_FALSE) { 451 rc = EINVAL; 452 goto fail1; 453 } 454 455 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 456 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 457 rc = ENOTSUP; 458 goto fail2; 459 } 460 rss_context = enp->en_rss_context; 461 } 462 463 if ((rc = efx_mcdi_rss_context_set_flags(enp, 464 rss_context, type)) != 0) 465 goto fail3; 466 467 return (0); 468 469 fail3: 470 EFSYS_PROBE(fail3); 471 fail2: 472 EFSYS_PROBE(fail2); 473 fail1: 474 EFSYS_PROBE1(fail1, efx_rc_t, rc); 475 476 return (rc); 477 } 478 #endif /* EFSYS_OPT_RX_SCALE */ 479 480 #if EFSYS_OPT_RX_SCALE 481 __checkReturn efx_rc_t 482 ef10_rx_scale_key_set( 483 __in efx_nic_t *enp, 484 __in uint32_t rss_context, 485 __in_ecount(n) uint8_t *key, 486 __in size_t n) 487 { 488 efx_rc_t rc; 489 490 EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE == 491 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 492 493 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 494 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 495 rc = ENOTSUP; 496 goto fail1; 497 } 498 rss_context = enp->en_rss_context; 499 } 500 501 if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0) 502 goto fail2; 503 504 return (0); 505 506 fail2: 507 EFSYS_PROBE(fail2); 508 fail1: 509 EFSYS_PROBE1(fail1, efx_rc_t, rc); 510 511 return (rc); 512 } 513 #endif /* EFSYS_OPT_RX_SCALE */ 514 515 #if EFSYS_OPT_RX_SCALE 516 __checkReturn efx_rc_t 517 ef10_rx_scale_tbl_set( 518 __in efx_nic_t *enp, 519 __in uint32_t rss_context, 520 __in_ecount(n) unsigned int *table, 521 __in size_t n) 522 { 523 efx_rc_t rc; 524 525 526 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 527 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 528 rc = ENOTSUP; 529 goto fail1; 530 } 531 rss_context = enp->en_rss_context; 532 } 533 534 if ((rc = efx_mcdi_rss_context_set_table(enp, 535 rss_context, table, n)) != 0) 536 goto fail2; 537 538 return (0); 539 540 fail2: 541 EFSYS_PROBE(fail2); 542 fail1: 543 EFSYS_PROBE1(fail1, efx_rc_t, rc); 544 545 return (rc); 546 } 547 #endif /* EFSYS_OPT_RX_SCALE */ 548 549 #if EFX_OPTS_EF10() 550 551 /* 552 * EF10 RX pseudo-header (aka Rx prefix) 553 * ------------------------------------- 554 * 555 * Receive packets are prefixed by an (optional) 14 byte pseudo-header: 556 * 557 * +00: Toeplitz hash value. 558 * (32bit little-endian) 559 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag. 560 * (16bit big-endian) 561 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag. 562 * (16bit big-endian) 563 * +08: Packet Length. Zero if the RX datapath was in cut-through mode. 564 * (16bit little-endian) 565 * +10: MAC timestamp. Zero if timestamping is not enabled. 566 * (32bit little-endian) 567 * 568 * See "The RX Pseudo-header" in SF-109306-TC. 569 * 570 * EF10 does not support Rx prefix choice using MC_CMD_GET_RX_PREFIX_ID 571 * and query its layout using MC_CMD_QUERY_RX_PREFIX_ID. 572 */ 573 static const efx_rx_prefix_layout_t ef10_default_rx_prefix_layout = { 574 .erpl_id = 0, 575 .erpl_length = 14, 576 .erpl_fields = { 577 [EFX_RX_PREFIX_FIELD_RSS_HASH] = 578 { 0, 32, B_FALSE }, 579 [EFX_RX_PREFIX_FIELD_VLAN_STRIP_TCI] = 580 { 32, 16, B_TRUE }, 581 [EFX_RX_PREFIX_FIELD_INNER_VLAN_STRIP_TCI] = 582 { 48, 16, B_TRUE }, 583 [EFX_RX_PREFIX_FIELD_LENGTH] = 584 { 64, 16, B_FALSE }, 585 [EFX_RX_PREFIX_FIELD_PARTIAL_TSTAMP] = 586 { 80, 32, B_FALSE }, 587 } 588 }; 589 590 #if EFSYS_OPT_RX_PACKED_STREAM 591 592 /* 593 * EF10 packed stream Rx prefix layout. 594 * 595 * See SF-112241-TC Full speed capture for Huntington and Medford section 4.5. 596 */ 597 static const efx_rx_prefix_layout_t ef10_packed_stream_rx_prefix_layout = { 598 .erpl_id = 0, 599 .erpl_length = 8, 600 .erpl_fields = { 601 #define EF10_PS_RX_PREFIX_FIELD(_efx, _ef10) \ 602 EFX_RX_PREFIX_FIELD(_efx, ES_DZ_PS_RX_PREFIX_ ## _ef10, B_FALSE) 603 604 EF10_PS_RX_PREFIX_FIELD(PARTIAL_TSTAMP, TSTAMP), 605 EF10_PS_RX_PREFIX_FIELD(LENGTH, CAP_LEN), 606 EF10_PS_RX_PREFIX_FIELD(ORIG_LENGTH, ORIG_LEN), 607 608 #undef EF10_PS_RX_PREFIX_FIELD 609 } 610 }; 611 612 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 613 614 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 615 616 /* 617 * EF10 equal stride super-buffer Rx prefix layout. 618 * 619 * See SF-119419-TC DPDK Firmware Driver Interface section 3.4. 620 */ 621 static const efx_rx_prefix_layout_t ef10_essb_rx_prefix_layout = { 622 .erpl_id = 0, 623 .erpl_length = ES_EZ_ESSB_RX_PREFIX_LEN, 624 .erpl_fields = { 625 #define EF10_ESSB_RX_PREFIX_FIELD(_efx, _ef10) \ 626 EFX_RX_PREFIX_FIELD(_efx, ES_EZ_ESSB_RX_PREFIX_ ## _ef10, B_FALSE) 627 628 EF10_ESSB_RX_PREFIX_FIELD(LENGTH, DATA_LEN), 629 EF10_ESSB_RX_PREFIX_FIELD(USER_MARK, MARK), 630 EF10_ESSB_RX_PREFIX_FIELD(RSS_HASH_VALID, HASH_VALID), 631 EF10_ESSB_RX_PREFIX_FIELD(USER_MARK_VALID, MARK_VALID), 632 EF10_ESSB_RX_PREFIX_FIELD(USER_FLAG, MATCH_FLAG), 633 EF10_ESSB_RX_PREFIX_FIELD(RSS_HASH, HASH), 634 635 #undef EF10_ESSB_RX_PREFIX_FIELD 636 } 637 }; 638 639 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 640 641 __checkReturn efx_rc_t 642 ef10_rx_prefix_pktlen( 643 __in efx_nic_t *enp, 644 __in uint8_t *buffer, 645 __out uint16_t *lengthp) 646 { 647 _NOTE(ARGUNUSED(enp)) 648 649 /* 650 * The RX pseudo-header contains the packet length, excluding the 651 * pseudo-header. If the hardware receive datapath was operating in 652 * cut-through mode then the length in the RX pseudo-header will be 653 * zero, and the packet length must be obtained from the DMA length 654 * reported in the RX event. 655 */ 656 *lengthp = buffer[8] | (buffer[9] << 8); 657 return (0); 658 } 659 660 #if EFSYS_OPT_RX_SCALE 661 __checkReturn uint32_t 662 ef10_rx_prefix_hash( 663 __in efx_nic_t *enp, 664 __in efx_rx_hash_alg_t func, 665 __in uint8_t *buffer) 666 { 667 _NOTE(ARGUNUSED(enp)) 668 669 switch (func) { 670 case EFX_RX_HASHALG_PACKED_STREAM: 671 case EFX_RX_HASHALG_TOEPLITZ: 672 return (buffer[0] | 673 (buffer[1] << 8) | 674 (buffer[2] << 16) | 675 (buffer[3] << 24)); 676 677 default: 678 EFSYS_ASSERT(0); 679 return (0); 680 } 681 } 682 #endif /* EFSYS_OPT_RX_SCALE */ 683 684 #if EFSYS_OPT_RX_PACKED_STREAM 685 /* 686 * Fake length for RXQ descriptors in packed stream mode 687 * to make hardware happy 688 */ 689 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32 690 #endif 691 692 void 693 ef10_rx_qpost( 694 __in efx_rxq_t *erp, 695 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 696 __in size_t size, 697 __in unsigned int ndescs, 698 __in unsigned int completed, 699 __in unsigned int added) 700 { 701 efx_qword_t qword; 702 unsigned int i; 703 unsigned int offset; 704 unsigned int id; 705 706 _NOTE(ARGUNUSED(completed)) 707 708 #if EFSYS_OPT_RX_PACKED_STREAM 709 /* 710 * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT 711 * and equal to 0 after applying mask. Hardware does not like it. 712 */ 713 if (erp->er_ev_qstate->eers_rx_packed_stream) 714 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE; 715 #endif 716 717 /* The client driver must not overfill the queue */ 718 EFSYS_ASSERT3U(added - completed + ndescs, <=, 719 EFX_RXQ_LIMIT(erp->er_mask + 1)); 720 721 id = added & (erp->er_mask); 722 for (i = 0; i < ndescs; i++) { 723 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 724 unsigned int, id, efsys_dma_addr_t, addrp[i], 725 size_t, size); 726 727 EFX_POPULATE_QWORD_3(qword, 728 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size), 729 ESF_DZ_RX_KER_BUF_ADDR_DW0, 730 (uint32_t)(addrp[i] & 0xffffffff), 731 ESF_DZ_RX_KER_BUF_ADDR_DW1, 732 (uint32_t)(addrp[i] >> 32)); 733 734 offset = id * sizeof (efx_qword_t); 735 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 736 737 id = (id + 1) & (erp->er_mask); 738 } 739 } 740 741 void 742 ef10_rx_qpush( 743 __in efx_rxq_t *erp, 744 __in unsigned int added, 745 __inout unsigned int *pushedp) 746 { 747 efx_nic_t *enp = erp->er_enp; 748 unsigned int pushed = *pushedp; 749 uint32_t wptr; 750 efx_dword_t dword; 751 752 /* Hardware has alignment restriction for WPTR */ 753 wptr = EFX_P2ALIGN(unsigned int, added, EF10_RX_WPTR_ALIGN); 754 if (pushed == wptr) 755 return; 756 757 *pushedp = wptr; 758 759 /* Push the populated descriptors out */ 760 wptr &= erp->er_mask; 761 762 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr); 763 764 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 765 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 766 EF10_RXQ_DESC_SIZE, wptr, pushed & erp->er_mask); 767 EFSYS_PIO_WRITE_BARRIER(); 768 EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 769 erp->er_index, &dword, B_FALSE); 770 } 771 772 #if EFSYS_OPT_RX_PACKED_STREAM 773 774 void 775 ef10_rx_qpush_ps_credits( 776 __in efx_rxq_t *erp) 777 { 778 efx_nic_t *enp = erp->er_enp; 779 efx_dword_t dword; 780 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; 781 uint32_t credits; 782 783 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); 784 785 if (rxq_state->eers_rx_packed_stream_credits == 0) 786 return; 787 788 /* 789 * It is a bug if we think that FW has utilized more 790 * credits than it is allowed to have (maximum). However, 791 * make sure that we do not credit more than maximum anyway. 792 */ 793 credits = MIN(rxq_state->eers_rx_packed_stream_credits, 794 EFX_RX_PACKED_STREAM_MAX_CREDITS); 795 EFX_POPULATE_DWORD_3(dword, 796 ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1, 797 ERF_DZ_RX_DESC_MAGIC_CMD, 798 ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS, 799 ERF_DZ_RX_DESC_MAGIC_DATA, credits); 800 EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 801 erp->er_index, &dword, B_FALSE); 802 803 rxq_state->eers_rx_packed_stream_credits = 0; 804 } 805 806 /* 807 * In accordance with SF-112241-TC the received data has the following layout: 808 * - 8 byte pseudo-header which consist of: 809 * - 4 byte little-endian timestamp 810 * - 2 byte little-endian captured length in bytes 811 * - 2 byte little-endian original packet length in bytes 812 * - captured packet bytes 813 * - optional padding to align to 64 bytes boundary 814 * - 64 bytes scratch space for the host software 815 */ 816 __checkReturn uint8_t * 817 ef10_rx_qps_packet_info( 818 __in efx_rxq_t *erp, 819 __in uint8_t *buffer, 820 __in uint32_t buffer_length, 821 __in uint32_t current_offset, 822 __out uint16_t *lengthp, 823 __out uint32_t *next_offsetp, 824 __out uint32_t *timestamp) 825 { 826 uint16_t buf_len; 827 uint8_t *pkt_start; 828 efx_qword_t *qwordp; 829 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; 830 831 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); 832 833 buffer += current_offset; 834 pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE; 835 836 qwordp = (efx_qword_t *)buffer; 837 *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP); 838 *lengthp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN); 839 buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN); 840 841 buf_len = EFX_P2ROUNDUP(uint16_t, 842 buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE, 843 EFX_RX_PACKED_STREAM_ALIGNMENT); 844 *next_offsetp = 845 current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT; 846 847 EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length); 848 EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp); 849 850 if ((*next_offsetp ^ current_offset) & 851 EFX_RX_PACKED_STREAM_MEM_PER_CREDIT) 852 rxq_state->eers_rx_packed_stream_credits++; 853 854 return (pkt_start); 855 } 856 857 858 #endif 859 860 __checkReturn efx_rc_t 861 ef10_rx_qflush( 862 __in efx_rxq_t *erp) 863 { 864 efx_nic_t *enp = erp->er_enp; 865 efx_rc_t rc; 866 867 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0) 868 goto fail1; 869 870 return (0); 871 872 fail1: 873 /* 874 * EALREADY is not an error, but indicates that the MC has rebooted and 875 * that the RXQ has already been destroyed. Callers need to know that 876 * the RXQ flush has completed to avoid waiting until timeout for a 877 * flush done event that will not be delivered. 878 */ 879 if (rc != EALREADY) 880 EFSYS_PROBE1(fail1, efx_rc_t, rc); 881 882 return (rc); 883 } 884 885 void 886 ef10_rx_qenable( 887 __in efx_rxq_t *erp) 888 { 889 /* FIXME */ 890 _NOTE(ARGUNUSED(erp)) 891 /* FIXME */ 892 } 893 894 __checkReturn efx_rc_t 895 ef10_rx_qcreate( 896 __in efx_nic_t *enp, 897 __in unsigned int index, 898 __in unsigned int label, 899 __in efx_rxq_type_t type, 900 __in_opt const efx_rxq_type_data_t *type_data, 901 __in efsys_mem_t *esmp, 902 __in size_t ndescs, 903 __in uint32_t id, 904 __in unsigned int flags, 905 __in efx_evq_t *eep, 906 __in efx_rxq_t *erp) 907 { 908 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 909 efx_mcdi_init_rxq_params_t params; 910 const efx_rx_prefix_layout_t *erpl; 911 efx_rc_t rc; 912 913 _NOTE(ARGUNUSED(id, erp)) 914 915 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH)); 916 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 917 918 memset(¶ms, 0, sizeof (params)); 919 params.buf_size = erp->er_buf_size; 920 921 switch (type) { 922 case EFX_RXQ_TYPE_DEFAULT: 923 erpl = &ef10_default_rx_prefix_layout; 924 if (type_data == NULL) { 925 rc = EINVAL; 926 goto fail1; 927 } 928 erp->er_buf_size = type_data->ertd_default.ed_buf_size; 929 /* 930 * Ignore EFX_RXQ_FLAG_RSS_HASH since if RSS hash is calculated 931 * it is always delivered from HW in the pseudo-header. 932 */ 933 break; 934 #if EFSYS_OPT_RX_PACKED_STREAM 935 case EFX_RXQ_TYPE_PACKED_STREAM: 936 erpl = &ef10_packed_stream_rx_prefix_layout; 937 if (type_data == NULL) { 938 rc = EINVAL; 939 goto fail2; 940 } 941 switch (type_data->ertd_packed_stream.eps_buf_size) { 942 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M: 943 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M; 944 break; 945 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K: 946 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K; 947 break; 948 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K: 949 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K; 950 break; 951 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K: 952 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K; 953 break; 954 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K: 955 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K; 956 break; 957 default: 958 rc = ENOTSUP; 959 goto fail3; 960 } 961 erp->er_buf_size = type_data->ertd_packed_stream.eps_buf_size; 962 /* Packed stream pseudo header does not have RSS hash value */ 963 if (flags & EFX_RXQ_FLAG_RSS_HASH) { 964 rc = ENOTSUP; 965 goto fail4; 966 } 967 break; 968 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 969 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 970 case EFX_RXQ_TYPE_ES_SUPER_BUFFER: 971 erpl = &ef10_essb_rx_prefix_layout; 972 if (type_data == NULL) { 973 rc = EINVAL; 974 goto fail5; 975 } 976 params.es_bufs_per_desc = 977 type_data->ertd_es_super_buffer.eessb_bufs_per_desc; 978 params.es_max_dma_len = 979 type_data->ertd_es_super_buffer.eessb_max_dma_len; 980 params.es_buf_stride = 981 type_data->ertd_es_super_buffer.eessb_buf_stride; 982 params.hol_block_timeout = 983 type_data->ertd_es_super_buffer.eessb_hol_block_timeout; 984 /* 985 * Ignore EFX_RXQ_FLAG_RSS_HASH since if RSS hash is calculated 986 * it is always delivered from HW in the pseudo-header. 987 */ 988 break; 989 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 990 default: 991 rc = ENOTSUP; 992 goto fail6; 993 } 994 995 #if EFSYS_OPT_RX_PACKED_STREAM 996 if (params.ps_buf_size != 0) { 997 /* Check if datapath firmware supports packed stream mode */ 998 if (encp->enc_rx_packed_stream_supported == B_FALSE) { 999 rc = ENOTSUP; 1000 goto fail7; 1001 } 1002 /* Check if packed stream allows configurable buffer sizes */ 1003 if ((params.ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) && 1004 (encp->enc_rx_var_packed_stream_supported == B_FALSE)) { 1005 rc = ENOTSUP; 1006 goto fail8; 1007 } 1008 } 1009 #else /* EFSYS_OPT_RX_PACKED_STREAM */ 1010 EFSYS_ASSERT(params.ps_buf_size == 0); 1011 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1012 1013 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 1014 if (params.es_bufs_per_desc > 0) { 1015 if (encp->enc_rx_es_super_buffer_supported == B_FALSE) { 1016 rc = ENOTSUP; 1017 goto fail9; 1018 } 1019 if (!EFX_IS_P2ALIGNED(uint32_t, params.es_max_dma_len, 1020 EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) { 1021 rc = EINVAL; 1022 goto fail10; 1023 } 1024 if (!EFX_IS_P2ALIGNED(uint32_t, params.es_buf_stride, 1025 EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) { 1026 rc = EINVAL; 1027 goto fail11; 1028 } 1029 } 1030 #else /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1031 EFSYS_ASSERT(params.es_bufs_per_desc == 0); 1032 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1033 1034 /* Scatter can only be disabled if the firmware supports doing so */ 1035 if (flags & EFX_RXQ_FLAG_SCATTER) 1036 params.disable_scatter = B_FALSE; 1037 else 1038 params.disable_scatter = encp->enc_rx_disable_scatter_supported; 1039 1040 if (flags & EFX_RXQ_FLAG_INNER_CLASSES) 1041 params.want_inner_classes = B_TRUE; 1042 else 1043 params.want_inner_classes = B_FALSE; 1044 1045 if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep, label, index, 1046 esmp, ¶ms)) != 0) 1047 goto fail12; 1048 1049 erp->er_eep = eep; 1050 erp->er_label = label; 1051 1052 ef10_ev_rxlabel_init(eep, erp, label, type); 1053 1054 erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label]; 1055 1056 erp->er_prefix_layout = *erpl; 1057 1058 return (0); 1059 1060 fail12: 1061 EFSYS_PROBE(fail12); 1062 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 1063 fail11: 1064 EFSYS_PROBE(fail11); 1065 fail10: 1066 EFSYS_PROBE(fail10); 1067 fail9: 1068 EFSYS_PROBE(fail9); 1069 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1070 #if EFSYS_OPT_RX_PACKED_STREAM 1071 fail8: 1072 EFSYS_PROBE(fail8); 1073 fail7: 1074 EFSYS_PROBE(fail7); 1075 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1076 fail6: 1077 EFSYS_PROBE(fail6); 1078 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 1079 fail5: 1080 EFSYS_PROBE(fail5); 1081 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1082 #if EFSYS_OPT_RX_PACKED_STREAM 1083 fail4: 1084 EFSYS_PROBE(fail4); 1085 fail3: 1086 EFSYS_PROBE(fail3); 1087 fail2: 1088 EFSYS_PROBE(fail2); 1089 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1090 fail1: 1091 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1092 1093 return (rc); 1094 } 1095 1096 void 1097 ef10_rx_qdestroy( 1098 __in efx_rxq_t *erp) 1099 { 1100 efx_evq_t *eep = erp->er_eep; 1101 unsigned int label = erp->er_label; 1102 1103 ef10_ev_rxlabel_fini(eep, label); 1104 } 1105 1106 #endif /* EFX_OPTS_EF10() */ 1107 1108 void 1109 ef10_rx_fini( 1110 __in efx_nic_t *enp) 1111 { 1112 #if EFSYS_OPT_RX_SCALE 1113 if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE) 1114 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context); 1115 enp->en_rss_context = 0; 1116 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; 1117 #else 1118 _NOTE(ARGUNUSED(enp)) 1119 #endif /* EFSYS_OPT_RX_SCALE */ 1120 } 1121 1122 #endif /* EFSYS_OPT_RIVERHEAD || EFX_OPTS_EF10() */ 1123