1 /* SPDX-License-Identifier: BSD-3-Clause 2 * 3 * Copyright(c) 2019-2021 Xilinx, Inc. 4 * Copyright(c) 2012-2019 Solarflare Communications Inc. 5 */ 6 7 #include "efx.h" 8 #include "efx_impl.h" 9 10 11 #if EFSYS_OPT_RIVERHEAD || EFX_OPTS_EF10() 12 13 #if EFSYS_OPT_RX_SCALE 14 static __checkReturn efx_rc_t 15 efx_mcdi_rss_context_alloc( 16 __in efx_nic_t *enp, 17 __in efx_rx_scale_context_type_t type, 18 __in uint32_t num_queues, 19 __out uint32_t *rss_contextp) 20 { 21 efx_mcdi_req_t req; 22 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN, 23 MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN); 24 uint32_t rss_context; 25 uint32_t context_type; 26 efx_rc_t rc; 27 28 if (num_queues > EFX_MAXRSS) { 29 rc = EINVAL; 30 goto fail1; 31 } 32 33 switch (type) { 34 case EFX_RX_SCALE_EXCLUSIVE: 35 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_EXCLUSIVE; 36 break; 37 case EFX_RX_SCALE_SHARED: 38 context_type = MC_CMD_RSS_CONTEXT_ALLOC_IN_TYPE_SHARED; 39 break; 40 default: 41 rc = EINVAL; 42 goto fail2; 43 } 44 45 req.emr_cmd = MC_CMD_RSS_CONTEXT_ALLOC; 46 req.emr_in_buf = payload; 47 req.emr_in_length = MC_CMD_RSS_CONTEXT_ALLOC_IN_LEN; 48 req.emr_out_buf = payload; 49 req.emr_out_length = MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN; 50 51 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_UPSTREAM_PORT_ID, 52 enp->en_vport_id); 53 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_TYPE, context_type); 54 55 /* 56 * For exclusive contexts, NUM_QUEUES is only used to validate 57 * indirection table offsets. 58 * For shared contexts, the provided context will spread traffic over 59 * NUM_QUEUES many queues. 60 */ 61 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_ALLOC_IN_NUM_QUEUES, num_queues); 62 63 efx_mcdi_execute(enp, &req); 64 65 if (req.emr_rc != 0) { 66 rc = req.emr_rc; 67 goto fail3; 68 } 69 70 if (req.emr_out_length_used < MC_CMD_RSS_CONTEXT_ALLOC_OUT_LEN) { 71 rc = EMSGSIZE; 72 goto fail4; 73 } 74 75 rss_context = MCDI_OUT_DWORD(req, RSS_CONTEXT_ALLOC_OUT_RSS_CONTEXT_ID); 76 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 77 rc = ENOENT; 78 goto fail5; 79 } 80 81 *rss_contextp = rss_context; 82 83 return (0); 84 85 fail5: 86 EFSYS_PROBE(fail5); 87 fail4: 88 EFSYS_PROBE(fail4); 89 fail3: 90 EFSYS_PROBE(fail3); 91 fail2: 92 EFSYS_PROBE(fail2); 93 fail1: 94 EFSYS_PROBE1(fail1, efx_rc_t, rc); 95 96 return (rc); 97 } 98 #endif /* EFSYS_OPT_RX_SCALE */ 99 100 #if EFSYS_OPT_RX_SCALE 101 static efx_rc_t 102 efx_mcdi_rss_context_free( 103 __in efx_nic_t *enp, 104 __in uint32_t rss_context) 105 { 106 efx_mcdi_req_t req; 107 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_FREE_IN_LEN, 108 MC_CMD_RSS_CONTEXT_FREE_OUT_LEN); 109 efx_rc_t rc; 110 111 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 112 rc = EINVAL; 113 goto fail1; 114 } 115 116 req.emr_cmd = MC_CMD_RSS_CONTEXT_FREE; 117 req.emr_in_buf = payload; 118 req.emr_in_length = MC_CMD_RSS_CONTEXT_FREE_IN_LEN; 119 req.emr_out_buf = payload; 120 req.emr_out_length = MC_CMD_RSS_CONTEXT_FREE_OUT_LEN; 121 122 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_FREE_IN_RSS_CONTEXT_ID, rss_context); 123 124 efx_mcdi_execute_quiet(enp, &req); 125 126 if (req.emr_rc != 0) { 127 rc = req.emr_rc; 128 goto fail2; 129 } 130 131 return (0); 132 133 fail2: 134 EFSYS_PROBE(fail2); 135 fail1: 136 EFSYS_PROBE1(fail1, efx_rc_t, rc); 137 138 return (rc); 139 } 140 #endif /* EFSYS_OPT_RX_SCALE */ 141 142 #if EFSYS_OPT_RX_SCALE 143 static efx_rc_t 144 efx_mcdi_rss_context_set_flags( 145 __in efx_nic_t *enp, 146 __in uint32_t rss_context, 147 __in efx_rx_hash_type_t type) 148 { 149 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 150 efx_mcdi_req_t req; 151 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN, 152 MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN); 153 efx_rc_t rc; 154 155 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_LBN == 156 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_LBN); 157 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_TCP_WIDTH == 158 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE_WIDTH); 159 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_LBN == 160 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_LBN); 161 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV4_WIDTH == 162 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE_WIDTH); 163 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_LBN == 164 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_LBN); 165 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_TCP_WIDTH == 166 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE_WIDTH); 167 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_LBN == 168 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_LBN); 169 EFX_STATIC_ASSERT(EFX_RX_CLASS_IPV6_WIDTH == 170 MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE_WIDTH); 171 172 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 173 rc = EINVAL; 174 goto fail1; 175 } 176 177 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_FLAGS; 178 req.emr_in_buf = payload; 179 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_IN_LEN; 180 req.emr_out_buf = payload; 181 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_FLAGS_OUT_LEN; 182 183 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_FLAGS_IN_RSS_CONTEXT_ID, 184 rss_context); 185 186 /* 187 * If the firmware lacks support for additional modes, RSS_MODE 188 * fields must contain zeros, otherwise the operation will fail. 189 */ 190 if (encp->enc_rx_scale_additional_modes_supported == B_FALSE) 191 type &= EFX_RX_HASH_LEGACY_MASK; 192 193 MCDI_IN_POPULATE_DWORD_10(req, RSS_CONTEXT_SET_FLAGS_IN_FLAGS, 194 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV4_EN, 195 (type & EFX_RX_HASH_IPV4) ? 1 : 0, 196 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV4_EN, 197 (type & EFX_RX_HASH_TCPIPV4) ? 1 : 0, 198 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_IPV6_EN, 199 (type & EFX_RX_HASH_IPV6) ? 1 : 0, 200 RSS_CONTEXT_SET_FLAGS_IN_TOEPLITZ_TCPV6_EN, 201 (type & EFX_RX_HASH_TCPIPV6) ? 1 : 0, 202 RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV4_RSS_MODE, 203 (type >> EFX_RX_CLASS_IPV4_TCP_LBN) & 204 EFX_MASK32(EFX_RX_CLASS_IPV4_TCP), 205 RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV4_RSS_MODE, 206 (type >> EFX_RX_CLASS_IPV4_UDP_LBN) & 207 EFX_MASK32(EFX_RX_CLASS_IPV4_UDP), 208 RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV4_RSS_MODE, 209 (type >> EFX_RX_CLASS_IPV4_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV4), 210 RSS_CONTEXT_SET_FLAGS_IN_TCP_IPV6_RSS_MODE, 211 (type >> EFX_RX_CLASS_IPV6_TCP_LBN) & 212 EFX_MASK32(EFX_RX_CLASS_IPV6_TCP), 213 RSS_CONTEXT_SET_FLAGS_IN_UDP_IPV6_RSS_MODE, 214 (type >> EFX_RX_CLASS_IPV6_UDP_LBN) & 215 EFX_MASK32(EFX_RX_CLASS_IPV6_UDP), 216 RSS_CONTEXT_SET_FLAGS_IN_OTHER_IPV6_RSS_MODE, 217 (type >> EFX_RX_CLASS_IPV6_LBN) & EFX_MASK32(EFX_RX_CLASS_IPV6)); 218 219 efx_mcdi_execute(enp, &req); 220 221 if (req.emr_rc != 0) { 222 rc = req.emr_rc; 223 goto fail2; 224 } 225 226 return (0); 227 228 fail2: 229 EFSYS_PROBE(fail2); 230 fail1: 231 EFSYS_PROBE1(fail1, efx_rc_t, rc); 232 233 return (rc); 234 } 235 #endif /* EFSYS_OPT_RX_SCALE */ 236 237 #if EFSYS_OPT_RX_SCALE 238 static efx_rc_t 239 efx_mcdi_rss_context_set_key( 240 __in efx_nic_t *enp, 241 __in uint32_t rss_context, 242 __in_ecount(n) uint8_t *key, 243 __in size_t n) 244 { 245 efx_mcdi_req_t req; 246 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN, 247 MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN); 248 efx_rc_t rc; 249 250 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 251 rc = EINVAL; 252 goto fail1; 253 } 254 255 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_KEY; 256 req.emr_in_buf = payload; 257 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_KEY_IN_LEN; 258 req.emr_out_buf = payload; 259 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_KEY_OUT_LEN; 260 261 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_KEY_IN_RSS_CONTEXT_ID, 262 rss_context); 263 264 EFSYS_ASSERT3U(n, ==, MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 265 if (n != MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN) { 266 rc = EINVAL; 267 goto fail2; 268 } 269 270 memcpy(MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY), 271 key, n); 272 273 efx_mcdi_execute(enp, &req); 274 275 if (req.emr_rc != 0) { 276 rc = req.emr_rc; 277 goto fail3; 278 } 279 280 return (0); 281 282 fail3: 283 EFSYS_PROBE(fail3); 284 fail2: 285 EFSYS_PROBE(fail2); 286 fail1: 287 EFSYS_PROBE1(fail1, efx_rc_t, rc); 288 289 return (rc); 290 } 291 #endif /* EFSYS_OPT_RX_SCALE */ 292 293 #if EFSYS_OPT_RX_SCALE 294 static efx_rc_t 295 efx_mcdi_rss_context_set_table( 296 __in efx_nic_t *enp, 297 __in uint32_t rss_context, 298 __in_ecount(n) unsigned int *table, 299 __in size_t n) 300 { 301 efx_mcdi_req_t req; 302 EFX_MCDI_DECLARE_BUF(payload, MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN, 303 MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN); 304 uint8_t *req_table; 305 int i, rc; 306 307 if (rss_context == EF10_RSS_CONTEXT_INVALID) { 308 rc = EINVAL; 309 goto fail1; 310 } 311 312 req.emr_cmd = MC_CMD_RSS_CONTEXT_SET_TABLE; 313 req.emr_in_buf = payload; 314 req.emr_in_length = MC_CMD_RSS_CONTEXT_SET_TABLE_IN_LEN; 315 req.emr_out_buf = payload; 316 req.emr_out_length = MC_CMD_RSS_CONTEXT_SET_TABLE_OUT_LEN; 317 318 MCDI_IN_SET_DWORD(req, RSS_CONTEXT_SET_TABLE_IN_RSS_CONTEXT_ID, 319 rss_context); 320 321 req_table = 322 MCDI_IN2(req, uint8_t, RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE); 323 324 for (i = 0; 325 i < MC_CMD_RSS_CONTEXT_SET_TABLE_IN_INDIRECTION_TABLE_LEN; 326 i++) { 327 req_table[i] = (n > 0) ? (uint8_t)table[i % n] : 0; 328 } 329 330 efx_mcdi_execute(enp, &req); 331 332 if (req.emr_rc != 0) { 333 rc = req.emr_rc; 334 goto fail2; 335 } 336 337 return (0); 338 339 fail2: 340 EFSYS_PROBE(fail2); 341 fail1: 342 EFSYS_PROBE1(fail1, efx_rc_t, rc); 343 344 return (rc); 345 } 346 #endif /* EFSYS_OPT_RX_SCALE */ 347 348 349 __checkReturn efx_rc_t 350 ef10_rx_init( 351 __in efx_nic_t *enp) 352 { 353 #if EFSYS_OPT_RX_SCALE 354 355 if (efx_mcdi_rss_context_alloc(enp, EFX_RX_SCALE_EXCLUSIVE, EFX_MAXRSS, 356 &enp->en_rss_context) == 0) { 357 /* 358 * Allocated an exclusive RSS context, which allows both the 359 * indirection table and key to be modified. 360 */ 361 enp->en_rss_context_type = EFX_RX_SCALE_EXCLUSIVE; 362 enp->en_hash_support = EFX_RX_HASH_AVAILABLE; 363 } else { 364 /* 365 * Failed to allocate an exclusive RSS context. Continue 366 * operation without support for RSS. The pseudo-header in 367 * received packets will not contain a Toeplitz hash value. 368 */ 369 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; 370 enp->en_hash_support = EFX_RX_HASH_UNAVAILABLE; 371 } 372 373 #endif /* EFSYS_OPT_RX_SCALE */ 374 375 return (0); 376 } 377 378 #if EFX_OPTS_EF10() 379 380 #if EFSYS_OPT_RX_SCATTER 381 __checkReturn efx_rc_t 382 ef10_rx_scatter_enable( 383 __in efx_nic_t *enp, 384 __in unsigned int buf_size) 385 { 386 _NOTE(ARGUNUSED(enp, buf_size)) 387 return (0); 388 } 389 #endif /* EFSYS_OPT_RX_SCATTER */ 390 391 #endif /* EFX_OPTS_EF10() */ 392 393 #if EFSYS_OPT_RX_SCALE 394 __checkReturn efx_rc_t 395 ef10_rx_scale_context_alloc( 396 __in efx_nic_t *enp, 397 __in efx_rx_scale_context_type_t type, 398 __in uint32_t num_queues, 399 __out uint32_t *rss_contextp) 400 { 401 efx_rc_t rc; 402 403 rc = efx_mcdi_rss_context_alloc(enp, type, num_queues, rss_contextp); 404 if (rc != 0) 405 goto fail1; 406 407 return (0); 408 409 fail1: 410 EFSYS_PROBE1(fail1, efx_rc_t, rc); 411 return (rc); 412 } 413 #endif /* EFSYS_OPT_RX_SCALE */ 414 415 #if EFSYS_OPT_RX_SCALE 416 __checkReturn efx_rc_t 417 ef10_rx_scale_context_free( 418 __in efx_nic_t *enp, 419 __in uint32_t rss_context) 420 { 421 efx_rc_t rc; 422 423 rc = efx_mcdi_rss_context_free(enp, rss_context); 424 if (rc != 0) 425 goto fail1; 426 427 return (0); 428 429 fail1: 430 EFSYS_PROBE1(fail1, efx_rc_t, rc); 431 return (rc); 432 } 433 #endif /* EFSYS_OPT_RX_SCALE */ 434 435 #if EFSYS_OPT_RX_SCALE 436 __checkReturn efx_rc_t 437 ef10_rx_scale_mode_set( 438 __in efx_nic_t *enp, 439 __in uint32_t rss_context, 440 __in efx_rx_hash_alg_t alg, 441 __in efx_rx_hash_type_t type, 442 __in boolean_t insert) 443 { 444 efx_nic_cfg_t *encp = &enp->en_nic_cfg; 445 efx_rc_t rc; 446 447 EFSYS_ASSERT3U(insert, ==, B_TRUE); 448 449 if ((encp->enc_rx_scale_hash_alg_mask & (1U << alg)) == 0 || 450 insert == B_FALSE) { 451 rc = EINVAL; 452 goto fail1; 453 } 454 455 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 456 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 457 rc = ENOTSUP; 458 goto fail2; 459 } 460 rss_context = enp->en_rss_context; 461 } 462 463 if ((rc = efx_mcdi_rss_context_set_flags(enp, 464 rss_context, type)) != 0) 465 goto fail3; 466 467 return (0); 468 469 fail3: 470 EFSYS_PROBE(fail3); 471 fail2: 472 EFSYS_PROBE(fail2); 473 fail1: 474 EFSYS_PROBE1(fail1, efx_rc_t, rc); 475 476 return (rc); 477 } 478 #endif /* EFSYS_OPT_RX_SCALE */ 479 480 #if EFSYS_OPT_RX_SCALE 481 __checkReturn efx_rc_t 482 ef10_rx_scale_key_set( 483 __in efx_nic_t *enp, 484 __in uint32_t rss_context, 485 __in_ecount(n) uint8_t *key, 486 __in size_t n) 487 { 488 efx_rc_t rc; 489 490 EFX_STATIC_ASSERT(EFX_RSS_KEY_SIZE == 491 MC_CMD_RSS_CONTEXT_SET_KEY_IN_TOEPLITZ_KEY_LEN); 492 493 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 494 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 495 rc = ENOTSUP; 496 goto fail1; 497 } 498 rss_context = enp->en_rss_context; 499 } 500 501 if ((rc = efx_mcdi_rss_context_set_key(enp, rss_context, key, n)) != 0) 502 goto fail2; 503 504 return (0); 505 506 fail2: 507 EFSYS_PROBE(fail2); 508 fail1: 509 EFSYS_PROBE1(fail1, efx_rc_t, rc); 510 511 return (rc); 512 } 513 #endif /* EFSYS_OPT_RX_SCALE */ 514 515 #if EFSYS_OPT_RX_SCALE 516 __checkReturn efx_rc_t 517 ef10_rx_scale_tbl_set( 518 __in efx_nic_t *enp, 519 __in uint32_t rss_context, 520 __in_ecount(n) unsigned int *table, 521 __in size_t n) 522 { 523 efx_rc_t rc; 524 525 526 if (rss_context == EFX_RSS_CONTEXT_DEFAULT) { 527 if (enp->en_rss_context_type == EFX_RX_SCALE_UNAVAILABLE) { 528 rc = ENOTSUP; 529 goto fail1; 530 } 531 rss_context = enp->en_rss_context; 532 } 533 534 if ((rc = efx_mcdi_rss_context_set_table(enp, 535 rss_context, table, n)) != 0) 536 goto fail2; 537 538 return (0); 539 540 fail2: 541 EFSYS_PROBE(fail2); 542 fail1: 543 EFSYS_PROBE1(fail1, efx_rc_t, rc); 544 545 return (rc); 546 } 547 #endif /* EFSYS_OPT_RX_SCALE */ 548 549 #if EFX_OPTS_EF10() 550 551 /* 552 * EF10 RX pseudo-header (aka Rx prefix) 553 * ------------------------------------- 554 * 555 * Receive packets are prefixed by an (optional) 14 byte pseudo-header: 556 * 557 * +00: Toeplitz hash value. 558 * (32bit little-endian) 559 * +04: Outer VLAN tag. Zero if the packet did not have an outer VLAN tag. 560 * (16bit big-endian) 561 * +06: Inner VLAN tag. Zero if the packet did not have an inner VLAN tag. 562 * (16bit big-endian) 563 * +08: Packet Length. Zero if the RX datapath was in cut-through mode. 564 * (16bit little-endian) 565 * +10: MAC timestamp. Zero if timestamping is not enabled. 566 * (32bit little-endian) 567 * 568 * See "The RX Pseudo-header" in SF-109306-TC. 569 * 570 * EF10 does not support Rx prefix choice using MC_CMD_GET_RX_PREFIX_ID 571 * and query its layout using MC_CMD_QUERY_RX_PREFIX_ID. 572 */ 573 static const efx_rx_prefix_layout_t ef10_default_rx_prefix_layout = { 574 .erpl_id = 0, 575 .erpl_length = 14, 576 .erpl_fields = { 577 [EFX_RX_PREFIX_FIELD_RSS_HASH] = 578 { 0, 32, B_FALSE }, 579 [EFX_RX_PREFIX_FIELD_VLAN_STRIP_TCI] = 580 { 32, 16, B_TRUE }, 581 [EFX_RX_PREFIX_FIELD_INNER_VLAN_STRIP_TCI] = 582 { 48, 16, B_TRUE }, 583 [EFX_RX_PREFIX_FIELD_LENGTH] = 584 { 64, 16, B_FALSE }, 585 [EFX_RX_PREFIX_FIELD_PARTIAL_TSTAMP] = 586 { 80, 32, B_FALSE }, 587 } 588 }; 589 590 #if EFSYS_OPT_RX_PACKED_STREAM 591 592 /* 593 * EF10 packed stream Rx prefix layout. 594 * 595 * See SF-112241-TC Full speed capture for Huntington and Medford section 4.5. 596 */ 597 static const efx_rx_prefix_layout_t ef10_packed_stream_rx_prefix_layout = { 598 .erpl_id = 0, 599 .erpl_length = 8, 600 .erpl_fields = { 601 #define EF10_PS_RX_PREFIX_FIELD(_efx, _ef10) \ 602 EFX_RX_PREFIX_FIELD(_efx, ES_DZ_PS_RX_PREFIX_ ## _ef10, B_FALSE) 603 604 EF10_PS_RX_PREFIX_FIELD(PARTIAL_TSTAMP, TSTAMP), 605 EF10_PS_RX_PREFIX_FIELD(LENGTH, CAP_LEN), 606 EF10_PS_RX_PREFIX_FIELD(ORIG_LENGTH, ORIG_LEN), 607 608 #undef EF10_PS_RX_PREFIX_FIELD 609 } 610 }; 611 612 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 613 614 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 615 616 /* 617 * EF10 equal stride super-buffer Rx prefix layout. 618 * 619 * See SF-119419-TC DPDK Firmware Driver Interface section 3.4. 620 */ 621 static const efx_rx_prefix_layout_t ef10_essb_rx_prefix_layout = { 622 .erpl_id = 0, 623 .erpl_length = ES_EZ_ESSB_RX_PREFIX_LEN, 624 .erpl_fields = { 625 #define EF10_ESSB_RX_PREFIX_FIELD(_efx, _ef10) \ 626 EFX_RX_PREFIX_FIELD(_efx, ES_EZ_ESSB_RX_PREFIX_ ## _ef10, B_FALSE) 627 628 EF10_ESSB_RX_PREFIX_FIELD(LENGTH, DATA_LEN), 629 EF10_ESSB_RX_PREFIX_FIELD(USER_MARK, MARK), 630 EF10_ESSB_RX_PREFIX_FIELD(RSS_HASH_VALID, HASH_VALID), 631 EF10_ESSB_RX_PREFIX_FIELD(USER_MARK_VALID, MARK_VALID), 632 EF10_ESSB_RX_PREFIX_FIELD(USER_FLAG, MATCH_FLAG), 633 EF10_ESSB_RX_PREFIX_FIELD(RSS_HASH, HASH), 634 635 #undef EF10_ESSB_RX_PREFIX_FIELD 636 } 637 }; 638 639 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 640 641 __checkReturn efx_rc_t 642 ef10_rx_prefix_pktlen( 643 __in efx_nic_t *enp, 644 __in uint8_t *buffer, 645 __out uint16_t *lengthp) 646 { 647 _NOTE(ARGUNUSED(enp)) 648 649 /* 650 * The RX pseudo-header contains the packet length, excluding the 651 * pseudo-header. If the hardware receive datapath was operating in 652 * cut-through mode then the length in the RX pseudo-header will be 653 * zero, and the packet length must be obtained from the DMA length 654 * reported in the RX event. 655 */ 656 *lengthp = buffer[8] | (buffer[9] << 8); 657 return (0); 658 } 659 660 #if EFSYS_OPT_RX_SCALE 661 __checkReturn uint32_t 662 ef10_rx_prefix_hash( 663 __in efx_nic_t *enp, 664 __in efx_rx_hash_alg_t func, 665 __in uint8_t *buffer) 666 { 667 _NOTE(ARGUNUSED(enp)) 668 669 switch (func) { 670 case EFX_RX_HASHALG_PACKED_STREAM: 671 case EFX_RX_HASHALG_TOEPLITZ: 672 return (buffer[0] | 673 (buffer[1] << 8) | 674 (buffer[2] << 16) | 675 (buffer[3] << 24)); 676 677 default: 678 EFSYS_ASSERT(0); 679 return (0); 680 } 681 } 682 #endif /* EFSYS_OPT_RX_SCALE */ 683 684 #if EFSYS_OPT_RX_PACKED_STREAM 685 /* 686 * Fake length for RXQ descriptors in packed stream mode 687 * to make hardware happy 688 */ 689 #define EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE 32 690 #endif 691 692 void 693 ef10_rx_qpost( 694 __in efx_rxq_t *erp, 695 __in_ecount(ndescs) efsys_dma_addr_t *addrp, 696 __in size_t size, 697 __in unsigned int ndescs, 698 __in unsigned int completed, 699 __in unsigned int added) 700 { 701 efx_qword_t qword; 702 unsigned int i; 703 unsigned int offset; 704 unsigned int id; 705 706 _NOTE(ARGUNUSED(completed)) 707 708 #if EFSYS_OPT_RX_PACKED_STREAM 709 /* 710 * Real size of the buffer does not fit into ESF_DZ_RX_KER_BYTE_CNT 711 * and equal to 0 after applying mask. Hardware does not like it. 712 */ 713 if (erp->er_ev_qstate->eers_rx_packed_stream) 714 size = EFX_RXQ_PACKED_STREAM_FAKE_BUF_SIZE; 715 #endif 716 717 /* The client driver must not overfill the queue */ 718 EFSYS_ASSERT3U(added - completed + ndescs, <=, 719 EFX_RXQ_LIMIT(erp->er_mask + 1)); 720 721 id = added & (erp->er_mask); 722 for (i = 0; i < ndescs; i++) { 723 EFSYS_PROBE4(rx_post, unsigned int, erp->er_index, 724 unsigned int, id, efsys_dma_addr_t, addrp[i], 725 size_t, size); 726 727 EFX_POPULATE_QWORD_3(qword, 728 ESF_DZ_RX_KER_BYTE_CNT, (uint32_t)(size), 729 ESF_DZ_RX_KER_BUF_ADDR_DW0, 730 (uint32_t)(addrp[i] & 0xffffffff), 731 ESF_DZ_RX_KER_BUF_ADDR_DW1, 732 (uint32_t)(addrp[i] >> 32)); 733 734 offset = id * sizeof (efx_qword_t); 735 EFSYS_MEM_WRITEQ(erp->er_esmp, offset, &qword); 736 737 id = (id + 1) & (erp->er_mask); 738 } 739 } 740 741 void 742 ef10_rx_qpush( 743 __in efx_rxq_t *erp, 744 __in unsigned int added, 745 __inout unsigned int *pushedp) 746 { 747 efx_nic_t *enp = erp->er_enp; 748 unsigned int pushed = *pushedp; 749 uint32_t wptr; 750 efx_dword_t dword; 751 752 /* Hardware has alignment restriction for WPTR */ 753 wptr = EFX_P2ALIGN(unsigned int, added, EF10_RX_WPTR_ALIGN); 754 if (pushed == wptr) 755 return; 756 757 *pushedp = wptr; 758 759 /* Push the populated descriptors out */ 760 wptr &= erp->er_mask; 761 762 EFX_POPULATE_DWORD_1(dword, ERF_DZ_RX_DESC_WPTR, wptr); 763 764 /* Guarantee ordering of memory (descriptors) and PIO (doorbell) */ 765 EFX_DMA_SYNC_QUEUE_FOR_DEVICE(erp->er_esmp, erp->er_mask + 1, 766 EF10_RXQ_DESC_SIZE, wptr, pushed & erp->er_mask); 767 EFSYS_PIO_WRITE_BARRIER(); 768 EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 769 erp->er_index, &dword, B_FALSE); 770 } 771 772 #if EFSYS_OPT_RX_PACKED_STREAM 773 774 void 775 ef10_rx_qpush_ps_credits( 776 __in efx_rxq_t *erp) 777 { 778 efx_nic_t *enp = erp->er_enp; 779 efx_dword_t dword; 780 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; 781 uint32_t credits; 782 783 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); 784 785 if (rxq_state->eers_rx_packed_stream_credits == 0) 786 return; 787 788 /* 789 * It is a bug if we think that FW has utilized more 790 * credits than it is allowed to have (maximum). However, 791 * make sure that we do not credit more than maximum anyway. 792 */ 793 credits = MIN(rxq_state->eers_rx_packed_stream_credits, 794 EFX_RX_PACKED_STREAM_MAX_CREDITS); 795 EFX_POPULATE_DWORD_3(dword, 796 ERF_DZ_RX_DESC_MAGIC_DOORBELL, 1, 797 ERF_DZ_RX_DESC_MAGIC_CMD, 798 ERE_DZ_RX_DESC_MAGIC_CMD_PS_CREDITS, 799 ERF_DZ_RX_DESC_MAGIC_DATA, credits); 800 EFX_BAR_VI_WRITED(enp, ER_DZ_RX_DESC_UPD_REG, 801 erp->er_index, &dword, B_FALSE); 802 803 rxq_state->eers_rx_packed_stream_credits = 0; 804 } 805 806 /* 807 * In accordance with SF-112241-TC the received data has the following layout: 808 * - 8 byte pseudo-header which consist of: 809 * - 4 byte little-endian timestamp 810 * - 2 byte little-endian captured length in bytes 811 * - 2 byte little-endian original packet length in bytes 812 * - captured packet bytes 813 * - optional padding to align to 64 bytes boundary 814 * - 64 bytes scratch space for the host software 815 */ 816 __checkReturn uint8_t * 817 ef10_rx_qps_packet_info( 818 __in efx_rxq_t *erp, 819 __in uint8_t *buffer, 820 __in uint32_t buffer_length, 821 __in uint32_t current_offset, 822 __out uint16_t *lengthp, 823 __out uint32_t *next_offsetp, 824 __out uint32_t *timestamp) 825 { 826 uint16_t buf_len; 827 uint8_t *pkt_start; 828 efx_qword_t *qwordp; 829 efx_evq_rxq_state_t *rxq_state = erp->er_ev_qstate; 830 831 EFSYS_ASSERT(rxq_state->eers_rx_packed_stream); 832 833 buffer += current_offset; 834 pkt_start = buffer + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE; 835 836 qwordp = (efx_qword_t *)buffer; 837 *timestamp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_TSTAMP); 838 *lengthp = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_ORIG_LEN); 839 buf_len = EFX_QWORD_FIELD(*qwordp, ES_DZ_PS_RX_PREFIX_CAP_LEN); 840 841 buf_len = EFX_P2ROUNDUP(uint16_t, 842 buf_len + EFX_RX_PACKED_STREAM_RX_PREFIX_SIZE, 843 EFX_RX_PACKED_STREAM_ALIGNMENT); 844 *next_offsetp = 845 current_offset + buf_len + EFX_RX_PACKED_STREAM_ALIGNMENT; 846 847 EFSYS_ASSERT3U(*next_offsetp, <=, buffer_length); 848 EFSYS_ASSERT3U(current_offset + *lengthp, <, *next_offsetp); 849 850 if ((*next_offsetp ^ current_offset) & 851 EFX_RX_PACKED_STREAM_MEM_PER_CREDIT) 852 rxq_state->eers_rx_packed_stream_credits++; 853 854 return (pkt_start); 855 } 856 857 858 #endif 859 860 __checkReturn efx_rc_t 861 ef10_rx_qflush( 862 __in efx_rxq_t *erp) 863 { 864 efx_nic_t *enp = erp->er_enp; 865 efx_rc_t rc; 866 867 if ((rc = efx_mcdi_fini_rxq(enp, erp->er_index)) != 0) 868 goto fail1; 869 870 return (0); 871 872 fail1: 873 /* 874 * EALREADY is not an error, but indicates that the MC has rebooted and 875 * that the RXQ has already been destroyed. Callers need to know that 876 * the RXQ flush has completed to avoid waiting until timeout for a 877 * flush done event that will not be delivered. 878 */ 879 if (rc != EALREADY) 880 EFSYS_PROBE1(fail1, efx_rc_t, rc); 881 882 return (rc); 883 } 884 885 void 886 ef10_rx_qenable( 887 __in efx_rxq_t *erp) 888 { 889 /* FIXME */ 890 _NOTE(ARGUNUSED(erp)) 891 /* FIXME */ 892 } 893 894 __checkReturn efx_rc_t 895 ef10_rx_qcreate( 896 __in efx_nic_t *enp, 897 __in unsigned int index, 898 __in unsigned int label, 899 __in efx_rxq_type_t type, 900 __in_opt const efx_rxq_type_data_t *type_data, 901 __in efsys_mem_t *esmp, 902 __in size_t ndescs, 903 __in uint32_t id, 904 __in unsigned int flags, 905 __in efx_evq_t *eep, 906 __in efx_rxq_t *erp) 907 { 908 efx_nic_cfg_t *encp = &(enp->en_nic_cfg); 909 efx_mcdi_init_rxq_params_t params; 910 const efx_rx_prefix_layout_t *erpl; 911 efx_rc_t rc; 912 913 _NOTE(ARGUNUSED(id, erp)) 914 915 EFX_STATIC_ASSERT(EFX_EV_RX_NLABELS == (1 << ESF_DZ_RX_QLABEL_WIDTH)); 916 EFSYS_ASSERT3U(label, <, EFX_EV_RX_NLABELS); 917 918 memset(¶ms, 0, sizeof (params)); 919 params.buf_size = erp->er_buf_size; 920 921 switch (type) { 922 case EFX_RXQ_TYPE_DEFAULT: 923 erpl = &ef10_default_rx_prefix_layout; 924 if (type_data == NULL) { 925 rc = EINVAL; 926 goto fail1; 927 } 928 erp->er_buf_size = type_data->ertd_default.ed_buf_size; 929 if (flags & EFX_RXQ_FLAG_USER_MARK) { 930 rc = ENOTSUP; 931 goto fail2; 932 } 933 if (flags & EFX_RXQ_FLAG_USER_FLAG) { 934 rc = ENOTSUP; 935 goto fail3; 936 } 937 /* 938 * Ignore EFX_RXQ_FLAG_RSS_HASH since if RSS hash is calculated 939 * it is always delivered from HW in the pseudo-header. 940 */ 941 break; 942 #if EFSYS_OPT_RX_PACKED_STREAM 943 case EFX_RXQ_TYPE_PACKED_STREAM: 944 erpl = &ef10_packed_stream_rx_prefix_layout; 945 if (type_data == NULL) { 946 rc = EINVAL; 947 goto fail4; 948 } 949 switch (type_data->ertd_packed_stream.eps_buf_size) { 950 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_1M: 951 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M; 952 break; 953 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_512K: 954 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_512K; 955 break; 956 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_256K: 957 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_256K; 958 break; 959 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_128K: 960 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_128K; 961 break; 962 case EFX_RXQ_PACKED_STREAM_BUF_SIZE_64K: 963 params.ps_buf_size = MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_64K; 964 break; 965 default: 966 rc = ENOTSUP; 967 goto fail5; 968 } 969 erp->er_buf_size = type_data->ertd_packed_stream.eps_buf_size; 970 /* Packed stream pseudo header does not have RSS hash value */ 971 if (flags & EFX_RXQ_FLAG_RSS_HASH) { 972 rc = ENOTSUP; 973 goto fail6; 974 } 975 if (flags & EFX_RXQ_FLAG_USER_MARK) { 976 rc = ENOTSUP; 977 goto fail7; 978 } 979 if (flags & EFX_RXQ_FLAG_USER_FLAG) { 980 rc = ENOTSUP; 981 goto fail8; 982 } 983 break; 984 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 985 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 986 case EFX_RXQ_TYPE_ES_SUPER_BUFFER: 987 erpl = &ef10_essb_rx_prefix_layout; 988 if (type_data == NULL) { 989 rc = EINVAL; 990 goto fail9; 991 } 992 params.es_bufs_per_desc = 993 type_data->ertd_es_super_buffer.eessb_bufs_per_desc; 994 params.es_max_dma_len = 995 type_data->ertd_es_super_buffer.eessb_max_dma_len; 996 params.es_buf_stride = 997 type_data->ertd_es_super_buffer.eessb_buf_stride; 998 params.hol_block_timeout = 999 type_data->ertd_es_super_buffer.eessb_hol_block_timeout; 1000 /* 1001 * Ignore EFX_RXQ_FLAG_RSS_HASH since if RSS hash is calculated 1002 * it is always delivered from HW in the pseudo-header. 1003 */ 1004 break; 1005 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1006 default: 1007 rc = ENOTSUP; 1008 goto fail10; 1009 } 1010 1011 #if EFSYS_OPT_RX_PACKED_STREAM 1012 if (params.ps_buf_size != 0) { 1013 /* Check if datapath firmware supports packed stream mode */ 1014 if (encp->enc_rx_packed_stream_supported == B_FALSE) { 1015 rc = ENOTSUP; 1016 goto fail11; 1017 } 1018 /* Check if packed stream allows configurable buffer sizes */ 1019 if ((params.ps_buf_size != MC_CMD_INIT_RXQ_EXT_IN_PS_BUFF_1M) && 1020 (encp->enc_rx_var_packed_stream_supported == B_FALSE)) { 1021 rc = ENOTSUP; 1022 goto fail12; 1023 } 1024 } 1025 #else /* EFSYS_OPT_RX_PACKED_STREAM */ 1026 EFSYS_ASSERT(params.ps_buf_size == 0); 1027 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1028 1029 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 1030 if (params.es_bufs_per_desc > 0) { 1031 if (encp->enc_rx_es_super_buffer_supported == B_FALSE) { 1032 rc = ENOTSUP; 1033 goto fail13; 1034 } 1035 if (!EFX_IS_P2ALIGNED(uint32_t, params.es_max_dma_len, 1036 EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) { 1037 rc = EINVAL; 1038 goto fail14; 1039 } 1040 if (!EFX_IS_P2ALIGNED(uint32_t, params.es_buf_stride, 1041 EFX_RX_ES_SUPER_BUFFER_BUF_ALIGNMENT)) { 1042 rc = EINVAL; 1043 goto fail15; 1044 } 1045 } 1046 #else /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1047 EFSYS_ASSERT(params.es_bufs_per_desc == 0); 1048 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1049 1050 if (flags & EFX_RXQ_FLAG_INGRESS_MPORT) { 1051 rc = ENOTSUP; 1052 goto fail16; 1053 } 1054 1055 /* Scatter can only be disabled if the firmware supports doing so */ 1056 if (flags & EFX_RXQ_FLAG_SCATTER) 1057 params.disable_scatter = B_FALSE; 1058 else 1059 params.disable_scatter = encp->enc_rx_disable_scatter_supported; 1060 1061 if (flags & EFX_RXQ_FLAG_INNER_CLASSES) 1062 params.want_inner_classes = B_TRUE; 1063 else 1064 params.want_inner_classes = B_FALSE; 1065 1066 if ((rc = efx_mcdi_init_rxq(enp, ndescs, eep, label, index, 1067 esmp, ¶ms)) != 0) 1068 goto fail17; 1069 1070 erp->er_eep = eep; 1071 erp->er_label = label; 1072 1073 ef10_ev_rxlabel_init(eep, erp, label, type); 1074 1075 erp->er_ev_qstate = &erp->er_eep->ee_rxq_state[label]; 1076 1077 erp->er_prefix_layout = *erpl; 1078 1079 return (0); 1080 1081 fail17: 1082 EFSYS_PROBE(fail15); 1083 fail16: 1084 EFSYS_PROBE(fail14); 1085 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 1086 fail15: 1087 EFSYS_PROBE(fail15); 1088 fail14: 1089 EFSYS_PROBE(fail14); 1090 fail13: 1091 EFSYS_PROBE(fail13); 1092 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1093 #if EFSYS_OPT_RX_PACKED_STREAM 1094 fail12: 1095 EFSYS_PROBE(fail12); 1096 fail11: 1097 EFSYS_PROBE(fail11); 1098 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1099 fail10: 1100 EFSYS_PROBE(fail10); 1101 #if EFSYS_OPT_RX_ES_SUPER_BUFFER 1102 fail9: 1103 EFSYS_PROBE(fail9); 1104 #endif /* EFSYS_OPT_RX_ES_SUPER_BUFFER */ 1105 #if EFSYS_OPT_RX_PACKED_STREAM 1106 fail8: 1107 EFSYS_PROBE(fail8); 1108 fail7: 1109 EFSYS_PROBE(fail7); 1110 fail6: 1111 EFSYS_PROBE(fail6); 1112 fail5: 1113 EFSYS_PROBE(fail5); 1114 fail4: 1115 EFSYS_PROBE(fail4); 1116 #endif /* EFSYS_OPT_RX_PACKED_STREAM */ 1117 fail3: 1118 EFSYS_PROBE(fail3); 1119 fail2: 1120 EFSYS_PROBE(fail2); 1121 fail1: 1122 EFSYS_PROBE1(fail1, efx_rc_t, rc); 1123 1124 return (rc); 1125 } 1126 1127 void 1128 ef10_rx_qdestroy( 1129 __in efx_rxq_t *erp) 1130 { 1131 efx_evq_t *eep = erp->er_eep; 1132 unsigned int label = erp->er_label; 1133 1134 ef10_ev_rxlabel_fini(eep, label); 1135 } 1136 1137 #endif /* EFX_OPTS_EF10() */ 1138 1139 void 1140 ef10_rx_fini( 1141 __in efx_nic_t *enp) 1142 { 1143 #if EFSYS_OPT_RX_SCALE 1144 if (enp->en_rss_context_type != EFX_RX_SCALE_UNAVAILABLE) 1145 (void) efx_mcdi_rss_context_free(enp, enp->en_rss_context); 1146 enp->en_rss_context = 0; 1147 enp->en_rss_context_type = EFX_RX_SCALE_UNAVAILABLE; 1148 #else 1149 _NOTE(ARGUNUSED(enp)) 1150 #endif /* EFSYS_OPT_RX_SCALE */ 1151 } 1152 1153 #endif /* EFSYS_OPT_RIVERHEAD || EFX_OPTS_EF10() */ 1154