1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2009-2018 Microsoft Corp. 3 * Copyright (c) 2010-2012 Citrix Inc. 4 * Copyright (c) 2012 NetApp Inc. 5 * All rights reserved. 6 */ 7 8 #include <stdint.h> 9 #include <string.h> 10 #include <stdio.h> 11 #include <errno.h> 12 #include <unistd.h> 13 14 #include <rte_ethdev_driver.h> 15 #include <rte_ethdev.h> 16 #include <rte_string_fns.h> 17 #include <rte_memzone.h> 18 #include <rte_malloc.h> 19 #include <rte_atomic.h> 20 #include <rte_branch_prediction.h> 21 #include <rte_ether.h> 22 #include <rte_common.h> 23 #include <rte_errno.h> 24 #include <rte_cycles.h> 25 #include <rte_memory.h> 26 #include <rte_eal.h> 27 #include <rte_dev.h> 28 #include <rte_bus_vmbus.h> 29 30 #include "hn_logs.h" 31 #include "hn_var.h" 32 #include "hn_nvs.h" 33 #include "hn_rndis.h" 34 #include "ndis.h" 35 36 #define HN_RNDIS_XFER_SIZE 0x4000 37 38 #define HN_NDIS_TXCSUM_CAP_IP4 \ 39 (NDIS_TXCSUM_CAP_IP4 | NDIS_TXCSUM_CAP_IP4OPT) 40 #define HN_NDIS_TXCSUM_CAP_TCP4 \ 41 (NDIS_TXCSUM_CAP_TCP4 | NDIS_TXCSUM_CAP_TCP4OPT) 42 #define HN_NDIS_TXCSUM_CAP_TCP6 \ 43 (NDIS_TXCSUM_CAP_TCP6 | NDIS_TXCSUM_CAP_TCP6OPT | \ 44 NDIS_TXCSUM_CAP_IP6EXT) 45 #define HN_NDIS_TXCSUM_CAP_UDP6 \ 46 (NDIS_TXCSUM_CAP_UDP6 | NDIS_TXCSUM_CAP_IP6EXT) 47 #define HN_NDIS_LSOV2_CAP_IP6 \ 48 (NDIS_LSOV2_CAP_IP6EXT | NDIS_LSOV2_CAP_TCP6OPT) 49 50 /* Get unique request id */ 51 static inline uint32_t 52 hn_rndis_rid(struct hn_data *hv) 53 { 54 uint32_t rid; 55 56 do { 57 rid = rte_atomic32_add_return(&hv->rndis_req_id, 1); 58 } while (rid == 0); 59 60 return rid; 61 } 62 63 static void *hn_rndis_alloc(struct hn_data *hv, size_t size) 64 { 65 return rte_zmalloc_socket("RNDIS", size, PAGE_SIZE, 66 hv->vmbus->device.numa_node); 67 } 68 69 #ifdef RTE_LIBRTE_NETVSC_DEBUG_DUMP 70 void hn_rndis_dump(const void *buf) 71 { 72 const union { 73 struct rndis_msghdr hdr; 74 struct rndis_packet_msg pkt; 75 struct rndis_init_req init_request; 76 struct rndis_init_comp init_complete; 77 struct rndis_halt_req halt; 78 struct rndis_query_req query_request; 79 struct rndis_query_comp query_complete; 80 struct rndis_set_req set_request; 81 struct rndis_set_comp set_complete; 82 struct rndis_reset_req reset_request; 83 struct rndis_reset_comp reset_complete; 84 struct rndis_keepalive_req keepalive_request; 85 struct rndis_keepalive_comp keepalive_complete; 86 struct rndis_status_msg indicate_status; 87 } *rndis_msg = buf; 88 89 switch (rndis_msg->hdr.type) { 90 case RNDIS_PACKET_MSG: { 91 const struct rndis_pktinfo *ppi; 92 unsigned int ppi_len; 93 94 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 95 "RNDIS_MSG_PACKET (len %u, data %u:%u, # oob %u %u:%u, pkt %u:%u)\n", 96 rndis_msg->pkt.len, 97 rndis_msg->pkt.dataoffset, 98 rndis_msg->pkt.datalen, 99 rndis_msg->pkt.oobdataelements, 100 rndis_msg->pkt.oobdataoffset, 101 rndis_msg->pkt.oobdatalen, 102 rndis_msg->pkt.pktinfooffset, 103 rndis_msg->pkt.pktinfolen); 104 105 ppi = (const struct rndis_pktinfo *) 106 ((const char *)buf 107 + RNDIS_PACKET_MSG_OFFSET_ABS(rndis_msg->pkt.pktinfooffset)); 108 109 ppi_len = rndis_msg->pkt.pktinfolen; 110 while (ppi_len > 0) { 111 const void *ppi_data; 112 113 ppi_data = ppi->data; 114 115 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 116 " PPI (size %u, type %u, offs %u data %#x)\n", 117 ppi->size, ppi->type, ppi->offset, 118 *(const uint32_t *)ppi_data); 119 if (ppi->size == 0) 120 break; 121 ppi_len -= ppi->size; 122 ppi = (const struct rndis_pktinfo *) 123 ((const char *)ppi + ppi->size); 124 } 125 break; 126 } 127 case RNDIS_INITIALIZE_MSG: 128 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 129 "RNDIS_MSG_INIT (len %u id %#x, ver %u.%u max xfer %u)\n", 130 rndis_msg->init_request.len, 131 rndis_msg->init_request.rid, 132 rndis_msg->init_request.ver_major, 133 rndis_msg->init_request.ver_minor, 134 rndis_msg->init_request.max_xfersz); 135 break; 136 137 case RNDIS_INITIALIZE_CMPLT: 138 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 139 "RNDIS_MSG_INIT_C (len %u, id %#x, status 0x%x, vers %u.%u, " 140 "flags %d, max xfer %u, max pkts %u, aligned %u)\n", 141 rndis_msg->init_complete.len, 142 rndis_msg->init_complete.rid, 143 rndis_msg->init_complete.status, 144 rndis_msg->init_complete.ver_major, 145 rndis_msg->init_complete.ver_minor, 146 rndis_msg->init_complete.devflags, 147 rndis_msg->init_complete.pktmaxsz, 148 rndis_msg->init_complete.pktmaxcnt, 149 rndis_msg->init_complete.align); 150 break; 151 152 case RNDIS_HALT_MSG: 153 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 154 "RNDIS_HALT (len %u id %#x)\n", 155 rndis_msg->halt.len, rndis_msg->halt.rid); 156 break; 157 158 case RNDIS_QUERY_MSG: 159 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 160 "RNDIS_QUERY (len %u, id %#x, oid %#x, info %u:%u)\n", 161 rndis_msg->query_request.len, 162 rndis_msg->query_request.rid, 163 rndis_msg->query_request.oid, 164 rndis_msg->query_request.infobuflen, 165 rndis_msg->query_request.infobufoffset); 166 break; 167 168 case RNDIS_QUERY_CMPLT: 169 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 170 "RNDIS_MSG_QUERY_C (len %u, id %#x, status 0x%x, buf %u:%u)\n", 171 rndis_msg->query_complete.len, 172 rndis_msg->query_complete.rid, 173 rndis_msg->query_complete.status, 174 rndis_msg->query_complete.infobuflen, 175 rndis_msg->query_complete.infobufoffset); 176 break; 177 178 case RNDIS_SET_MSG: 179 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 180 "RNDIS_SET (len %u, id %#x, oid %#x, info %u:%u)\n", 181 rndis_msg->set_request.len, 182 rndis_msg->set_request.rid, 183 rndis_msg->set_request.oid, 184 rndis_msg->set_request.infobuflen, 185 rndis_msg->set_request.infobufoffset); 186 break; 187 188 case RNDIS_SET_CMPLT: 189 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 190 "RNDIS_MSG_SET_C (len %u, id 0x%x, status 0x%x)\n", 191 rndis_msg->set_complete.len, 192 rndis_msg->set_complete.rid, 193 rndis_msg->set_complete.status); 194 break; 195 196 case RNDIS_INDICATE_STATUS_MSG: 197 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 198 "RNDIS_MSG_INDICATE (len %u, status %#x, buf len %u, buf offset %u)\n", 199 rndis_msg->indicate_status.len, 200 rndis_msg->indicate_status.status, 201 rndis_msg->indicate_status.stbuflen, 202 rndis_msg->indicate_status.stbufoffset); 203 break; 204 205 case RNDIS_RESET_MSG: 206 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 207 "RNDIS_RESET (len %u, id %#x)\n", 208 rndis_msg->reset_request.len, 209 rndis_msg->reset_request.rid); 210 break; 211 212 case RNDIS_RESET_CMPLT: 213 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 214 "RNDIS_RESET_C (len %u, status %#x address %#x)\n", 215 rndis_msg->reset_complete.len, 216 rndis_msg->reset_complete.status, 217 rndis_msg->reset_complete.adrreset); 218 break; 219 220 case RNDIS_KEEPALIVE_MSG: 221 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 222 "RNDIS_KEEPALIVE (len %u, id %#x)\n", 223 rndis_msg->keepalive_request.len, 224 rndis_msg->keepalive_request.rid); 225 break; 226 227 case RNDIS_KEEPALIVE_CMPLT: 228 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 229 "RNDIS_KEEPALIVE_C (len %u, id %#x address %#x)\n", 230 rndis_msg->keepalive_complete.len, 231 rndis_msg->keepalive_complete.rid, 232 rndis_msg->keepalive_complete.status); 233 break; 234 235 default: 236 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 237 "RNDIS type %#x len %u\n", 238 rndis_msg->hdr.type, 239 rndis_msg->hdr.len); 240 break; 241 } 242 } 243 #endif 244 245 static int hn_nvs_send_rndis_ctrl(struct vmbus_channel *chan, 246 const void *req, uint32_t reqlen) 247 248 { 249 struct hn_nvs_rndis nvs_rndis = { 250 .type = NVS_TYPE_RNDIS, 251 .rndis_mtype = NVS_RNDIS_MTYPE_CTRL, 252 .chim_idx = NVS_CHIM_IDX_INVALID, 253 .chim_sz = 0 254 }; 255 struct vmbus_gpa sg; 256 rte_iova_t addr; 257 258 addr = rte_malloc_virt2iova(req); 259 if (unlikely(addr == RTE_BAD_IOVA)) { 260 PMD_DRV_LOG(ERR, "RNDIS send request can not get iova"); 261 return -EINVAL; 262 } 263 264 if (unlikely(reqlen > PAGE_SIZE)) { 265 PMD_DRV_LOG(ERR, "RNDIS request %u greater than page size", 266 reqlen); 267 return -EINVAL; 268 } 269 270 sg.page = addr / PAGE_SIZE; 271 sg.ofs = addr & PAGE_MASK; 272 sg.len = reqlen; 273 274 if (sg.ofs + reqlen > PAGE_SIZE) { 275 PMD_DRV_LOG(ERR, "RNDIS request crosses page bounary"); 276 return -EINVAL; 277 } 278 279 hn_rndis_dump(req); 280 281 return hn_nvs_send_sglist(chan, &sg, 1, 282 &nvs_rndis, sizeof(nvs_rndis), 0U, NULL); 283 } 284 285 void hn_rndis_link_status(struct rte_eth_dev *dev, const void *msg) 286 { 287 const struct rndis_status_msg *indicate = msg; 288 289 hn_rndis_dump(msg); 290 291 PMD_DRV_LOG(DEBUG, "link status %#x", indicate->status); 292 293 switch (indicate->status) { 294 case RNDIS_STATUS_NETWORK_CHANGE: 295 case RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG: 296 /* ignore not in DPDK API */ 297 break; 298 299 case RNDIS_STATUS_LINK_SPEED_CHANGE: 300 case RNDIS_STATUS_MEDIA_CONNECT: 301 case RNDIS_STATUS_MEDIA_DISCONNECT: 302 if (dev->data->dev_conf.intr_conf.lsc && 303 hn_dev_link_update(dev, 0) == 0) 304 _rte_eth_dev_callback_process(dev, 305 RTE_ETH_EVENT_INTR_LSC, 306 NULL); 307 break; 308 default: 309 PMD_DRV_LOG(NOTICE, "unknown RNDIS indication: %#x", 310 indicate->status); 311 } 312 } 313 314 /* Callback from hn_process_events when response is visible */ 315 void hn_rndis_receive_response(struct hn_data *hv, 316 const void *data, uint32_t len) 317 { 318 const struct rndis_init_comp *hdr = data; 319 320 hn_rndis_dump(data); 321 322 if (len < sizeof(3 * sizeof(uint32_t))) { 323 PMD_DRV_LOG(ERR, 324 "missing RNDIS header %u", len); 325 return; 326 } 327 328 if (len < hdr->len) { 329 PMD_DRV_LOG(ERR, 330 "truncated RNDIS response %u", len); 331 return; 332 } 333 334 if (len > sizeof(hv->rndis_resp)) { 335 PMD_DRV_LOG(NOTICE, 336 "RNDIS response exceeds buffer"); 337 len = sizeof(hv->rndis_resp); 338 } 339 340 if (hdr->rid == 0) { 341 PMD_DRV_LOG(NOTICE, 342 "RNDIS response id zero!"); 343 } 344 345 memcpy(hv->rndis_resp, data, len); 346 347 /* make sure response copied before update */ 348 rte_smp_wmb(); 349 350 if (rte_atomic32_cmpset(&hv->rndis_pending, hdr->rid, 0) == 0) { 351 PMD_DRV_LOG(ERR, 352 "received id %#x pending id %#x", 353 hdr->rid, (uint32_t)hv->rndis_pending); 354 } 355 } 356 357 /* Do request/response transaction */ 358 static int hn_rndis_exec1(struct hn_data *hv, 359 const void *req, uint32_t reqlen, 360 void *comp, uint32_t comp_len) 361 { 362 const struct rndis_halt_req *hdr = req; 363 uint32_t rid = hdr->rid; 364 struct vmbus_channel *chan = hn_primary_chan(hv); 365 int error; 366 367 if (comp_len > sizeof(hv->rndis_resp)) { 368 PMD_DRV_LOG(ERR, 369 "Expected completion size %u exceeds buffer %zu", 370 comp_len, sizeof(hv->rndis_resp)); 371 return -EIO; 372 } 373 374 if (comp != NULL && 375 rte_atomic32_cmpset(&hv->rndis_pending, 0, rid) == 0) { 376 PMD_DRV_LOG(ERR, 377 "Request already pending"); 378 return -EBUSY; 379 } 380 381 error = hn_nvs_send_rndis_ctrl(chan, req, reqlen); 382 if (error) { 383 PMD_DRV_LOG(ERR, "RNDIS ctrl send failed: %d", error); 384 return error; 385 } 386 387 if (comp) { 388 /* Poll primary channel until response received */ 389 while (hv->rndis_pending == rid) 390 hn_process_events(hv, 0, 1); 391 392 memcpy(comp, hv->rndis_resp, comp_len); 393 } 394 395 return 0; 396 } 397 398 /* Do transaction and validate response */ 399 static int hn_rndis_execute(struct hn_data *hv, uint32_t rid, 400 const void *req, uint32_t reqlen, 401 void *comp, uint32_t comp_len, uint32_t comp_type) 402 { 403 const struct rndis_comp_hdr *hdr = comp; 404 int ret; 405 406 memset(comp, 0, comp_len); 407 408 ret = hn_rndis_exec1(hv, req, reqlen, comp, comp_len); 409 if (ret < 0) 410 return ret; 411 /* 412 * Check this RNDIS complete message. 413 */ 414 if (unlikely(hdr->type != comp_type)) { 415 PMD_DRV_LOG(ERR, 416 "unexpected RNDIS response complete %#x expect %#x", 417 hdr->type, comp_type); 418 419 return -ENXIO; 420 } 421 if (unlikely(hdr->rid != rid)) { 422 PMD_DRV_LOG(ERR, 423 "RNDIS comp rid mismatch %#x, expect %#x", 424 hdr->rid, rid); 425 return -EINVAL; 426 } 427 428 /* All pass! */ 429 return 0; 430 } 431 432 static int 433 hn_rndis_query(struct hn_data *hv, uint32_t oid, 434 const void *idata, uint32_t idlen, 435 void *odata, uint32_t odlen) 436 { 437 struct rndis_query_req *req; 438 struct rndis_query_comp *comp; 439 uint32_t reqlen, comp_len; 440 int error = -EIO; 441 unsigned int ofs; 442 uint32_t rid; 443 444 reqlen = sizeof(*req) + idlen; 445 req = hn_rndis_alloc(hv, reqlen); 446 if (req == NULL) 447 return -ENOMEM; 448 449 comp_len = sizeof(*comp) + odlen; 450 comp = rte_zmalloc("QUERY", comp_len, PAGE_SIZE); 451 if (!comp) { 452 error = -ENOMEM; 453 goto done; 454 } 455 comp->status = RNDIS_STATUS_PENDING; 456 457 rid = hn_rndis_rid(hv); 458 459 req->type = RNDIS_QUERY_MSG; 460 req->len = reqlen; 461 req->rid = rid; 462 req->oid = oid; 463 req->infobufoffset = RNDIS_QUERY_REQ_INFOBUFOFFSET; 464 req->infobuflen = idlen; 465 466 /* Input data immediately follows RNDIS query. */ 467 memcpy(req + 1, idata, idlen); 468 469 error = hn_rndis_execute(hv, rid, req, reqlen, 470 comp, comp_len, RNDIS_QUERY_CMPLT); 471 472 if (error) 473 goto done; 474 475 if (comp->status != RNDIS_STATUS_SUCCESS) { 476 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x failed: status 0x%08x", 477 oid, comp->status); 478 error = -EINVAL; 479 goto done; 480 } 481 482 if (comp->infobuflen == 0 || comp->infobufoffset == 0) { 483 /* No output data! */ 484 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x, no data", oid); 485 error = 0; 486 goto done; 487 } 488 489 /* 490 * Check output data length and offset. 491 */ 492 /* ofs is the offset from the beginning of comp. */ 493 ofs = RNDIS_QUERY_COMP_INFOBUFOFFSET_ABS(comp->infobufoffset); 494 if (ofs < sizeof(*comp) || ofs + comp->infobuflen > comp_len) { 495 PMD_DRV_LOG(ERR, "RNDIS query invalid comp ib off/len, %u/%u", 496 comp->infobufoffset, comp->infobuflen); 497 error = -EINVAL; 498 goto done; 499 } 500 501 /* Save output data. */ 502 if (comp->infobuflen < odlen) 503 odlen = comp->infobuflen; 504 505 /* ofs is the offset from the beginning of comp. */ 506 memcpy(odata, (const char *)comp + ofs, odlen); 507 508 error = 0; 509 done: 510 rte_free(comp); 511 rte_free(req); 512 return error; 513 } 514 515 static int 516 hn_rndis_halt(struct hn_data *hv) 517 { 518 struct rndis_halt_req *halt; 519 520 halt = hn_rndis_alloc(hv, sizeof(*halt)); 521 if (halt == NULL) 522 return -ENOMEM; 523 524 halt->type = RNDIS_HALT_MSG; 525 halt->len = sizeof(*halt); 526 halt->rid = hn_rndis_rid(hv); 527 528 /* No RNDIS completion; rely on NVS message send completion */ 529 hn_rndis_exec1(hv, halt, sizeof(*halt), NULL, 0); 530 531 rte_free(halt); 532 533 PMD_INIT_LOG(DEBUG, "RNDIS halt done"); 534 return 0; 535 } 536 537 static int 538 hn_rndis_query_hwcaps(struct hn_data *hv, struct ndis_offload *caps) 539 { 540 struct ndis_offload in; 541 uint32_t caps_len, size; 542 int error; 543 544 memset(caps, 0, sizeof(*caps)); 545 memset(&in, 0, sizeof(in)); 546 in.ndis_hdr.ndis_type = NDIS_OBJTYPE_OFFLOAD; 547 548 if (hv->ndis_ver >= NDIS_VERSION_6_30) { 549 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_3; 550 size = NDIS_OFFLOAD_SIZE; 551 } else if (hv->ndis_ver >= NDIS_VERSION_6_1) { 552 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_2; 553 size = NDIS_OFFLOAD_SIZE_6_1; 554 } else { 555 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_1; 556 size = NDIS_OFFLOAD_SIZE_6_0; 557 } 558 in.ndis_hdr.ndis_size = size; 559 560 caps_len = NDIS_OFFLOAD_SIZE; 561 error = hn_rndis_query(hv, OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES, 562 &in, size, caps, caps_len); 563 if (error) 564 return error; 565 566 /* Preliminary verification. */ 567 if (caps->ndis_hdr.ndis_type != NDIS_OBJTYPE_OFFLOAD) { 568 PMD_DRV_LOG(NOTICE, "invalid NDIS objtype 0x%02x", 569 caps->ndis_hdr.ndis_type); 570 return -EINVAL; 571 } 572 if (caps->ndis_hdr.ndis_rev < NDIS_OFFLOAD_REV_1) { 573 PMD_DRV_LOG(NOTICE, "invalid NDIS objrev 0x%02x", 574 caps->ndis_hdr.ndis_rev); 575 return -EINVAL; 576 } 577 if (caps->ndis_hdr.ndis_size > caps_len) { 578 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u, data size %u", 579 caps->ndis_hdr.ndis_size, caps_len); 580 return -EINVAL; 581 } else if (caps->ndis_hdr.ndis_size < NDIS_OFFLOAD_SIZE_6_0) { 582 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u", 583 caps->ndis_hdr.ndis_size); 584 return -EINVAL; 585 } 586 587 return 0; 588 } 589 590 int 591 hn_rndis_query_rsscaps(struct hn_data *hv, 592 unsigned int *rxr_cnt0) 593 { 594 struct ndis_rss_caps in, caps; 595 unsigned int indsz, rxr_cnt; 596 uint32_t caps_len; 597 int error; 598 599 *rxr_cnt0 = 0; 600 601 if (hv->ndis_ver < NDIS_VERSION_6_20) { 602 PMD_DRV_LOG(DEBUG, "RSS not supported on this host"); 603 return -EOPNOTSUPP; 604 } 605 606 memset(&in, 0, sizeof(in)); 607 in.ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_CAPS; 608 in.ndis_hdr.ndis_rev = NDIS_RSS_CAPS_REV_2; 609 in.ndis_hdr.ndis_size = NDIS_RSS_CAPS_SIZE; 610 611 caps_len = NDIS_RSS_CAPS_SIZE; 612 error = hn_rndis_query(hv, OID_GEN_RECEIVE_SCALE_CAPABILITIES, 613 &in, NDIS_RSS_CAPS_SIZE, 614 &caps, caps_len); 615 if (error) 616 return error; 617 618 PMD_INIT_LOG(DEBUG, "RX rings %u indirect %u caps %#x", 619 caps.ndis_nrxr, caps.ndis_nind, caps.ndis_caps); 620 /* 621 * Preliminary verification. 622 */ 623 if (caps.ndis_hdr.ndis_type != NDIS_OBJTYPE_RSS_CAPS) { 624 PMD_DRV_LOG(ERR, "invalid NDIS objtype 0x%02x", 625 caps.ndis_hdr.ndis_type); 626 return -EINVAL; 627 } 628 if (caps.ndis_hdr.ndis_rev < NDIS_RSS_CAPS_REV_1) { 629 PMD_DRV_LOG(ERR, "invalid NDIS objrev 0x%02x", 630 caps.ndis_hdr.ndis_rev); 631 return -EINVAL; 632 } 633 if (caps.ndis_hdr.ndis_size > caps_len) { 634 PMD_DRV_LOG(ERR, 635 "invalid NDIS objsize %u, data size %u", 636 caps.ndis_hdr.ndis_size, caps_len); 637 return -EINVAL; 638 } else if (caps.ndis_hdr.ndis_size < NDIS_RSS_CAPS_SIZE_6_0) { 639 PMD_DRV_LOG(ERR, "invalid NDIS objsize %u", 640 caps.ndis_hdr.ndis_size); 641 return -EINVAL; 642 } 643 644 /* 645 * Save information for later RSS configuration. 646 */ 647 if (caps.ndis_nrxr == 0) { 648 PMD_DRV_LOG(ERR, "0 RX rings!?"); 649 return -EINVAL; 650 } 651 rxr_cnt = caps.ndis_nrxr; 652 653 if (caps.ndis_hdr.ndis_size == NDIS_RSS_CAPS_SIZE && 654 caps.ndis_hdr.ndis_rev >= NDIS_RSS_CAPS_REV_2) { 655 if (caps.ndis_nind > NDIS_HASH_INDCNT) { 656 PMD_DRV_LOG(ERR, 657 "too many RSS indirect table entries %u", 658 caps.ndis_nind); 659 return -EOPNOTSUPP; 660 } 661 if (!rte_is_power_of_2(caps.ndis_nind)) { 662 PMD_DRV_LOG(ERR, 663 "RSS indirect table size is not power-of-2 %u", 664 caps.ndis_nind); 665 } 666 667 indsz = caps.ndis_nind; 668 } else { 669 indsz = NDIS_HASH_INDCNT; 670 } 671 672 if (indsz < rxr_cnt) { 673 PMD_DRV_LOG(NOTICE, 674 "# of RX rings (%d) > RSS indirect table size %d", 675 rxr_cnt, indsz); 676 rxr_cnt = indsz; 677 } 678 679 hv->rss_offloads = 0; 680 if (caps.ndis_caps & NDIS_RSS_CAP_IPV4) 681 hv->rss_offloads |= ETH_RSS_IPV4 682 | ETH_RSS_NONFRAG_IPV4_TCP 683 | ETH_RSS_NONFRAG_IPV4_UDP; 684 if (caps.ndis_caps & NDIS_RSS_CAP_IPV6) 685 hv->rss_offloads |= ETH_RSS_IPV6 686 | ETH_RSS_NONFRAG_IPV6_TCP; 687 if (caps.ndis_caps & NDIS_RSS_CAP_IPV6_EX) 688 hv->rss_offloads |= ETH_RSS_IPV6_EX 689 | ETH_RSS_IPV6_TCP_EX; 690 691 /* Commit! */ 692 *rxr_cnt0 = rxr_cnt; 693 694 return 0; 695 } 696 697 static int 698 hn_rndis_set(struct hn_data *hv, uint32_t oid, const void *data, uint32_t dlen) 699 { 700 struct rndis_set_req *req; 701 struct rndis_set_comp comp; 702 uint32_t reqlen, comp_len; 703 uint32_t rid; 704 int error; 705 706 reqlen = sizeof(*req) + dlen; 707 req = rte_zmalloc("RNDIS_SET", reqlen, PAGE_SIZE); 708 if (!req) 709 return -ENOMEM; 710 711 rid = hn_rndis_rid(hv); 712 req->type = RNDIS_SET_MSG; 713 req->len = reqlen; 714 req->rid = rid; 715 req->oid = oid; 716 req->infobuflen = dlen; 717 req->infobufoffset = RNDIS_SET_REQ_INFOBUFOFFSET; 718 719 /* Data immediately follows RNDIS set. */ 720 memcpy(req + 1, data, dlen); 721 722 comp_len = sizeof(comp); 723 error = hn_rndis_execute(hv, rid, req, reqlen, 724 &comp, comp_len, 725 RNDIS_SET_CMPLT); 726 if (error) { 727 PMD_DRV_LOG(ERR, "exec RNDIS set %#" PRIx32 " failed", 728 oid); 729 error = EIO; 730 goto done; 731 } 732 733 if (comp.status != RNDIS_STATUS_SUCCESS) { 734 PMD_DRV_LOG(ERR, 735 "RNDIS set %#" PRIx32 " failed: status %#" PRIx32, 736 oid, comp.status); 737 error = EIO; 738 goto done; 739 } 740 741 done: 742 rte_free(req); 743 return error; 744 } 745 746 int hn_rndis_conf_offload(struct hn_data *hv, 747 uint64_t tx_offloads, uint64_t rx_offloads) 748 { 749 struct ndis_offload_params params; 750 struct ndis_offload hwcaps; 751 int error; 752 753 error = hn_rndis_query_hwcaps(hv, &hwcaps); 754 if (error) { 755 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 756 return error; 757 } 758 759 /* NOTE: 0 means "no change" */ 760 memset(¶ms, 0, sizeof(params)); 761 762 params.ndis_hdr.ndis_type = NDIS_OBJTYPE_DEFAULT; 763 if (hv->ndis_ver < NDIS_VERSION_6_30) { 764 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_2; 765 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE_6_1; 766 } else { 767 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_3; 768 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE; 769 } 770 771 if (tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM) { 772 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_TCP4) 773 params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_TX; 774 else 775 goto unsupported; 776 777 if (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_TCP6) 778 params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_TX; 779 else 780 goto unsupported; 781 } 782 783 if (rx_offloads & DEV_RX_OFFLOAD_TCP_CKSUM) { 784 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) 785 == NDIS_RXCSUM_CAP_TCP4) 786 params.ndis_tcp4csum |= NDIS_OFFLOAD_PARAM_RX; 787 else 788 goto unsupported; 789 790 if ((hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6) 791 == NDIS_RXCSUM_CAP_TCP6) 792 params.ndis_tcp6csum |= NDIS_OFFLOAD_PARAM_RX; 793 else 794 goto unsupported; 795 } 796 797 if (tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) { 798 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) 799 params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_TX; 800 else 801 goto unsupported; 802 803 if ((hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6) 804 == NDIS_TXCSUM_CAP_UDP6) 805 params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_TX; 806 else 807 goto unsupported; 808 } 809 810 if (rx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) { 811 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) 812 params.ndis_udp4csum |= NDIS_OFFLOAD_PARAM_RX; 813 else 814 goto unsupported; 815 816 if (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6) 817 params.ndis_udp6csum |= NDIS_OFFLOAD_PARAM_RX; 818 else 819 goto unsupported; 820 } 821 822 if (tx_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) { 823 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_IP4) 824 == NDIS_TXCSUM_CAP_IP4) 825 params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_TX; 826 else 827 goto unsupported; 828 } 829 if (rx_offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) { 830 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 831 params.ndis_ip4csum |= NDIS_OFFLOAD_PARAM_RX; 832 else 833 goto unsupported; 834 } 835 836 if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO) { 837 if (hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) 838 params.ndis_lsov2_ip4 = NDIS_OFFLOAD_LSOV2_ON; 839 else 840 goto unsupported; 841 842 if ((hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6) 843 == HN_NDIS_LSOV2_CAP_IP6) 844 params.ndis_lsov2_ip6 = NDIS_OFFLOAD_LSOV2_ON; 845 else 846 goto unsupported; 847 } 848 849 error = hn_rndis_set(hv, OID_TCP_OFFLOAD_PARAMETERS, ¶ms, 850 params.ndis_hdr.ndis_size); 851 if (error) { 852 PMD_DRV_LOG(ERR, "offload config failed"); 853 return error; 854 } 855 856 return 0; 857 unsupported: 858 PMD_DRV_LOG(NOTICE, 859 "offload tx:%" PRIx64 " rx:%" PRIx64 " not supported by this version", 860 tx_offloads, rx_offloads); 861 return -EINVAL; 862 } 863 864 int hn_rndis_get_offload(struct hn_data *hv, 865 struct rte_eth_dev_info *dev_info) 866 { 867 struct ndis_offload hwcaps; 868 int error; 869 870 memset(&hwcaps, 0, sizeof(hwcaps)); 871 872 error = hn_rndis_query_hwcaps(hv, &hwcaps); 873 if (error) { 874 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 875 return error; 876 } 877 878 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS | 879 DEV_TX_OFFLOAD_VLAN_INSERT; 880 881 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_IP4) 882 == HN_NDIS_TXCSUM_CAP_IP4) 883 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM; 884 885 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_TCP4) 886 == HN_NDIS_TXCSUM_CAP_TCP4 && 887 (hwcaps.ndis_csum.ndis_ip6_txcsum & HN_NDIS_TXCSUM_CAP_TCP6) 888 == HN_NDIS_TXCSUM_CAP_TCP6) 889 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_CKSUM; 890 891 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) && 892 (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6)) 893 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_UDP_CKSUM; 894 895 if ((hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) && 896 (hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6) 897 == HN_NDIS_LSOV2_CAP_IP6) 898 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; 899 900 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP; 901 902 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 903 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM; 904 905 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) && 906 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)) 907 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_CKSUM; 908 909 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) && 910 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)) 911 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_UDP_CKSUM; 912 913 return 0; 914 } 915 916 uint32_t 917 hn_rndis_get_ptypes(struct hn_data *hv) 918 { 919 struct ndis_offload hwcaps; 920 uint32_t ptypes; 921 int error; 922 923 memset(&hwcaps, 0, sizeof(hwcaps)); 924 925 error = hn_rndis_query_hwcaps(hv, &hwcaps); 926 if (error) { 927 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 928 return RTE_PTYPE_L2_ETHER; 929 } 930 931 ptypes = RTE_PTYPE_L2_ETHER; 932 933 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 934 ptypes |= RTE_PTYPE_L3_IPV4; 935 936 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) || 937 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)) 938 ptypes |= RTE_PTYPE_L4_TCP; 939 940 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) || 941 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)) 942 ptypes |= RTE_PTYPE_L4_UDP; 943 944 return ptypes; 945 } 946 947 int 948 hn_rndis_set_rxfilter(struct hn_data *hv, uint32_t filter) 949 { 950 int error; 951 952 error = hn_rndis_set(hv, OID_GEN_CURRENT_PACKET_FILTER, 953 &filter, sizeof(filter)); 954 if (error) { 955 PMD_DRV_LOG(ERR, "set RX filter %#" PRIx32 " failed: %d", 956 filter, error); 957 } else { 958 PMD_DRV_LOG(DEBUG, "set RX filter %#" PRIx32 " done", filter); 959 } 960 961 return error; 962 } 963 964 /* The default RSS key. 965 * This value is the same as MLX5 so that flows will be 966 * received on same path for both VF ans synthetic NIC. 967 */ 968 static const uint8_t rss_default_key[NDIS_HASH_KEYSIZE_TOEPLITZ] = { 969 0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7, 970 0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94, 971 0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1, 972 0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59, 973 0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a, 974 }; 975 976 int hn_rndis_conf_rss(struct hn_data *hv, 977 const struct rte_eth_rss_conf *rss_conf) 978 { 979 struct ndis_rssprm_toeplitz rssp; 980 struct ndis_rss_params *prm = &rssp.rss_params; 981 const uint8_t *rss_key = rss_conf->rss_key ? : rss_default_key; 982 uint32_t rss_hash; 983 unsigned int i; 984 int error; 985 986 PMD_INIT_FUNC_TRACE(); 987 988 memset(&rssp, 0, sizeof(rssp)); 989 990 prm->ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_PARAMS; 991 prm->ndis_hdr.ndis_rev = NDIS_RSS_PARAMS_REV_2; 992 prm->ndis_hdr.ndis_size = sizeof(*prm); 993 prm->ndis_flags = 0; 994 995 rss_hash = NDIS_HASH_FUNCTION_TOEPLITZ; 996 if (rss_conf->rss_hf & ETH_RSS_IPV4) 997 rss_hash |= NDIS_HASH_IPV4; 998 if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) 999 rss_hash |= NDIS_HASH_TCP_IPV4; 1000 if (rss_conf->rss_hf & ETH_RSS_IPV6) 1001 rss_hash |= NDIS_HASH_IPV6; 1002 if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) 1003 rss_hash |= NDIS_HASH_TCP_IPV6; 1004 1005 prm->ndis_hash = rss_hash; 1006 prm->ndis_indsize = sizeof(rssp.rss_ind[0]) * NDIS_HASH_INDCNT; 1007 prm->ndis_indoffset = offsetof(struct ndis_rssprm_toeplitz, rss_ind[0]); 1008 prm->ndis_keysize = NDIS_HASH_KEYSIZE_TOEPLITZ; 1009 prm->ndis_keyoffset = offsetof(struct ndis_rssprm_toeplitz, rss_key[0]); 1010 1011 for (i = 0; i < NDIS_HASH_INDCNT; i++) 1012 rssp.rss_ind[i] = i % hv->num_queues; 1013 1014 /* Set hask key values */ 1015 memcpy(&rssp.rss_key, rss_key, NDIS_HASH_KEYSIZE_TOEPLITZ); 1016 1017 error = hn_rndis_set(hv, OID_GEN_RECEIVE_SCALE_PARAMETERS, 1018 &rssp, sizeof(rssp)); 1019 if (error) { 1020 PMD_DRV_LOG(ERR, 1021 "RSS config num queues=%u failed: %d", 1022 hv->num_queues, error); 1023 } 1024 return error; 1025 } 1026 1027 static int hn_rndis_init(struct hn_data *hv) 1028 { 1029 struct rndis_init_req *req; 1030 struct rndis_init_comp comp; 1031 uint32_t comp_len, rid; 1032 int error; 1033 1034 req = hn_rndis_alloc(hv, sizeof(*req)); 1035 if (!req) { 1036 PMD_DRV_LOG(ERR, "no memory for RNDIS init"); 1037 return -ENXIO; 1038 } 1039 1040 rid = hn_rndis_rid(hv); 1041 req->type = RNDIS_INITIALIZE_MSG; 1042 req->len = sizeof(*req); 1043 req->rid = rid; 1044 req->ver_major = RNDIS_VERSION_MAJOR; 1045 req->ver_minor = RNDIS_VERSION_MINOR; 1046 req->max_xfersz = HN_RNDIS_XFER_SIZE; 1047 1048 comp_len = RNDIS_INIT_COMP_SIZE_MIN; 1049 error = hn_rndis_execute(hv, rid, req, sizeof(*req), 1050 &comp, comp_len, 1051 RNDIS_INITIALIZE_CMPLT); 1052 if (error) 1053 goto done; 1054 1055 if (comp.status != RNDIS_STATUS_SUCCESS) { 1056 PMD_DRV_LOG(ERR, "RNDIS init failed: status 0x%08x", 1057 comp.status); 1058 error = -EIO; 1059 goto done; 1060 } 1061 1062 hv->rndis_agg_size = comp.pktmaxsz; 1063 hv->rndis_agg_pkts = comp.pktmaxcnt; 1064 hv->rndis_agg_align = 1U << comp.align; 1065 1066 if (hv->rndis_agg_align < sizeof(uint32_t)) { 1067 /* 1068 * The RNDIS packet message encap assumes that the RNDIS 1069 * packet message is at least 4 bytes aligned. Fix up the 1070 * alignment here, if the remote side sets the alignment 1071 * too low. 1072 */ 1073 PMD_DRV_LOG(NOTICE, 1074 "fixup RNDIS aggpkt align: %u -> %zu", 1075 hv->rndis_agg_align, sizeof(uint32_t)); 1076 hv->rndis_agg_align = sizeof(uint32_t); 1077 } 1078 1079 PMD_INIT_LOG(INFO, 1080 "RNDIS ver %u.%u, aggpkt size %u, aggpkt cnt %u, aggpkt align %u", 1081 comp.ver_major, comp.ver_minor, 1082 hv->rndis_agg_size, hv->rndis_agg_pkts, 1083 hv->rndis_agg_align); 1084 error = 0; 1085 done: 1086 rte_free(req); 1087 return error; 1088 } 1089 1090 int 1091 hn_rndis_get_eaddr(struct hn_data *hv, uint8_t *eaddr) 1092 { 1093 uint32_t eaddr_len; 1094 int error; 1095 1096 eaddr_len = RTE_ETHER_ADDR_LEN; 1097 error = hn_rndis_query(hv, OID_802_3_PERMANENT_ADDRESS, NULL, 0, 1098 eaddr, eaddr_len); 1099 if (error) 1100 return error; 1101 1102 PMD_DRV_LOG(INFO, "MAC address %02x:%02x:%02x:%02x:%02x:%02x", 1103 eaddr[0], eaddr[1], eaddr[2], 1104 eaddr[3], eaddr[4], eaddr[5]); 1105 return 0; 1106 } 1107 1108 int 1109 hn_rndis_get_linkstatus(struct hn_data *hv) 1110 { 1111 return hn_rndis_query(hv, OID_GEN_MEDIA_CONNECT_STATUS, NULL, 0, 1112 &hv->link_status, sizeof(uint32_t)); 1113 } 1114 1115 int 1116 hn_rndis_get_linkspeed(struct hn_data *hv) 1117 { 1118 return hn_rndis_query(hv, OID_GEN_LINK_SPEED, NULL, 0, 1119 &hv->link_speed, sizeof(uint32_t)); 1120 } 1121 1122 int 1123 hn_rndis_attach(struct hn_data *hv) 1124 { 1125 /* Initialize RNDIS. */ 1126 return hn_rndis_init(hv); 1127 } 1128 1129 void 1130 hn_rndis_detach(struct hn_data *hv) 1131 { 1132 /* Halt the RNDIS. */ 1133 hn_rndis_halt(hv); 1134 } 1135