1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2009-2018 Microsoft Corp. 3 * Copyright (c) 2010-2012 Citrix Inc. 4 * Copyright (c) 2012 NetApp Inc. 5 * All rights reserved. 6 */ 7 8 #include <stdint.h> 9 #include <string.h> 10 #include <stdio.h> 11 #include <errno.h> 12 #include <unistd.h> 13 #include <time.h> 14 15 #include <ethdev_driver.h> 16 #include <rte_ethdev.h> 17 #include <rte_string_fns.h> 18 #include <rte_memzone.h> 19 #include <rte_malloc.h> 20 #include <rte_atomic.h> 21 #include <rte_alarm.h> 22 #include <rte_branch_prediction.h> 23 #include <rte_ether.h> 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_cycles.h> 27 #include <rte_memory.h> 28 #include <rte_eal.h> 29 #include <dev_driver.h> 30 #include <bus_vmbus_driver.h> 31 32 #include "hn_logs.h" 33 #include "hn_var.h" 34 #include "hn_nvs.h" 35 #include "hn_rndis.h" 36 #include "ndis.h" 37 38 #define RNDIS_TIMEOUT_SEC 60 39 #define RNDIS_DELAY_MS 10 40 41 #define HN_RNDIS_XFER_SIZE 0x4000 42 43 #define HN_NDIS_TXCSUM_CAP_IP4 \ 44 (NDIS_TXCSUM_CAP_IP4 | NDIS_TXCSUM_CAP_IP4OPT) 45 #define HN_NDIS_TXCSUM_CAP_TCP4 \ 46 (NDIS_TXCSUM_CAP_TCP4 | NDIS_TXCSUM_CAP_TCP4OPT) 47 #define HN_NDIS_TXCSUM_CAP_TCP6 \ 48 (NDIS_TXCSUM_CAP_TCP6 | NDIS_TXCSUM_CAP_TCP6OPT | \ 49 NDIS_TXCSUM_CAP_IP6EXT) 50 #define HN_NDIS_TXCSUM_CAP_UDP6 \ 51 (NDIS_TXCSUM_CAP_UDP6 | NDIS_TXCSUM_CAP_IP6EXT) 52 #define HN_NDIS_LSOV2_CAP_IP6 \ 53 (NDIS_LSOV2_CAP_IP6EXT | NDIS_LSOV2_CAP_TCP6OPT) 54 55 /* Get unique request id */ 56 static inline uint32_t 57 hn_rndis_rid(struct hn_data *hv) 58 { 59 uint32_t rid; 60 61 do { 62 rid = rte_atomic32_add_return(&hv->rndis_req_id, 1); 63 } while (rid == 0); 64 65 return rid; 66 } 67 68 static void *hn_rndis_alloc(size_t size) 69 { 70 return rte_zmalloc("RNDIS", size, rte_mem_page_size()); 71 } 72 73 #ifdef RTE_LIBRTE_NETVSC_DEBUG_DUMP 74 void hn_rndis_dump(const void *buf) 75 { 76 const union { 77 struct rndis_msghdr hdr; 78 struct rndis_packet_msg pkt; 79 struct rndis_init_req init_request; 80 struct rndis_init_comp init_complete; 81 struct rndis_halt_req halt; 82 struct rndis_query_req query_request; 83 struct rndis_query_comp query_complete; 84 struct rndis_set_req set_request; 85 struct rndis_set_comp set_complete; 86 struct rndis_reset_req reset_request; 87 struct rndis_reset_comp reset_complete; 88 struct rndis_keepalive_req keepalive_request; 89 struct rndis_keepalive_comp keepalive_complete; 90 struct rndis_status_msg indicate_status; 91 } *rndis_msg = buf; 92 93 switch (rndis_msg->hdr.type) { 94 case RNDIS_PACKET_MSG: { 95 const struct rndis_pktinfo *ppi; 96 unsigned int ppi_len; 97 98 RTE_LOG_LINE(DEBUG, HN_DRIVER, 99 "RNDIS_MSG_PACKET (len %u, data %u:%u, # oob %u %u:%u, pkt %u:%u)", 100 rndis_msg->pkt.len, 101 rndis_msg->pkt.dataoffset, 102 rndis_msg->pkt.datalen, 103 rndis_msg->pkt.oobdataelements, 104 rndis_msg->pkt.oobdataoffset, 105 rndis_msg->pkt.oobdatalen, 106 rndis_msg->pkt.pktinfooffset, 107 rndis_msg->pkt.pktinfolen); 108 109 ppi = (const struct rndis_pktinfo *) 110 ((const char *)buf 111 + RNDIS_PACKET_MSG_OFFSET_ABS(rndis_msg->pkt.pktinfooffset)); 112 113 ppi_len = rndis_msg->pkt.pktinfolen; 114 while (ppi_len > 0) { 115 const void *ppi_data; 116 117 ppi_data = ppi->data; 118 119 RTE_LOG_LINE(DEBUG, HN_DRIVER, 120 " PPI (size %u, type %u, offs %u data %#x)", 121 ppi->size, ppi->type, ppi->offset, 122 *(const uint32_t *)ppi_data); 123 if (ppi->size == 0) 124 break; 125 ppi_len -= ppi->size; 126 ppi = (const struct rndis_pktinfo *) 127 ((const char *)ppi + ppi->size); 128 } 129 break; 130 } 131 case RNDIS_INITIALIZE_MSG: 132 RTE_LOG_LINE(DEBUG, HN_DRIVER, 133 "RNDIS_MSG_INIT (len %u id %#x, ver %u.%u max xfer %u)", 134 rndis_msg->init_request.len, 135 rndis_msg->init_request.rid, 136 rndis_msg->init_request.ver_major, 137 rndis_msg->init_request.ver_minor, 138 rndis_msg->init_request.max_xfersz); 139 break; 140 141 case RNDIS_INITIALIZE_CMPLT: 142 RTE_LOG_LINE(DEBUG, HN_DRIVER, 143 "RNDIS_MSG_INIT_C (len %u, id %#x, status 0x%x, vers %u.%u, " 144 "flags %d, max xfer %u, max pkts %u, aligned %u)", 145 rndis_msg->init_complete.len, 146 rndis_msg->init_complete.rid, 147 rndis_msg->init_complete.status, 148 rndis_msg->init_complete.ver_major, 149 rndis_msg->init_complete.ver_minor, 150 rndis_msg->init_complete.devflags, 151 rndis_msg->init_complete.pktmaxsz, 152 rndis_msg->init_complete.pktmaxcnt, 153 rndis_msg->init_complete.align); 154 break; 155 156 case RNDIS_HALT_MSG: 157 RTE_LOG_LINE(DEBUG, HN_DRIVER, 158 "RNDIS_HALT (len %u id %#x)", 159 rndis_msg->halt.len, rndis_msg->halt.rid); 160 break; 161 162 case RNDIS_QUERY_MSG: 163 RTE_LOG_LINE(DEBUG, HN_DRIVER, 164 "RNDIS_QUERY (len %u, id %#x, oid %#x, info %u:%u)", 165 rndis_msg->query_request.len, 166 rndis_msg->query_request.rid, 167 rndis_msg->query_request.oid, 168 rndis_msg->query_request.infobuflen, 169 rndis_msg->query_request.infobufoffset); 170 break; 171 172 case RNDIS_QUERY_CMPLT: 173 RTE_LOG_LINE(DEBUG, HN_DRIVER, 174 "RNDIS_MSG_QUERY_C (len %u, id %#x, status 0x%x, buf %u:%u)", 175 rndis_msg->query_complete.len, 176 rndis_msg->query_complete.rid, 177 rndis_msg->query_complete.status, 178 rndis_msg->query_complete.infobuflen, 179 rndis_msg->query_complete.infobufoffset); 180 break; 181 182 case RNDIS_SET_MSG: 183 RTE_LOG_LINE(DEBUG, HN_DRIVER, 184 "RNDIS_SET (len %u, id %#x, oid %#x, info %u:%u)", 185 rndis_msg->set_request.len, 186 rndis_msg->set_request.rid, 187 rndis_msg->set_request.oid, 188 rndis_msg->set_request.infobuflen, 189 rndis_msg->set_request.infobufoffset); 190 break; 191 192 case RNDIS_SET_CMPLT: 193 RTE_LOG_LINE(DEBUG, HN_DRIVER, 194 "RNDIS_MSG_SET_C (len %u, id 0x%x, status 0x%x)", 195 rndis_msg->set_complete.len, 196 rndis_msg->set_complete.rid, 197 rndis_msg->set_complete.status); 198 break; 199 200 case RNDIS_INDICATE_STATUS_MSG: 201 RTE_LOG_LINE(DEBUG, HN_DRIVER, 202 "RNDIS_MSG_INDICATE (len %u, status %#x, buf len %u, buf offset %u)", 203 rndis_msg->indicate_status.len, 204 rndis_msg->indicate_status.status, 205 rndis_msg->indicate_status.stbuflen, 206 rndis_msg->indicate_status.stbufoffset); 207 break; 208 209 case RNDIS_RESET_MSG: 210 RTE_LOG_LINE(DEBUG, HN_DRIVER, 211 "RNDIS_RESET (len %u, id %#x)", 212 rndis_msg->reset_request.len, 213 rndis_msg->reset_request.rid); 214 break; 215 216 case RNDIS_RESET_CMPLT: 217 RTE_LOG_LINE(DEBUG, HN_DRIVER, 218 "RNDIS_RESET_C (len %u, status %#x address %#x)", 219 rndis_msg->reset_complete.len, 220 rndis_msg->reset_complete.status, 221 rndis_msg->reset_complete.adrreset); 222 break; 223 224 case RNDIS_KEEPALIVE_MSG: 225 RTE_LOG_LINE(DEBUG, HN_DRIVER, 226 "RNDIS_KEEPALIVE (len %u, id %#x)", 227 rndis_msg->keepalive_request.len, 228 rndis_msg->keepalive_request.rid); 229 break; 230 231 case RNDIS_KEEPALIVE_CMPLT: 232 RTE_LOG_LINE(DEBUG, HN_DRIVER, 233 "RNDIS_KEEPALIVE_C (len %u, id %#x address %#x)", 234 rndis_msg->keepalive_complete.len, 235 rndis_msg->keepalive_complete.rid, 236 rndis_msg->keepalive_complete.status); 237 break; 238 239 default: 240 RTE_LOG_LINE(DEBUG, HN_DRIVER, 241 "RNDIS type %#x len %u", 242 rndis_msg->hdr.type, 243 rndis_msg->hdr.len); 244 break; 245 } 246 } 247 #endif 248 249 static int hn_nvs_send_rndis_ctrl(struct vmbus_channel *chan, 250 const void *req, uint32_t reqlen) 251 252 { 253 struct hn_nvs_rndis nvs_rndis = { 254 .type = NVS_TYPE_RNDIS, 255 .rndis_mtype = NVS_RNDIS_MTYPE_CTRL, 256 .chim_idx = NVS_CHIM_IDX_INVALID, 257 .chim_sz = 0 258 }; 259 struct vmbus_gpa sg; 260 rte_iova_t addr; 261 262 addr = rte_malloc_virt2iova(req); 263 if (unlikely(addr == RTE_BAD_IOVA)) { 264 PMD_DRV_LOG(ERR, "RNDIS send request can not get iova"); 265 return -EINVAL; 266 } 267 268 if (unlikely(reqlen > rte_mem_page_size())) { 269 PMD_DRV_LOG(ERR, "RNDIS request %u greater than page size", 270 reqlen); 271 return -EINVAL; 272 } 273 274 sg.page = addr / rte_mem_page_size(); 275 sg.ofs = addr & PAGE_MASK; 276 sg.len = reqlen; 277 278 if (sg.ofs + reqlen > rte_mem_page_size()) { 279 PMD_DRV_LOG(ERR, "RNDIS request crosses page boundary"); 280 return -EINVAL; 281 } 282 283 hn_rndis_dump(req); 284 285 return hn_nvs_send_sglist(chan, &sg, 1, 286 &nvs_rndis, sizeof(nvs_rndis), 0U, NULL); 287 } 288 289 /* 290 * Alarm callback to process link changed notifications. 291 * Can not directly since link_status is discovered while reading ring 292 */ 293 static void hn_rndis_link_alarm(void *arg) 294 { 295 rte_eth_dev_callback_process(arg, RTE_ETH_EVENT_INTR_LSC, NULL); 296 } 297 298 void hn_rndis_link_status(struct rte_eth_dev *dev, const void *msg) 299 { 300 const struct rndis_status_msg *indicate = msg; 301 302 hn_rndis_dump(msg); 303 304 PMD_DRV_LOG(DEBUG, "link status %#x", indicate->status); 305 306 switch (indicate->status) { 307 case RNDIS_STATUS_NETWORK_CHANGE: 308 case RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG: 309 /* ignore not in DPDK API */ 310 break; 311 312 case RNDIS_STATUS_LINK_SPEED_CHANGE: 313 case RNDIS_STATUS_MEDIA_CONNECT: 314 case RNDIS_STATUS_MEDIA_DISCONNECT: 315 if (dev->data->dev_conf.intr_conf.lsc) 316 rte_eal_alarm_set(10, hn_rndis_link_alarm, dev); 317 break; 318 default: 319 PMD_DRV_LOG(NOTICE, "unknown RNDIS indication: %#x", 320 indicate->status); 321 } 322 } 323 324 /* Callback from hn_process_events when response is visible */ 325 void hn_rndis_receive_response(struct hn_data *hv, 326 const void *data, uint32_t len) 327 { 328 const struct rndis_init_comp *hdr = data; 329 330 hn_rndis_dump(data); 331 332 /* Check we can read first three data fields from RNDIS header */ 333 if (len < 3 * sizeof(uint32_t)) { 334 PMD_DRV_LOG(ERR, 335 "missing RNDIS header %u", len); 336 return; 337 } 338 339 if (len < hdr->len) { 340 PMD_DRV_LOG(ERR, 341 "truncated RNDIS response %u", len); 342 return; 343 } 344 345 if (len > sizeof(hv->rndis_resp)) { 346 PMD_DRV_LOG(NOTICE, 347 "RNDIS response exceeds buffer"); 348 len = sizeof(hv->rndis_resp); 349 } 350 351 if (hdr->rid == 0) { 352 PMD_DRV_LOG(NOTICE, 353 "RNDIS response id zero!"); 354 } 355 356 memcpy(hv->rndis_resp, data, len); 357 358 /* make sure response copied before update */ 359 rte_smp_wmb(); 360 361 if (rte_atomic32_cmpset(&hv->rndis_pending, hdr->rid, 0) == 0) { 362 PMD_DRV_LOG(NOTICE, 363 "received id %#x pending id %#x", 364 hdr->rid, (uint32_t)hv->rndis_pending); 365 } 366 } 367 368 /* Do request/response transaction */ 369 static int hn_rndis_exec1(struct hn_data *hv, 370 const void *req, uint32_t reqlen, 371 void *comp, uint32_t comp_len) 372 { 373 const struct rndis_halt_req *hdr = req; 374 uint32_t rid = hdr->rid; 375 struct vmbus_channel *chan = hn_primary_chan(hv); 376 int error; 377 378 if (comp_len > sizeof(hv->rndis_resp)) { 379 PMD_DRV_LOG(ERR, 380 "Expected completion size %u exceeds buffer %zu", 381 comp_len, sizeof(hv->rndis_resp)); 382 return -EIO; 383 } 384 385 if (rid == 0) { 386 PMD_DRV_LOG(ERR, "Invalid request id"); 387 return -EINVAL; 388 } 389 390 if (comp != NULL && 391 rte_atomic32_cmpset(&hv->rndis_pending, 0, rid) == 0) { 392 PMD_DRV_LOG(ERR, 393 "Request already pending"); 394 return -EBUSY; 395 } 396 397 error = hn_nvs_send_rndis_ctrl(chan, req, reqlen); 398 if (error) { 399 PMD_DRV_LOG(ERR, "RNDIS ctrl send failed: %d", error); 400 return error; 401 } 402 403 if (comp) { 404 time_t start = time(NULL); 405 406 /* Poll primary channel until response received */ 407 while (hv->rndis_pending == rid) { 408 if (hv->closed) 409 return -ENETDOWN; 410 411 if (time(NULL) - start > RNDIS_TIMEOUT_SEC) { 412 PMD_DRV_LOG(ERR, 413 "RNDIS response timed out"); 414 415 rte_atomic32_cmpset(&hv->rndis_pending, rid, 0); 416 return -ETIMEDOUT; 417 } 418 419 if (rte_vmbus_chan_rx_empty(hv->primary->chan)) 420 rte_delay_ms(RNDIS_DELAY_MS); 421 422 hn_process_events(hv, 0, 1); 423 } 424 425 memcpy(comp, hv->rndis_resp, comp_len); 426 } 427 428 return 0; 429 } 430 431 /* Do transaction and validate response */ 432 static int hn_rndis_execute(struct hn_data *hv, uint32_t rid, 433 const void *req, uint32_t reqlen, 434 void *comp, uint32_t comp_len, uint32_t comp_type) 435 { 436 const struct rndis_comp_hdr *hdr = comp; 437 int ret; 438 439 memset(comp, 0, comp_len); 440 441 ret = hn_rndis_exec1(hv, req, reqlen, comp, comp_len); 442 if (ret < 0) 443 return ret; 444 /* 445 * Check this RNDIS complete message. 446 */ 447 if (unlikely(hdr->type != comp_type)) { 448 PMD_DRV_LOG(ERR, 449 "unexpected RNDIS response complete %#x expect %#x", 450 hdr->type, comp_type); 451 452 return -ENXIO; 453 } 454 if (unlikely(hdr->rid != rid)) { 455 PMD_DRV_LOG(ERR, 456 "RNDIS comp rid mismatch %#x, expect %#x", 457 hdr->rid, rid); 458 return -EINVAL; 459 } 460 461 /* All pass! */ 462 return 0; 463 } 464 465 static int 466 hn_rndis_query(struct hn_data *hv, uint32_t oid, 467 const void *idata, uint32_t idlen, 468 void *odata, uint32_t odlen) 469 { 470 struct rndis_query_req *req; 471 struct rndis_query_comp *comp; 472 uint32_t reqlen, comp_len; 473 int error = -EIO; 474 unsigned int ofs; 475 uint32_t rid; 476 477 reqlen = sizeof(*req) + idlen; 478 req = hn_rndis_alloc(reqlen); 479 if (req == NULL) 480 return -ENOMEM; 481 482 comp_len = sizeof(*comp) + odlen; 483 comp = rte_zmalloc("QUERY", comp_len, rte_mem_page_size()); 484 if (!comp) { 485 error = -ENOMEM; 486 goto done; 487 } 488 comp->status = RNDIS_STATUS_PENDING; 489 490 rid = hn_rndis_rid(hv); 491 492 req->type = RNDIS_QUERY_MSG; 493 req->len = reqlen; 494 req->rid = rid; 495 req->oid = oid; 496 req->infobufoffset = RNDIS_QUERY_REQ_INFOBUFOFFSET; 497 req->infobuflen = idlen; 498 499 /* Input data immediately follows RNDIS query. */ 500 memcpy(req + 1, idata, idlen); 501 502 error = hn_rndis_execute(hv, rid, req, reqlen, 503 comp, comp_len, RNDIS_QUERY_CMPLT); 504 505 if (error) 506 goto done; 507 508 if (comp->status != RNDIS_STATUS_SUCCESS) { 509 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x failed: status 0x%08x", 510 oid, comp->status); 511 error = -EINVAL; 512 goto done; 513 } 514 515 if (comp->infobuflen == 0 || comp->infobufoffset == 0) { 516 /* No output data! */ 517 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x, no data", oid); 518 error = 0; 519 goto done; 520 } 521 522 /* 523 * Check output data length and offset. 524 */ 525 /* ofs is the offset from the beginning of comp. */ 526 ofs = RNDIS_QUERY_COMP_INFOBUFOFFSET_ABS(comp->infobufoffset); 527 if (ofs < sizeof(*comp) || ofs + comp->infobuflen > comp_len) { 528 PMD_DRV_LOG(ERR, "RNDIS query invalid comp ib off/len, %u/%u", 529 comp->infobufoffset, comp->infobuflen); 530 error = -EINVAL; 531 goto done; 532 } 533 534 /* Save output data. */ 535 if (comp->infobuflen < odlen) 536 odlen = comp->infobuflen; 537 538 /* ofs is the offset from the beginning of comp. */ 539 memcpy(odata, (const char *)comp + ofs, odlen); 540 541 error = 0; 542 done: 543 rte_free(comp); 544 rte_free(req); 545 return error; 546 } 547 548 static int 549 hn_rndis_halt(struct hn_data *hv) 550 { 551 struct rndis_halt_req *halt; 552 553 halt = hn_rndis_alloc(sizeof(*halt)); 554 if (halt == NULL) 555 return -ENOMEM; 556 557 halt->type = RNDIS_HALT_MSG; 558 halt->len = sizeof(*halt); 559 halt->rid = hn_rndis_rid(hv); 560 561 /* No RNDIS completion; rely on NVS message send completion */ 562 hn_rndis_exec1(hv, halt, sizeof(*halt), NULL, 0); 563 564 rte_free(halt); 565 566 PMD_INIT_LOG(DEBUG, "RNDIS halt done"); 567 return 0; 568 } 569 570 static int 571 hn_rndis_query_hwcaps(struct hn_data *hv, struct ndis_offload *caps) 572 { 573 struct ndis_offload in; 574 uint32_t caps_len, size; 575 int error; 576 577 memset(caps, 0, sizeof(*caps)); 578 memset(&in, 0, sizeof(in)); 579 in.ndis_hdr.ndis_type = NDIS_OBJTYPE_OFFLOAD; 580 581 if (hv->ndis_ver >= NDIS_VERSION_6_30) { 582 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_3; 583 size = NDIS_OFFLOAD_SIZE; 584 } else if (hv->ndis_ver >= NDIS_VERSION_6_1) { 585 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_2; 586 size = NDIS_OFFLOAD_SIZE_6_1; 587 } else { 588 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_1; 589 size = NDIS_OFFLOAD_SIZE_6_0; 590 } 591 in.ndis_hdr.ndis_size = size; 592 593 caps_len = NDIS_OFFLOAD_SIZE; 594 error = hn_rndis_query(hv, OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES, 595 &in, size, caps, caps_len); 596 if (error) 597 return error; 598 599 /* Preliminary verification. */ 600 if (caps->ndis_hdr.ndis_type != NDIS_OBJTYPE_OFFLOAD) { 601 PMD_DRV_LOG(NOTICE, "invalid NDIS objtype 0x%02x", 602 caps->ndis_hdr.ndis_type); 603 return -EINVAL; 604 } 605 if (caps->ndis_hdr.ndis_rev < NDIS_OFFLOAD_REV_1) { 606 PMD_DRV_LOG(NOTICE, "invalid NDIS objrev 0x%02x", 607 caps->ndis_hdr.ndis_rev); 608 return -EINVAL; 609 } 610 if (caps->ndis_hdr.ndis_size > caps_len) { 611 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u, data size %u", 612 caps->ndis_hdr.ndis_size, caps_len); 613 return -EINVAL; 614 } else if (caps->ndis_hdr.ndis_size < NDIS_OFFLOAD_SIZE_6_0) { 615 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u", 616 caps->ndis_hdr.ndis_size); 617 return -EINVAL; 618 } 619 620 return 0; 621 } 622 623 int 624 hn_rndis_query_rsscaps(struct hn_data *hv, 625 unsigned int *rxr_cnt0) 626 { 627 struct ndis_rss_caps in, caps; 628 unsigned int indsz, rxr_cnt; 629 uint32_t caps_len; 630 int error; 631 632 *rxr_cnt0 = 0; 633 634 if (hv->ndis_ver < NDIS_VERSION_6_20) { 635 PMD_DRV_LOG(DEBUG, "RSS not supported on this host"); 636 return -EOPNOTSUPP; 637 } 638 639 memset(&in, 0, sizeof(in)); 640 in.ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_CAPS; 641 in.ndis_hdr.ndis_rev = NDIS_RSS_CAPS_REV_2; 642 in.ndis_hdr.ndis_size = NDIS_RSS_CAPS_SIZE; 643 644 caps_len = NDIS_RSS_CAPS_SIZE; 645 error = hn_rndis_query(hv, OID_GEN_RECEIVE_SCALE_CAPABILITIES, 646 &in, NDIS_RSS_CAPS_SIZE, 647 &caps, caps_len); 648 if (error) 649 return error; 650 651 PMD_INIT_LOG(DEBUG, "RX rings %u indirect %u caps %#x", 652 caps.ndis_nrxr, caps.ndis_nind, caps.ndis_caps); 653 /* 654 * Preliminary verification. 655 */ 656 if (caps.ndis_hdr.ndis_type != NDIS_OBJTYPE_RSS_CAPS) { 657 PMD_DRV_LOG(ERR, "invalid NDIS objtype 0x%02x", 658 caps.ndis_hdr.ndis_type); 659 return -EINVAL; 660 } 661 if (caps.ndis_hdr.ndis_rev < NDIS_RSS_CAPS_REV_1) { 662 PMD_DRV_LOG(ERR, "invalid NDIS objrev 0x%02x", 663 caps.ndis_hdr.ndis_rev); 664 return -EINVAL; 665 } 666 if (caps.ndis_hdr.ndis_size > caps_len) { 667 PMD_DRV_LOG(ERR, 668 "invalid NDIS objsize %u, data size %u", 669 caps.ndis_hdr.ndis_size, caps_len); 670 return -EINVAL; 671 } else if (caps.ndis_hdr.ndis_size < NDIS_RSS_CAPS_SIZE_6_0) { 672 PMD_DRV_LOG(ERR, "invalid NDIS objsize %u", 673 caps.ndis_hdr.ndis_size); 674 return -EINVAL; 675 } 676 677 /* 678 * Save information for later RSS configuration. 679 */ 680 if (caps.ndis_nrxr == 0) { 681 PMD_DRV_LOG(ERR, "0 RX rings!?"); 682 return -EINVAL; 683 } 684 rxr_cnt = caps.ndis_nrxr; 685 686 if (caps.ndis_hdr.ndis_size == NDIS_RSS_CAPS_SIZE && 687 caps.ndis_hdr.ndis_rev >= NDIS_RSS_CAPS_REV_2) { 688 if (caps.ndis_nind > NDIS_HASH_INDCNT) { 689 PMD_DRV_LOG(ERR, 690 "too many RSS indirect table entries %u", 691 caps.ndis_nind); 692 return -EOPNOTSUPP; 693 } 694 if (!rte_is_power_of_2(caps.ndis_nind)) { 695 PMD_DRV_LOG(ERR, 696 "RSS indirect table size is not power-of-2 %u", 697 caps.ndis_nind); 698 } 699 700 indsz = caps.ndis_nind; 701 } else { 702 indsz = NDIS_HASH_INDCNT; 703 } 704 705 if (indsz < rxr_cnt) { 706 PMD_DRV_LOG(NOTICE, 707 "# of RX rings (%d) > RSS indirect table size %d", 708 rxr_cnt, indsz); 709 rxr_cnt = indsz; 710 } 711 712 hv->rss_offloads = 0; 713 if (caps.ndis_caps & NDIS_RSS_CAP_IPV4) 714 hv->rss_offloads |= RTE_ETH_RSS_IPV4 715 | RTE_ETH_RSS_NONFRAG_IPV4_TCP 716 | RTE_ETH_RSS_NONFRAG_IPV4_UDP; 717 if (caps.ndis_caps & NDIS_RSS_CAP_IPV6) 718 hv->rss_offloads |= RTE_ETH_RSS_IPV6 719 | RTE_ETH_RSS_NONFRAG_IPV6_TCP; 720 if (caps.ndis_caps & NDIS_RSS_CAP_IPV6_EX) 721 hv->rss_offloads |= RTE_ETH_RSS_IPV6_EX 722 | RTE_ETH_RSS_IPV6_TCP_EX; 723 724 /* Commit! */ 725 *rxr_cnt0 = rxr_cnt; 726 727 return 0; 728 } 729 730 static int 731 hn_rndis_set(struct hn_data *hv, uint32_t oid, const void *data, uint32_t dlen) 732 { 733 struct rndis_set_req *req; 734 struct rndis_set_comp comp; 735 uint32_t reqlen, comp_len; 736 uint32_t rid; 737 int error; 738 739 reqlen = sizeof(*req) + dlen; 740 req = rte_zmalloc("RNDIS_SET", reqlen, rte_mem_page_size()); 741 if (!req) 742 return -ENOMEM; 743 744 rid = hn_rndis_rid(hv); 745 req->type = RNDIS_SET_MSG; 746 req->len = reqlen; 747 req->rid = rid; 748 req->oid = oid; 749 req->infobuflen = dlen; 750 req->infobufoffset = RNDIS_SET_REQ_INFOBUFOFFSET; 751 752 /* Data immediately follows RNDIS set. */ 753 memcpy(req + 1, data, dlen); 754 755 comp_len = sizeof(comp); 756 error = hn_rndis_execute(hv, rid, req, reqlen, 757 &comp, comp_len, 758 RNDIS_SET_CMPLT); 759 if (error) { 760 PMD_DRV_LOG(ERR, "exec RNDIS set %#" PRIx32 " failed", 761 oid); 762 error = EIO; 763 goto done; 764 } 765 766 if (comp.status != RNDIS_STATUS_SUCCESS) { 767 PMD_DRV_LOG(ERR, 768 "RNDIS set %#" PRIx32 " failed: status %#" PRIx32, 769 oid, comp.status); 770 error = EIO; 771 goto done; 772 } 773 774 done: 775 rte_free(req); 776 return error; 777 } 778 779 int hn_rndis_conf_offload(struct hn_data *hv, 780 uint64_t tx_offloads, uint64_t rx_offloads) 781 { 782 struct ndis_offload_params params; 783 struct ndis_offload hwcaps; 784 int error; 785 786 error = hn_rndis_query_hwcaps(hv, &hwcaps); 787 if (error) { 788 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 789 return error; 790 } 791 792 /* NOTE: 0 means "no change" */ 793 memset(¶ms, 0, sizeof(params)); 794 795 params.ndis_hdr.ndis_type = NDIS_OBJTYPE_DEFAULT; 796 if (hv->ndis_ver < NDIS_VERSION_6_30) { 797 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_2; 798 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE_6_1; 799 } else { 800 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_3; 801 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE; 802 } 803 804 if (tx_offloads & RTE_ETH_TX_OFFLOAD_TCP_CKSUM) { 805 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_TCP4) 806 params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_TX; 807 else 808 goto unsupported; 809 810 if (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_TCP6) 811 params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_TX; 812 else 813 goto unsupported; 814 } 815 816 if (rx_offloads & RTE_ETH_RX_OFFLOAD_TCP_CKSUM) { 817 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) 818 == NDIS_RXCSUM_CAP_TCP4) 819 params.ndis_tcp4csum |= NDIS_OFFLOAD_PARAM_RX; 820 else 821 goto unsupported; 822 823 if ((hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6) 824 == NDIS_RXCSUM_CAP_TCP6) 825 params.ndis_tcp6csum |= NDIS_OFFLOAD_PARAM_RX; 826 else 827 goto unsupported; 828 } 829 830 if (tx_offloads & RTE_ETH_TX_OFFLOAD_UDP_CKSUM) { 831 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) 832 params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_TX; 833 else 834 goto unsupported; 835 836 if ((hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6) 837 == NDIS_TXCSUM_CAP_UDP6) 838 params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_TX; 839 else 840 goto unsupported; 841 } 842 843 if (rx_offloads & RTE_ETH_TX_OFFLOAD_UDP_CKSUM) { 844 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) 845 params.ndis_udp4csum |= NDIS_OFFLOAD_PARAM_RX; 846 else 847 goto unsupported; 848 849 if (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6) 850 params.ndis_udp6csum |= NDIS_OFFLOAD_PARAM_RX; 851 else 852 goto unsupported; 853 } 854 855 if (tx_offloads & RTE_ETH_TX_OFFLOAD_IPV4_CKSUM) { 856 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_IP4) 857 == NDIS_TXCSUM_CAP_IP4) 858 params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_TX; 859 else 860 goto unsupported; 861 } 862 if (rx_offloads & RTE_ETH_RX_OFFLOAD_IPV4_CKSUM) { 863 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 864 params.ndis_ip4csum |= NDIS_OFFLOAD_PARAM_RX; 865 else 866 goto unsupported; 867 } 868 869 if (tx_offloads & RTE_ETH_TX_OFFLOAD_TCP_TSO) { 870 if (hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) 871 params.ndis_lsov2_ip4 = NDIS_OFFLOAD_LSOV2_ON; 872 else 873 goto unsupported; 874 875 if ((hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6) 876 == HN_NDIS_LSOV2_CAP_IP6) 877 params.ndis_lsov2_ip6 = NDIS_OFFLOAD_LSOV2_ON; 878 else 879 goto unsupported; 880 } 881 882 error = hn_rndis_set(hv, OID_TCP_OFFLOAD_PARAMETERS, ¶ms, 883 params.ndis_hdr.ndis_size); 884 if (error) { 885 PMD_DRV_LOG(ERR, "offload config failed"); 886 return error; 887 } 888 889 return 0; 890 unsupported: 891 PMD_DRV_LOG(NOTICE, 892 "offload tx:%" PRIx64 " rx:%" PRIx64 " not supported by this version", 893 tx_offloads, rx_offloads); 894 return -EINVAL; 895 } 896 897 int hn_rndis_get_offload(struct hn_data *hv, 898 struct rte_eth_dev_info *dev_info) 899 { 900 struct ndis_offload hwcaps; 901 int error; 902 903 memset(&hwcaps, 0, sizeof(hwcaps)); 904 905 error = hn_rndis_query_hwcaps(hv, &hwcaps); 906 if (error) { 907 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 908 return error; 909 } 910 911 dev_info->tx_offload_capa = RTE_ETH_TX_OFFLOAD_MULTI_SEGS | 912 RTE_ETH_TX_OFFLOAD_VLAN_INSERT; 913 914 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_IP4) 915 == HN_NDIS_TXCSUM_CAP_IP4) 916 dev_info->tx_offload_capa |= RTE_ETH_TX_OFFLOAD_IPV4_CKSUM; 917 918 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_TCP4) 919 == HN_NDIS_TXCSUM_CAP_TCP4 && 920 (hwcaps.ndis_csum.ndis_ip6_txcsum & HN_NDIS_TXCSUM_CAP_TCP6) 921 == HN_NDIS_TXCSUM_CAP_TCP6) 922 dev_info->tx_offload_capa |= RTE_ETH_TX_OFFLOAD_TCP_CKSUM; 923 924 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) && 925 (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6)) 926 dev_info->tx_offload_capa |= RTE_ETH_TX_OFFLOAD_UDP_CKSUM; 927 928 if ((hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) && 929 (hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6) 930 == HN_NDIS_LSOV2_CAP_IP6) 931 dev_info->tx_offload_capa |= RTE_ETH_TX_OFFLOAD_TCP_TSO; 932 933 dev_info->rx_offload_capa = RTE_ETH_RX_OFFLOAD_VLAN_STRIP | 934 RTE_ETH_RX_OFFLOAD_RSS_HASH; 935 936 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 937 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_IPV4_CKSUM; 938 939 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) && 940 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)) 941 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_TCP_CKSUM; 942 943 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) && 944 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)) 945 dev_info->rx_offload_capa |= RTE_ETH_RX_OFFLOAD_UDP_CKSUM; 946 947 return 0; 948 } 949 950 uint32_t 951 hn_rndis_get_ptypes(struct hn_data *hv) 952 { 953 struct ndis_offload hwcaps; 954 uint32_t ptypes; 955 int error; 956 957 memset(&hwcaps, 0, sizeof(hwcaps)); 958 959 error = hn_rndis_query_hwcaps(hv, &hwcaps); 960 if (error) { 961 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 962 return RTE_PTYPE_L2_ETHER; 963 } 964 965 ptypes = RTE_PTYPE_L2_ETHER; 966 967 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 968 ptypes |= RTE_PTYPE_L3_IPV4; 969 970 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) || 971 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)) 972 ptypes |= RTE_PTYPE_L4_TCP; 973 974 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) || 975 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)) 976 ptypes |= RTE_PTYPE_L4_UDP; 977 978 return ptypes; 979 } 980 981 int 982 hn_rndis_set_rxfilter(struct hn_data *hv, uint32_t filter) 983 { 984 int error; 985 986 error = hn_rndis_set(hv, OID_GEN_CURRENT_PACKET_FILTER, 987 &filter, sizeof(filter)); 988 if (error) { 989 PMD_DRV_LOG(ERR, "set RX filter %#" PRIx32 " failed: %d", 990 filter, error); 991 } else { 992 PMD_DRV_LOG(DEBUG, "set RX filter %#" PRIx32 " done", filter); 993 } 994 995 return error; 996 } 997 998 int hn_rndis_conf_rss(struct hn_data *hv, uint32_t flags) 999 { 1000 struct ndis_rssprm_toeplitz rssp; 1001 struct ndis_rss_params *prm = &rssp.rss_params; 1002 unsigned int i; 1003 int error; 1004 1005 memset(&rssp, 0, sizeof(rssp)); 1006 1007 prm->ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_PARAMS; 1008 prm->ndis_hdr.ndis_rev = NDIS_RSS_PARAMS_REV_2; 1009 prm->ndis_hdr.ndis_size = sizeof(*prm); 1010 prm->ndis_flags = flags; 1011 prm->ndis_hash = hv->rss_hash; 1012 prm->ndis_indsize = sizeof(rssp.rss_ind[0]) * NDIS_HASH_INDCNT; 1013 prm->ndis_indoffset = offsetof(struct ndis_rssprm_toeplitz, rss_ind[0]); 1014 prm->ndis_keysize = NDIS_HASH_KEYSIZE_TOEPLITZ; 1015 prm->ndis_keyoffset = offsetof(struct ndis_rssprm_toeplitz, rss_key[0]); 1016 1017 for (i = 0; i < NDIS_HASH_INDCNT; i++) 1018 rssp.rss_ind[i] = hv->rss_ind[i]; 1019 1020 /* Set hask key values */ 1021 memcpy(&rssp.rss_key, hv->rss_key, NDIS_HASH_KEYSIZE_TOEPLITZ); 1022 1023 error = hn_rndis_set(hv, OID_GEN_RECEIVE_SCALE_PARAMETERS, 1024 &rssp, sizeof(rssp)); 1025 if (error != 0) { 1026 PMD_DRV_LOG(ERR, 1027 "RSS config num queues=%u failed: %d", 1028 hv->num_queues, error); 1029 } 1030 return error; 1031 } 1032 1033 static int hn_rndis_init(struct hn_data *hv) 1034 { 1035 struct rndis_init_req *req; 1036 struct rndis_init_comp comp; 1037 uint32_t comp_len, rid; 1038 int error; 1039 1040 req = hn_rndis_alloc(sizeof(*req)); 1041 if (!req) { 1042 PMD_DRV_LOG(ERR, "no memory for RNDIS init"); 1043 return -ENXIO; 1044 } 1045 1046 rid = hn_rndis_rid(hv); 1047 req->type = RNDIS_INITIALIZE_MSG; 1048 req->len = sizeof(*req); 1049 req->rid = rid; 1050 req->ver_major = RNDIS_VERSION_MAJOR; 1051 req->ver_minor = RNDIS_VERSION_MINOR; 1052 req->max_xfersz = HN_RNDIS_XFER_SIZE; 1053 1054 comp_len = RNDIS_INIT_COMP_SIZE_MIN; 1055 error = hn_rndis_execute(hv, rid, req, sizeof(*req), 1056 &comp, comp_len, 1057 RNDIS_INITIALIZE_CMPLT); 1058 if (error) 1059 goto done; 1060 1061 if (comp.status != RNDIS_STATUS_SUCCESS) { 1062 PMD_DRV_LOG(ERR, "RNDIS init failed: status 0x%08x", 1063 comp.status); 1064 error = -EIO; 1065 goto done; 1066 } 1067 1068 hv->rndis_agg_size = comp.pktmaxsz; 1069 hv->rndis_agg_pkts = comp.pktmaxcnt; 1070 hv->rndis_agg_align = 1U << comp.align; 1071 1072 if (hv->rndis_agg_align < sizeof(uint32_t)) { 1073 /* 1074 * The RNDIS packet message encap assumes that the RNDIS 1075 * packet message is at least 4 bytes aligned. Fix up the 1076 * alignment here, if the remote side sets the alignment 1077 * too low. 1078 */ 1079 PMD_DRV_LOG(NOTICE, 1080 "fixup RNDIS aggpkt align: %u -> %zu", 1081 hv->rndis_agg_align, sizeof(uint32_t)); 1082 hv->rndis_agg_align = sizeof(uint32_t); 1083 } 1084 1085 PMD_INIT_LOG(INFO, 1086 "RNDIS ver %u.%u, aggpkt size %u, aggpkt cnt %u, aggpkt align %u", 1087 comp.ver_major, comp.ver_minor, 1088 hv->rndis_agg_size, hv->rndis_agg_pkts, 1089 hv->rndis_agg_align); 1090 error = 0; 1091 done: 1092 rte_free(req); 1093 return error; 1094 } 1095 1096 int 1097 hn_rndis_get_eaddr(struct hn_data *hv, uint8_t *eaddr) 1098 { 1099 uint32_t eaddr_len; 1100 int error; 1101 1102 eaddr_len = RTE_ETHER_ADDR_LEN; 1103 error = hn_rndis_query(hv, OID_802_3_PERMANENT_ADDRESS, NULL, 0, 1104 eaddr, eaddr_len); 1105 if (error) 1106 return error; 1107 1108 PMD_DRV_LOG(INFO, "MAC address " RTE_ETHER_ADDR_PRT_FMT, 1109 eaddr[0], eaddr[1], eaddr[2], 1110 eaddr[3], eaddr[4], eaddr[5]); 1111 return 0; 1112 } 1113 1114 int 1115 hn_rndis_get_mtu(struct hn_data *hv, uint32_t *mtu) 1116 { 1117 return hn_rndis_query(hv, OID_GEN_MAXIMUM_FRAME_SIZE, NULL, 0, 1118 mtu, sizeof(uint32_t)); 1119 } 1120 1121 int 1122 hn_rndis_get_linkstatus(struct hn_data *hv) 1123 { 1124 return hn_rndis_query(hv, OID_GEN_MEDIA_CONNECT_STATUS, NULL, 0, 1125 &hv->link_status, sizeof(uint32_t)); 1126 } 1127 1128 int 1129 hn_rndis_get_linkspeed(struct hn_data *hv) 1130 { 1131 return hn_rndis_query(hv, OID_GEN_LINK_SPEED, NULL, 0, 1132 &hv->link_speed, sizeof(uint32_t)); 1133 } 1134 1135 int 1136 hn_rndis_attach(struct hn_data *hv) 1137 { 1138 /* Initialize RNDIS. */ 1139 return hn_rndis_init(hv); 1140 } 1141 1142 void 1143 hn_rndis_detach(struct hn_data *hv) 1144 { 1145 struct rte_eth_dev *dev = &rte_eth_devices[hv->port_id]; 1146 1147 rte_eal_alarm_cancel(hn_rndis_link_alarm, dev); 1148 1149 /* Halt the RNDIS. */ 1150 hn_rndis_halt(hv); 1151 } 1152