1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2009-2018 Microsoft Corp. 3 * Copyright (c) 2010-2012 Citrix Inc. 4 * Copyright (c) 2012 NetApp Inc. 5 * All rights reserved. 6 */ 7 8 #include <stdint.h> 9 #include <string.h> 10 #include <stdio.h> 11 #include <errno.h> 12 #include <unistd.h> 13 14 #include <rte_ethdev_driver.h> 15 #include <rte_ethdev.h> 16 #include <rte_string_fns.h> 17 #include <rte_memzone.h> 18 #include <rte_malloc.h> 19 #include <rte_atomic.h> 20 #include <rte_branch_prediction.h> 21 #include <rte_ether.h> 22 #include <rte_common.h> 23 #include <rte_errno.h> 24 #include <rte_cycles.h> 25 #include <rte_memory.h> 26 #include <rte_eal.h> 27 #include <rte_dev.h> 28 #include <rte_bus_vmbus.h> 29 30 #include "hn_logs.h" 31 #include "hn_var.h" 32 #include "hn_nvs.h" 33 #include "hn_rndis.h" 34 #include "ndis.h" 35 36 #define HN_RNDIS_XFER_SIZE 0x4000 37 38 #define HN_NDIS_TXCSUM_CAP_IP4 \ 39 (NDIS_TXCSUM_CAP_IP4 | NDIS_TXCSUM_CAP_IP4OPT) 40 #define HN_NDIS_TXCSUM_CAP_TCP4 \ 41 (NDIS_TXCSUM_CAP_TCP4 | NDIS_TXCSUM_CAP_TCP4OPT) 42 #define HN_NDIS_TXCSUM_CAP_TCP6 \ 43 (NDIS_TXCSUM_CAP_TCP6 | NDIS_TXCSUM_CAP_TCP6OPT | \ 44 NDIS_TXCSUM_CAP_IP6EXT) 45 #define HN_NDIS_TXCSUM_CAP_UDP6 \ 46 (NDIS_TXCSUM_CAP_UDP6 | NDIS_TXCSUM_CAP_IP6EXT) 47 #define HN_NDIS_LSOV2_CAP_IP6 \ 48 (NDIS_LSOV2_CAP_IP6EXT | NDIS_LSOV2_CAP_TCP6OPT) 49 50 /* Get unique request id */ 51 static inline uint32_t 52 hn_rndis_rid(struct hn_data *hv) 53 { 54 uint32_t rid; 55 56 do { 57 rid = rte_atomic32_add_return(&hv->rndis_req_id, 1); 58 } while (rid == 0); 59 60 return rid; 61 } 62 63 static void *hn_rndis_alloc(size_t size) 64 { 65 return rte_zmalloc("RNDIS", size, PAGE_SIZE); 66 } 67 68 #ifdef RTE_LIBRTE_NETVSC_DEBUG_DUMP 69 void hn_rndis_dump(const void *buf) 70 { 71 const union { 72 struct rndis_msghdr hdr; 73 struct rndis_packet_msg pkt; 74 struct rndis_init_req init_request; 75 struct rndis_init_comp init_complete; 76 struct rndis_halt_req halt; 77 struct rndis_query_req query_request; 78 struct rndis_query_comp query_complete; 79 struct rndis_set_req set_request; 80 struct rndis_set_comp set_complete; 81 struct rndis_reset_req reset_request; 82 struct rndis_reset_comp reset_complete; 83 struct rndis_keepalive_req keepalive_request; 84 struct rndis_keepalive_comp keepalive_complete; 85 struct rndis_status_msg indicate_status; 86 } *rndis_msg = buf; 87 88 switch (rndis_msg->hdr.type) { 89 case RNDIS_PACKET_MSG: { 90 const struct rndis_pktinfo *ppi; 91 unsigned int ppi_len; 92 93 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 94 "RNDIS_MSG_PACKET (len %u, data %u:%u, # oob %u %u:%u, pkt %u:%u)\n", 95 rndis_msg->pkt.len, 96 rndis_msg->pkt.dataoffset, 97 rndis_msg->pkt.datalen, 98 rndis_msg->pkt.oobdataelements, 99 rndis_msg->pkt.oobdataoffset, 100 rndis_msg->pkt.oobdatalen, 101 rndis_msg->pkt.pktinfooffset, 102 rndis_msg->pkt.pktinfolen); 103 104 ppi = (const struct rndis_pktinfo *) 105 ((const char *)buf 106 + RNDIS_PACKET_MSG_OFFSET_ABS(rndis_msg->pkt.pktinfooffset)); 107 108 ppi_len = rndis_msg->pkt.pktinfolen; 109 while (ppi_len > 0) { 110 const void *ppi_data; 111 112 ppi_data = ppi->data; 113 114 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 115 " PPI (size %u, type %u, offs %u data %#x)\n", 116 ppi->size, ppi->type, ppi->offset, 117 *(const uint32_t *)ppi_data); 118 if (ppi->size == 0) 119 break; 120 ppi_len -= ppi->size; 121 ppi = (const struct rndis_pktinfo *) 122 ((const char *)ppi + ppi->size); 123 } 124 break; 125 } 126 case RNDIS_INITIALIZE_MSG: 127 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 128 "RNDIS_MSG_INIT (len %u id %#x, ver %u.%u max xfer %u)\n", 129 rndis_msg->init_request.len, 130 rndis_msg->init_request.rid, 131 rndis_msg->init_request.ver_major, 132 rndis_msg->init_request.ver_minor, 133 rndis_msg->init_request.max_xfersz); 134 break; 135 136 case RNDIS_INITIALIZE_CMPLT: 137 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 138 "RNDIS_MSG_INIT_C (len %u, id %#x, status 0x%x, vers %u.%u, " 139 "flags %d, max xfer %u, max pkts %u, aligned %u)\n", 140 rndis_msg->init_complete.len, 141 rndis_msg->init_complete.rid, 142 rndis_msg->init_complete.status, 143 rndis_msg->init_complete.ver_major, 144 rndis_msg->init_complete.ver_minor, 145 rndis_msg->init_complete.devflags, 146 rndis_msg->init_complete.pktmaxsz, 147 rndis_msg->init_complete.pktmaxcnt, 148 rndis_msg->init_complete.align); 149 break; 150 151 case RNDIS_HALT_MSG: 152 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 153 "RNDIS_HALT (len %u id %#x)\n", 154 rndis_msg->halt.len, rndis_msg->halt.rid); 155 break; 156 157 case RNDIS_QUERY_MSG: 158 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 159 "RNDIS_QUERY (len %u, id %#x, oid %#x, info %u:%u)\n", 160 rndis_msg->query_request.len, 161 rndis_msg->query_request.rid, 162 rndis_msg->query_request.oid, 163 rndis_msg->query_request.infobuflen, 164 rndis_msg->query_request.infobufoffset); 165 break; 166 167 case RNDIS_QUERY_CMPLT: 168 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 169 "RNDIS_MSG_QUERY_C (len %u, id %#x, status 0x%x, buf %u:%u)\n", 170 rndis_msg->query_complete.len, 171 rndis_msg->query_complete.rid, 172 rndis_msg->query_complete.status, 173 rndis_msg->query_complete.infobuflen, 174 rndis_msg->query_complete.infobufoffset); 175 break; 176 177 case RNDIS_SET_MSG: 178 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 179 "RNDIS_SET (len %u, id %#x, oid %#x, info %u:%u)\n", 180 rndis_msg->set_request.len, 181 rndis_msg->set_request.rid, 182 rndis_msg->set_request.oid, 183 rndis_msg->set_request.infobuflen, 184 rndis_msg->set_request.infobufoffset); 185 break; 186 187 case RNDIS_SET_CMPLT: 188 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 189 "RNDIS_MSG_SET_C (len %u, id 0x%x, status 0x%x)\n", 190 rndis_msg->set_complete.len, 191 rndis_msg->set_complete.rid, 192 rndis_msg->set_complete.status); 193 break; 194 195 case RNDIS_INDICATE_STATUS_MSG: 196 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 197 "RNDIS_MSG_INDICATE (len %u, status %#x, buf len %u, buf offset %u)\n", 198 rndis_msg->indicate_status.len, 199 rndis_msg->indicate_status.status, 200 rndis_msg->indicate_status.stbuflen, 201 rndis_msg->indicate_status.stbufoffset); 202 break; 203 204 case RNDIS_RESET_MSG: 205 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 206 "RNDIS_RESET (len %u, id %#x)\n", 207 rndis_msg->reset_request.len, 208 rndis_msg->reset_request.rid); 209 break; 210 211 case RNDIS_RESET_CMPLT: 212 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 213 "RNDIS_RESET_C (len %u, status %#x address %#x)\n", 214 rndis_msg->reset_complete.len, 215 rndis_msg->reset_complete.status, 216 rndis_msg->reset_complete.adrreset); 217 break; 218 219 case RNDIS_KEEPALIVE_MSG: 220 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 221 "RNDIS_KEEPALIVE (len %u, id %#x)\n", 222 rndis_msg->keepalive_request.len, 223 rndis_msg->keepalive_request.rid); 224 break; 225 226 case RNDIS_KEEPALIVE_CMPLT: 227 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 228 "RNDIS_KEEPALIVE_C (len %u, id %#x address %#x)\n", 229 rndis_msg->keepalive_complete.len, 230 rndis_msg->keepalive_complete.rid, 231 rndis_msg->keepalive_complete.status); 232 break; 233 234 default: 235 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 236 "RNDIS type %#x len %u\n", 237 rndis_msg->hdr.type, 238 rndis_msg->hdr.len); 239 break; 240 } 241 } 242 #endif 243 244 static int hn_nvs_send_rndis_ctrl(struct vmbus_channel *chan, 245 const void *req, uint32_t reqlen) 246 247 { 248 struct hn_nvs_rndis nvs_rndis = { 249 .type = NVS_TYPE_RNDIS, 250 .rndis_mtype = NVS_RNDIS_MTYPE_CTRL, 251 .chim_idx = NVS_CHIM_IDX_INVALID, 252 .chim_sz = 0 253 }; 254 struct vmbus_gpa sg; 255 rte_iova_t addr; 256 257 addr = rte_malloc_virt2iova(req); 258 if (unlikely(addr == RTE_BAD_IOVA)) { 259 PMD_DRV_LOG(ERR, "RNDIS send request can not get iova"); 260 return -EINVAL; 261 } 262 263 if (unlikely(reqlen > PAGE_SIZE)) { 264 PMD_DRV_LOG(ERR, "RNDIS request %u greater than page size", 265 reqlen); 266 return -EINVAL; 267 } 268 269 sg.page = addr / PAGE_SIZE; 270 sg.ofs = addr & PAGE_MASK; 271 sg.len = reqlen; 272 273 if (sg.ofs + reqlen > PAGE_SIZE) { 274 PMD_DRV_LOG(ERR, "RNDIS request crosses page bounary"); 275 return -EINVAL; 276 } 277 278 hn_rndis_dump(req); 279 280 return hn_nvs_send_sglist(chan, &sg, 1, 281 &nvs_rndis, sizeof(nvs_rndis), 0U, NULL); 282 } 283 284 void hn_rndis_link_status(struct rte_eth_dev *dev, const void *msg) 285 { 286 const struct rndis_status_msg *indicate = msg; 287 288 hn_rndis_dump(msg); 289 290 PMD_DRV_LOG(DEBUG, "link status %#x", indicate->status); 291 292 switch (indicate->status) { 293 case RNDIS_STATUS_NETWORK_CHANGE: 294 case RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG: 295 /* ignore not in DPDK API */ 296 break; 297 298 case RNDIS_STATUS_LINK_SPEED_CHANGE: 299 case RNDIS_STATUS_MEDIA_CONNECT: 300 case RNDIS_STATUS_MEDIA_DISCONNECT: 301 if (dev->data->dev_conf.intr_conf.lsc && 302 hn_dev_link_update(dev, 0) == 0) 303 _rte_eth_dev_callback_process(dev, 304 RTE_ETH_EVENT_INTR_LSC, 305 NULL); 306 break; 307 default: 308 PMD_DRV_LOG(NOTICE, "unknown RNDIS indication: %#x", 309 indicate->status); 310 } 311 } 312 313 /* Callback from hn_process_events when response is visible */ 314 void hn_rndis_receive_response(struct hn_data *hv, 315 const void *data, uint32_t len) 316 { 317 const struct rndis_init_comp *hdr = data; 318 319 hn_rndis_dump(data); 320 321 if (len < sizeof(3 * sizeof(uint32_t))) { 322 PMD_DRV_LOG(ERR, 323 "missing RNDIS header %u", len); 324 return; 325 } 326 327 if (len < hdr->len) { 328 PMD_DRV_LOG(ERR, 329 "truncated RNDIS response %u", len); 330 return; 331 } 332 333 if (len > sizeof(hv->rndis_resp)) { 334 PMD_DRV_LOG(NOTICE, 335 "RNDIS response exceeds buffer"); 336 len = sizeof(hv->rndis_resp); 337 } 338 339 if (hdr->rid == 0) { 340 PMD_DRV_LOG(NOTICE, 341 "RNDIS response id zero!"); 342 } 343 344 memcpy(hv->rndis_resp, data, len); 345 346 /* make sure response copied before update */ 347 rte_smp_wmb(); 348 349 if (rte_atomic32_cmpset(&hv->rndis_pending, hdr->rid, 0) == 0) { 350 PMD_DRV_LOG(ERR, 351 "received id %#x pending id %#x", 352 hdr->rid, (uint32_t)hv->rndis_pending); 353 } 354 } 355 356 /* Do request/response transaction */ 357 static int hn_rndis_exec1(struct hn_data *hv, 358 const void *req, uint32_t reqlen, 359 void *comp, uint32_t comp_len) 360 { 361 const struct rndis_halt_req *hdr = req; 362 uint32_t rid = hdr->rid; 363 struct vmbus_channel *chan = hn_primary_chan(hv); 364 int error; 365 366 if (comp_len > sizeof(hv->rndis_resp)) { 367 PMD_DRV_LOG(ERR, 368 "Expected completion size %u exceeds buffer %zu", 369 comp_len, sizeof(hv->rndis_resp)); 370 return -EIO; 371 } 372 373 if (comp != NULL && 374 rte_atomic32_cmpset(&hv->rndis_pending, 0, rid) == 0) { 375 PMD_DRV_LOG(ERR, 376 "Request already pending"); 377 return -EBUSY; 378 } 379 380 error = hn_nvs_send_rndis_ctrl(chan, req, reqlen); 381 if (error) { 382 PMD_DRV_LOG(ERR, "RNDIS ctrl send failed: %d", error); 383 return error; 384 } 385 386 if (comp) { 387 /* Poll primary channel until response received */ 388 while (hv->rndis_pending == rid) 389 hn_process_events(hv, 0, 1); 390 391 memcpy(comp, hv->rndis_resp, comp_len); 392 } 393 394 return 0; 395 } 396 397 /* Do transaction and validate response */ 398 static int hn_rndis_execute(struct hn_data *hv, uint32_t rid, 399 const void *req, uint32_t reqlen, 400 void *comp, uint32_t comp_len, uint32_t comp_type) 401 { 402 const struct rndis_comp_hdr *hdr = comp; 403 int ret; 404 405 memset(comp, 0, comp_len); 406 407 ret = hn_rndis_exec1(hv, req, reqlen, comp, comp_len); 408 if (ret < 0) 409 return ret; 410 /* 411 * Check this RNDIS complete message. 412 */ 413 if (unlikely(hdr->type != comp_type)) { 414 PMD_DRV_LOG(ERR, 415 "unexpected RNDIS response complete %#x expect %#x", 416 hdr->type, comp_type); 417 418 return -ENXIO; 419 } 420 if (unlikely(hdr->rid != rid)) { 421 PMD_DRV_LOG(ERR, 422 "RNDIS comp rid mismatch %#x, expect %#x", 423 hdr->rid, rid); 424 return -EINVAL; 425 } 426 427 /* All pass! */ 428 return 0; 429 } 430 431 static int 432 hn_rndis_query(struct hn_data *hv, uint32_t oid, 433 const void *idata, uint32_t idlen, 434 void *odata, uint32_t odlen) 435 { 436 struct rndis_query_req *req; 437 struct rndis_query_comp *comp; 438 uint32_t reqlen, comp_len; 439 int error = -EIO; 440 unsigned int ofs; 441 uint32_t rid; 442 443 reqlen = sizeof(*req) + idlen; 444 req = hn_rndis_alloc(reqlen); 445 if (req == NULL) 446 return -ENOMEM; 447 448 comp_len = sizeof(*comp) + odlen; 449 comp = rte_zmalloc("QUERY", comp_len, PAGE_SIZE); 450 if (!comp) { 451 error = -ENOMEM; 452 goto done; 453 } 454 comp->status = RNDIS_STATUS_PENDING; 455 456 rid = hn_rndis_rid(hv); 457 458 req->type = RNDIS_QUERY_MSG; 459 req->len = reqlen; 460 req->rid = rid; 461 req->oid = oid; 462 req->infobufoffset = RNDIS_QUERY_REQ_INFOBUFOFFSET; 463 req->infobuflen = idlen; 464 465 /* Input data immediately follows RNDIS query. */ 466 memcpy(req + 1, idata, idlen); 467 468 error = hn_rndis_execute(hv, rid, req, reqlen, 469 comp, comp_len, RNDIS_QUERY_CMPLT); 470 471 if (error) 472 goto done; 473 474 if (comp->status != RNDIS_STATUS_SUCCESS) { 475 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x failed: status 0x%08x", 476 oid, comp->status); 477 error = -EINVAL; 478 goto done; 479 } 480 481 if (comp->infobuflen == 0 || comp->infobufoffset == 0) { 482 /* No output data! */ 483 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x, no data", oid); 484 error = 0; 485 goto done; 486 } 487 488 /* 489 * Check output data length and offset. 490 */ 491 /* ofs is the offset from the beginning of comp. */ 492 ofs = RNDIS_QUERY_COMP_INFOBUFOFFSET_ABS(comp->infobufoffset); 493 if (ofs < sizeof(*comp) || ofs + comp->infobuflen > comp_len) { 494 PMD_DRV_LOG(ERR, "RNDIS query invalid comp ib off/len, %u/%u", 495 comp->infobufoffset, comp->infobuflen); 496 error = -EINVAL; 497 goto done; 498 } 499 500 /* Save output data. */ 501 if (comp->infobuflen < odlen) 502 odlen = comp->infobuflen; 503 504 /* ofs is the offset from the beginning of comp. */ 505 memcpy(odata, (const char *)comp + ofs, odlen); 506 507 error = 0; 508 done: 509 rte_free(comp); 510 rte_free(req); 511 return error; 512 } 513 514 static int 515 hn_rndis_halt(struct hn_data *hv) 516 { 517 struct rndis_halt_req *halt; 518 519 halt = hn_rndis_alloc(sizeof(*halt)); 520 if (halt == NULL) 521 return -ENOMEM; 522 523 halt->type = RNDIS_HALT_MSG; 524 halt->len = sizeof(*halt); 525 halt->rid = hn_rndis_rid(hv); 526 527 /* No RNDIS completion; rely on NVS message send completion */ 528 hn_rndis_exec1(hv, halt, sizeof(*halt), NULL, 0); 529 530 rte_free(halt); 531 532 PMD_INIT_LOG(DEBUG, "RNDIS halt done"); 533 return 0; 534 } 535 536 static int 537 hn_rndis_query_hwcaps(struct hn_data *hv, struct ndis_offload *caps) 538 { 539 struct ndis_offload in; 540 uint32_t caps_len, size; 541 int error; 542 543 memset(caps, 0, sizeof(*caps)); 544 memset(&in, 0, sizeof(in)); 545 in.ndis_hdr.ndis_type = NDIS_OBJTYPE_OFFLOAD; 546 547 if (hv->ndis_ver >= NDIS_VERSION_6_30) { 548 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_3; 549 size = NDIS_OFFLOAD_SIZE; 550 } else if (hv->ndis_ver >= NDIS_VERSION_6_1) { 551 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_2; 552 size = NDIS_OFFLOAD_SIZE_6_1; 553 } else { 554 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_1; 555 size = NDIS_OFFLOAD_SIZE_6_0; 556 } 557 in.ndis_hdr.ndis_size = size; 558 559 caps_len = NDIS_OFFLOAD_SIZE; 560 error = hn_rndis_query(hv, OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES, 561 &in, size, caps, caps_len); 562 if (error) 563 return error; 564 565 /* Preliminary verification. */ 566 if (caps->ndis_hdr.ndis_type != NDIS_OBJTYPE_OFFLOAD) { 567 PMD_DRV_LOG(NOTICE, "invalid NDIS objtype 0x%02x", 568 caps->ndis_hdr.ndis_type); 569 return -EINVAL; 570 } 571 if (caps->ndis_hdr.ndis_rev < NDIS_OFFLOAD_REV_1) { 572 PMD_DRV_LOG(NOTICE, "invalid NDIS objrev 0x%02x", 573 caps->ndis_hdr.ndis_rev); 574 return -EINVAL; 575 } 576 if (caps->ndis_hdr.ndis_size > caps_len) { 577 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u, data size %u", 578 caps->ndis_hdr.ndis_size, caps_len); 579 return -EINVAL; 580 } else if (caps->ndis_hdr.ndis_size < NDIS_OFFLOAD_SIZE_6_0) { 581 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u", 582 caps->ndis_hdr.ndis_size); 583 return -EINVAL; 584 } 585 586 return 0; 587 } 588 589 int 590 hn_rndis_query_rsscaps(struct hn_data *hv, 591 unsigned int *rxr_cnt0) 592 { 593 struct ndis_rss_caps in, caps; 594 unsigned int indsz, rxr_cnt; 595 uint32_t caps_len; 596 int error; 597 598 *rxr_cnt0 = 0; 599 600 if (hv->ndis_ver < NDIS_VERSION_6_20) { 601 PMD_DRV_LOG(DEBUG, "RSS not supported on this host"); 602 return -EOPNOTSUPP; 603 } 604 605 memset(&in, 0, sizeof(in)); 606 in.ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_CAPS; 607 in.ndis_hdr.ndis_rev = NDIS_RSS_CAPS_REV_2; 608 in.ndis_hdr.ndis_size = NDIS_RSS_CAPS_SIZE; 609 610 caps_len = NDIS_RSS_CAPS_SIZE; 611 error = hn_rndis_query(hv, OID_GEN_RECEIVE_SCALE_CAPABILITIES, 612 &in, NDIS_RSS_CAPS_SIZE, 613 &caps, caps_len); 614 if (error) 615 return error; 616 617 PMD_INIT_LOG(DEBUG, "RX rings %u indirect %u caps %#x", 618 caps.ndis_nrxr, caps.ndis_nind, caps.ndis_caps); 619 /* 620 * Preliminary verification. 621 */ 622 if (caps.ndis_hdr.ndis_type != NDIS_OBJTYPE_RSS_CAPS) { 623 PMD_DRV_LOG(ERR, "invalid NDIS objtype 0x%02x", 624 caps.ndis_hdr.ndis_type); 625 return -EINVAL; 626 } 627 if (caps.ndis_hdr.ndis_rev < NDIS_RSS_CAPS_REV_1) { 628 PMD_DRV_LOG(ERR, "invalid NDIS objrev 0x%02x", 629 caps.ndis_hdr.ndis_rev); 630 return -EINVAL; 631 } 632 if (caps.ndis_hdr.ndis_size > caps_len) { 633 PMD_DRV_LOG(ERR, 634 "invalid NDIS objsize %u, data size %u", 635 caps.ndis_hdr.ndis_size, caps_len); 636 return -EINVAL; 637 } else if (caps.ndis_hdr.ndis_size < NDIS_RSS_CAPS_SIZE_6_0) { 638 PMD_DRV_LOG(ERR, "invalid NDIS objsize %u", 639 caps.ndis_hdr.ndis_size); 640 return -EINVAL; 641 } 642 643 /* 644 * Save information for later RSS configuration. 645 */ 646 if (caps.ndis_nrxr == 0) { 647 PMD_DRV_LOG(ERR, "0 RX rings!?"); 648 return -EINVAL; 649 } 650 rxr_cnt = caps.ndis_nrxr; 651 652 if (caps.ndis_hdr.ndis_size == NDIS_RSS_CAPS_SIZE && 653 caps.ndis_hdr.ndis_rev >= NDIS_RSS_CAPS_REV_2) { 654 if (caps.ndis_nind > NDIS_HASH_INDCNT) { 655 PMD_DRV_LOG(ERR, 656 "too many RSS indirect table entries %u", 657 caps.ndis_nind); 658 return -EOPNOTSUPP; 659 } 660 if (!rte_is_power_of_2(caps.ndis_nind)) { 661 PMD_DRV_LOG(ERR, 662 "RSS indirect table size is not power-of-2 %u", 663 caps.ndis_nind); 664 } 665 666 indsz = caps.ndis_nind; 667 } else { 668 indsz = NDIS_HASH_INDCNT; 669 } 670 671 if (indsz < rxr_cnt) { 672 PMD_DRV_LOG(NOTICE, 673 "# of RX rings (%d) > RSS indirect table size %d", 674 rxr_cnt, indsz); 675 rxr_cnt = indsz; 676 } 677 678 hv->rss_offloads = 0; 679 if (caps.ndis_caps & NDIS_RSS_CAP_IPV4) 680 hv->rss_offloads |= ETH_RSS_IPV4 681 | ETH_RSS_NONFRAG_IPV4_TCP 682 | ETH_RSS_NONFRAG_IPV4_UDP; 683 if (caps.ndis_caps & NDIS_RSS_CAP_IPV6) 684 hv->rss_offloads |= ETH_RSS_IPV6 685 | ETH_RSS_NONFRAG_IPV6_TCP; 686 if (caps.ndis_caps & NDIS_RSS_CAP_IPV6_EX) 687 hv->rss_offloads |= ETH_RSS_IPV6_EX 688 | ETH_RSS_IPV6_TCP_EX; 689 690 /* Commit! */ 691 *rxr_cnt0 = rxr_cnt; 692 693 return 0; 694 } 695 696 static int 697 hn_rndis_set(struct hn_data *hv, uint32_t oid, const void *data, uint32_t dlen) 698 { 699 struct rndis_set_req *req; 700 struct rndis_set_comp comp; 701 uint32_t reqlen, comp_len; 702 uint32_t rid; 703 int error; 704 705 reqlen = sizeof(*req) + dlen; 706 req = rte_zmalloc("RNDIS_SET", reqlen, PAGE_SIZE); 707 if (!req) 708 return -ENOMEM; 709 710 rid = hn_rndis_rid(hv); 711 req->type = RNDIS_SET_MSG; 712 req->len = reqlen; 713 req->rid = rid; 714 req->oid = oid; 715 req->infobuflen = dlen; 716 req->infobufoffset = RNDIS_SET_REQ_INFOBUFOFFSET; 717 718 /* Data immediately follows RNDIS set. */ 719 memcpy(req + 1, data, dlen); 720 721 comp_len = sizeof(comp); 722 error = hn_rndis_execute(hv, rid, req, reqlen, 723 &comp, comp_len, 724 RNDIS_SET_CMPLT); 725 if (error) { 726 PMD_DRV_LOG(ERR, "exec RNDIS set %#" PRIx32 " failed", 727 oid); 728 error = EIO; 729 goto done; 730 } 731 732 if (comp.status != RNDIS_STATUS_SUCCESS) { 733 PMD_DRV_LOG(ERR, 734 "RNDIS set %#" PRIx32 " failed: status %#" PRIx32, 735 oid, comp.status); 736 error = EIO; 737 goto done; 738 } 739 740 done: 741 rte_free(req); 742 return error; 743 } 744 745 int hn_rndis_conf_offload(struct hn_data *hv, 746 uint64_t tx_offloads, uint64_t rx_offloads) 747 { 748 struct ndis_offload_params params; 749 struct ndis_offload hwcaps; 750 int error; 751 752 error = hn_rndis_query_hwcaps(hv, &hwcaps); 753 if (error) { 754 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 755 return error; 756 } 757 758 /* NOTE: 0 means "no change" */ 759 memset(¶ms, 0, sizeof(params)); 760 761 params.ndis_hdr.ndis_type = NDIS_OBJTYPE_DEFAULT; 762 if (hv->ndis_ver < NDIS_VERSION_6_30) { 763 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_2; 764 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE_6_1; 765 } else { 766 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_3; 767 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE; 768 } 769 770 if (tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM) { 771 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_TCP4) 772 params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_TX; 773 else 774 goto unsupported; 775 776 if (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_TCP6) 777 params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_TX; 778 else 779 goto unsupported; 780 } 781 782 if (rx_offloads & DEV_RX_OFFLOAD_TCP_CKSUM) { 783 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) 784 == NDIS_RXCSUM_CAP_TCP4) 785 params.ndis_tcp4csum |= NDIS_OFFLOAD_PARAM_RX; 786 else 787 goto unsupported; 788 789 if ((hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6) 790 == NDIS_RXCSUM_CAP_TCP6) 791 params.ndis_tcp6csum |= NDIS_OFFLOAD_PARAM_RX; 792 else 793 goto unsupported; 794 } 795 796 if (tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) { 797 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) 798 params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_TX; 799 else 800 goto unsupported; 801 802 if ((hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6) 803 == NDIS_TXCSUM_CAP_UDP6) 804 params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_TX; 805 else 806 goto unsupported; 807 } 808 809 if (rx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) { 810 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) 811 params.ndis_udp4csum |= NDIS_OFFLOAD_PARAM_RX; 812 else 813 goto unsupported; 814 815 if (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6) 816 params.ndis_udp6csum |= NDIS_OFFLOAD_PARAM_RX; 817 else 818 goto unsupported; 819 } 820 821 if (tx_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) { 822 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_IP4) 823 == NDIS_TXCSUM_CAP_IP4) 824 params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_TX; 825 else 826 goto unsupported; 827 } 828 if (rx_offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) { 829 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 830 params.ndis_ip4csum |= NDIS_OFFLOAD_PARAM_RX; 831 else 832 goto unsupported; 833 } 834 835 if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO) { 836 if (hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) 837 params.ndis_lsov2_ip4 = NDIS_OFFLOAD_LSOV2_ON; 838 else 839 goto unsupported; 840 841 if ((hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6) 842 == HN_NDIS_LSOV2_CAP_IP6) 843 params.ndis_lsov2_ip6 = NDIS_OFFLOAD_LSOV2_ON; 844 else 845 goto unsupported; 846 } 847 848 error = hn_rndis_set(hv, OID_TCP_OFFLOAD_PARAMETERS, ¶ms, 849 params.ndis_hdr.ndis_size); 850 if (error) { 851 PMD_DRV_LOG(ERR, "offload config failed"); 852 return error; 853 } 854 855 return 0; 856 unsupported: 857 PMD_DRV_LOG(NOTICE, 858 "offload tx:%" PRIx64 " rx:%" PRIx64 " not supported by this version", 859 tx_offloads, rx_offloads); 860 return -EINVAL; 861 } 862 863 int hn_rndis_get_offload(struct hn_data *hv, 864 struct rte_eth_dev_info *dev_info) 865 { 866 struct ndis_offload hwcaps; 867 int error; 868 869 memset(&hwcaps, 0, sizeof(hwcaps)); 870 871 error = hn_rndis_query_hwcaps(hv, &hwcaps); 872 if (error) { 873 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 874 return error; 875 } 876 877 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS | 878 DEV_TX_OFFLOAD_VLAN_INSERT; 879 880 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_IP4) 881 == HN_NDIS_TXCSUM_CAP_IP4) 882 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM; 883 884 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_TCP4) 885 == HN_NDIS_TXCSUM_CAP_TCP4 && 886 (hwcaps.ndis_csum.ndis_ip6_txcsum & HN_NDIS_TXCSUM_CAP_TCP6) 887 == HN_NDIS_TXCSUM_CAP_TCP6) 888 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_CKSUM; 889 890 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) && 891 (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6)) 892 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_UDP_CKSUM; 893 894 if ((hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) && 895 (hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6) 896 == HN_NDIS_LSOV2_CAP_IP6) 897 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; 898 899 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP | 900 DEV_RX_OFFLOAD_RSS_HASH; 901 902 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 903 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM; 904 905 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) && 906 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)) 907 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_CKSUM; 908 909 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) && 910 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)) 911 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_UDP_CKSUM; 912 913 return 0; 914 } 915 916 uint32_t 917 hn_rndis_get_ptypes(struct hn_data *hv) 918 { 919 struct ndis_offload hwcaps; 920 uint32_t ptypes; 921 int error; 922 923 memset(&hwcaps, 0, sizeof(hwcaps)); 924 925 error = hn_rndis_query_hwcaps(hv, &hwcaps); 926 if (error) { 927 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 928 return RTE_PTYPE_L2_ETHER; 929 } 930 931 ptypes = RTE_PTYPE_L2_ETHER; 932 933 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 934 ptypes |= RTE_PTYPE_L3_IPV4; 935 936 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) || 937 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)) 938 ptypes |= RTE_PTYPE_L4_TCP; 939 940 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) || 941 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)) 942 ptypes |= RTE_PTYPE_L4_UDP; 943 944 return ptypes; 945 } 946 947 int 948 hn_rndis_set_rxfilter(struct hn_data *hv, uint32_t filter) 949 { 950 int error; 951 952 error = hn_rndis_set(hv, OID_GEN_CURRENT_PACKET_FILTER, 953 &filter, sizeof(filter)); 954 if (error) { 955 PMD_DRV_LOG(ERR, "set RX filter %#" PRIx32 " failed: %d", 956 filter, error); 957 } else { 958 PMD_DRV_LOG(DEBUG, "set RX filter %#" PRIx32 " done", filter); 959 } 960 961 return error; 962 } 963 964 int hn_rndis_conf_rss(struct hn_data *hv, uint32_t flags) 965 { 966 struct ndis_rssprm_toeplitz rssp; 967 struct ndis_rss_params *prm = &rssp.rss_params; 968 unsigned int i; 969 int error; 970 971 memset(&rssp, 0, sizeof(rssp)); 972 973 prm->ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_PARAMS; 974 prm->ndis_hdr.ndis_rev = NDIS_RSS_PARAMS_REV_2; 975 prm->ndis_hdr.ndis_size = sizeof(*prm); 976 prm->ndis_flags = flags; 977 prm->ndis_hash = hv->rss_hash; 978 prm->ndis_indsize = sizeof(rssp.rss_ind[0]) * NDIS_HASH_INDCNT; 979 prm->ndis_indoffset = offsetof(struct ndis_rssprm_toeplitz, rss_ind[0]); 980 prm->ndis_keysize = NDIS_HASH_KEYSIZE_TOEPLITZ; 981 prm->ndis_keyoffset = offsetof(struct ndis_rssprm_toeplitz, rss_key[0]); 982 983 for (i = 0; i < NDIS_HASH_INDCNT; i++) 984 rssp.rss_ind[i] = hv->rss_ind[i]; 985 986 /* Set hask key values */ 987 memcpy(&rssp.rss_key, hv->rss_key, NDIS_HASH_KEYSIZE_TOEPLITZ); 988 989 error = hn_rndis_set(hv, OID_GEN_RECEIVE_SCALE_PARAMETERS, 990 &rssp, sizeof(rssp)); 991 if (error != 0) { 992 PMD_DRV_LOG(ERR, 993 "RSS config num queues=%u failed: %d", 994 hv->num_queues, error); 995 } 996 return error; 997 } 998 999 static int hn_rndis_init(struct hn_data *hv) 1000 { 1001 struct rndis_init_req *req; 1002 struct rndis_init_comp comp; 1003 uint32_t comp_len, rid; 1004 int error; 1005 1006 req = hn_rndis_alloc(sizeof(*req)); 1007 if (!req) { 1008 PMD_DRV_LOG(ERR, "no memory for RNDIS init"); 1009 return -ENXIO; 1010 } 1011 1012 rid = hn_rndis_rid(hv); 1013 req->type = RNDIS_INITIALIZE_MSG; 1014 req->len = sizeof(*req); 1015 req->rid = rid; 1016 req->ver_major = RNDIS_VERSION_MAJOR; 1017 req->ver_minor = RNDIS_VERSION_MINOR; 1018 req->max_xfersz = HN_RNDIS_XFER_SIZE; 1019 1020 comp_len = RNDIS_INIT_COMP_SIZE_MIN; 1021 error = hn_rndis_execute(hv, rid, req, sizeof(*req), 1022 &comp, comp_len, 1023 RNDIS_INITIALIZE_CMPLT); 1024 if (error) 1025 goto done; 1026 1027 if (comp.status != RNDIS_STATUS_SUCCESS) { 1028 PMD_DRV_LOG(ERR, "RNDIS init failed: status 0x%08x", 1029 comp.status); 1030 error = -EIO; 1031 goto done; 1032 } 1033 1034 hv->rndis_agg_size = comp.pktmaxsz; 1035 hv->rndis_agg_pkts = comp.pktmaxcnt; 1036 hv->rndis_agg_align = 1U << comp.align; 1037 1038 if (hv->rndis_agg_align < sizeof(uint32_t)) { 1039 /* 1040 * The RNDIS packet message encap assumes that the RNDIS 1041 * packet message is at least 4 bytes aligned. Fix up the 1042 * alignment here, if the remote side sets the alignment 1043 * too low. 1044 */ 1045 PMD_DRV_LOG(NOTICE, 1046 "fixup RNDIS aggpkt align: %u -> %zu", 1047 hv->rndis_agg_align, sizeof(uint32_t)); 1048 hv->rndis_agg_align = sizeof(uint32_t); 1049 } 1050 1051 PMD_INIT_LOG(INFO, 1052 "RNDIS ver %u.%u, aggpkt size %u, aggpkt cnt %u, aggpkt align %u", 1053 comp.ver_major, comp.ver_minor, 1054 hv->rndis_agg_size, hv->rndis_agg_pkts, 1055 hv->rndis_agg_align); 1056 error = 0; 1057 done: 1058 rte_free(req); 1059 return error; 1060 } 1061 1062 int 1063 hn_rndis_get_eaddr(struct hn_data *hv, uint8_t *eaddr) 1064 { 1065 uint32_t eaddr_len; 1066 int error; 1067 1068 eaddr_len = RTE_ETHER_ADDR_LEN; 1069 error = hn_rndis_query(hv, OID_802_3_PERMANENT_ADDRESS, NULL, 0, 1070 eaddr, eaddr_len); 1071 if (error) 1072 return error; 1073 1074 PMD_DRV_LOG(INFO, "MAC address %02x:%02x:%02x:%02x:%02x:%02x", 1075 eaddr[0], eaddr[1], eaddr[2], 1076 eaddr[3], eaddr[4], eaddr[5]); 1077 return 0; 1078 } 1079 1080 int 1081 hn_rndis_get_linkstatus(struct hn_data *hv) 1082 { 1083 return hn_rndis_query(hv, OID_GEN_MEDIA_CONNECT_STATUS, NULL, 0, 1084 &hv->link_status, sizeof(uint32_t)); 1085 } 1086 1087 int 1088 hn_rndis_get_linkspeed(struct hn_data *hv) 1089 { 1090 return hn_rndis_query(hv, OID_GEN_LINK_SPEED, NULL, 0, 1091 &hv->link_speed, sizeof(uint32_t)); 1092 } 1093 1094 int 1095 hn_rndis_attach(struct hn_data *hv) 1096 { 1097 /* Initialize RNDIS. */ 1098 return hn_rndis_init(hv); 1099 } 1100 1101 void 1102 hn_rndis_detach(struct hn_data *hv) 1103 { 1104 /* Halt the RNDIS. */ 1105 hn_rndis_halt(hv); 1106 } 1107