1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2009-2018 Microsoft Corp. 3 * Copyright (c) 2010-2012 Citrix Inc. 4 * Copyright (c) 2012 NetApp Inc. 5 * All rights reserved. 6 */ 7 8 #include <stdint.h> 9 #include <string.h> 10 #include <stdio.h> 11 #include <errno.h> 12 #include <unistd.h> 13 14 #include <rte_ethdev_driver.h> 15 #include <rte_ethdev.h> 16 #include <rte_string_fns.h> 17 #include <rte_memzone.h> 18 #include <rte_malloc.h> 19 #include <rte_atomic.h> 20 #include <rte_branch_prediction.h> 21 #include <rte_ether.h> 22 #include <rte_common.h> 23 #include <rte_errno.h> 24 #include <rte_cycles.h> 25 #include <rte_memory.h> 26 #include <rte_eal.h> 27 #include <rte_dev.h> 28 #include <rte_bus_vmbus.h> 29 30 #include "hn_logs.h" 31 #include "hn_var.h" 32 #include "hn_nvs.h" 33 #include "hn_rndis.h" 34 #include "ndis.h" 35 36 #define HN_RNDIS_XFER_SIZE 0x4000 37 38 #define HN_NDIS_TXCSUM_CAP_IP4 \ 39 (NDIS_TXCSUM_CAP_IP4 | NDIS_TXCSUM_CAP_IP4OPT) 40 #define HN_NDIS_TXCSUM_CAP_TCP4 \ 41 (NDIS_TXCSUM_CAP_TCP4 | NDIS_TXCSUM_CAP_TCP4OPT) 42 #define HN_NDIS_TXCSUM_CAP_TCP6 \ 43 (NDIS_TXCSUM_CAP_TCP6 | NDIS_TXCSUM_CAP_TCP6OPT | \ 44 NDIS_TXCSUM_CAP_IP6EXT) 45 #define HN_NDIS_TXCSUM_CAP_UDP6 \ 46 (NDIS_TXCSUM_CAP_UDP6 | NDIS_TXCSUM_CAP_IP6EXT) 47 #define HN_NDIS_LSOV2_CAP_IP6 \ 48 (NDIS_LSOV2_CAP_IP6EXT | NDIS_LSOV2_CAP_TCP6OPT) 49 50 /* Get unique request id */ 51 static inline uint32_t 52 hn_rndis_rid(struct hn_data *hv) 53 { 54 uint32_t rid; 55 56 do { 57 rid = rte_atomic32_add_return(&hv->rndis_req_id, 1); 58 } while (rid == 0); 59 60 return rid; 61 } 62 63 static void *hn_rndis_alloc(struct hn_data *hv, size_t size) 64 { 65 return rte_zmalloc_socket("RNDIS", size, PAGE_SIZE, 66 hv->vmbus->device.numa_node); 67 } 68 69 #ifdef RTE_LIBRTE_NETVSC_DEBUG_DUMP 70 void hn_rndis_dump(const void *buf) 71 { 72 const union { 73 struct rndis_msghdr hdr; 74 struct rndis_packet_msg pkt; 75 struct rndis_init_req init_request; 76 struct rndis_init_comp init_complete; 77 struct rndis_halt_req halt; 78 struct rndis_query_req query_request; 79 struct rndis_query_comp query_complete; 80 struct rndis_set_req set_request; 81 struct rndis_set_comp set_complete; 82 struct rndis_reset_req reset_request; 83 struct rndis_reset_comp reset_complete; 84 struct rndis_keepalive_req keepalive_request; 85 struct rndis_keepalive_comp keepalive_complete; 86 struct rndis_status_msg indicate_status; 87 } *rndis_msg = buf; 88 89 switch (rndis_msg->hdr.type) { 90 case RNDIS_PACKET_MSG: { 91 const struct rndis_pktinfo *ppi; 92 unsigned int ppi_len; 93 94 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 95 "RNDIS_MSG_PACKET (len %u, data %u:%u, # oob %u %u:%u, pkt %u:%u)\n", 96 rndis_msg->pkt.len, 97 rndis_msg->pkt.dataoffset, 98 rndis_msg->pkt.datalen, 99 rndis_msg->pkt.oobdataelements, 100 rndis_msg->pkt.oobdataoffset, 101 rndis_msg->pkt.oobdatalen, 102 rndis_msg->pkt.pktinfooffset, 103 rndis_msg->pkt.pktinfolen); 104 105 ppi = (const struct rndis_pktinfo *) 106 ((const char *)buf 107 + RNDIS_PACKET_MSG_OFFSET_ABS(rndis_msg->pkt.pktinfooffset)); 108 109 ppi_len = rndis_msg->pkt.pktinfolen; 110 while (ppi_len > 0) { 111 const void *ppi_data; 112 113 ppi_data = ppi->data; 114 115 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 116 " PPI (size %u, type %u, offs %u data %#x)\n", 117 ppi->size, ppi->type, ppi->offset, 118 *(const uint32_t *)ppi_data); 119 if (ppi->size == 0) 120 break; 121 ppi_len -= ppi->size; 122 ppi = (const struct rndis_pktinfo *) 123 ((const char *)ppi + ppi->size); 124 } 125 break; 126 } 127 case RNDIS_INITIALIZE_MSG: 128 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 129 "RNDIS_MSG_INIT (len %u id %#x, ver %u.%u max xfer %u)\n", 130 rndis_msg->init_request.len, 131 rndis_msg->init_request.rid, 132 rndis_msg->init_request.ver_major, 133 rndis_msg->init_request.ver_minor, 134 rndis_msg->init_request.max_xfersz); 135 break; 136 137 case RNDIS_INITIALIZE_CMPLT: 138 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 139 "RNDIS_MSG_INIT_C (len %u, id %#x, status 0x%x, vers %u.%u, " 140 "flags %d, max xfer %u, max pkts %u, aligned %u)\n", 141 rndis_msg->init_complete.len, 142 rndis_msg->init_complete.rid, 143 rndis_msg->init_complete.status, 144 rndis_msg->init_complete.ver_major, 145 rndis_msg->init_complete.ver_minor, 146 rndis_msg->init_complete.devflags, 147 rndis_msg->init_complete.pktmaxsz, 148 rndis_msg->init_complete.pktmaxcnt, 149 rndis_msg->init_complete.align); 150 break; 151 152 case RNDIS_HALT_MSG: 153 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 154 "RNDIS_HALT (len %u id %#x)\n", 155 rndis_msg->halt.len, rndis_msg->halt.rid); 156 break; 157 158 case RNDIS_QUERY_MSG: 159 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 160 "RNDIS_QUERY (len %u, id %#x, oid %#x, info %u:%u)\n", 161 rndis_msg->query_request.len, 162 rndis_msg->query_request.rid, 163 rndis_msg->query_request.oid, 164 rndis_msg->query_request.infobuflen, 165 rndis_msg->query_request.infobufoffset); 166 break; 167 168 case RNDIS_QUERY_CMPLT: 169 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 170 "RNDIS_MSG_QUERY_C (len %u, id %#x, status 0x%x, buf %u:%u)\n", 171 rndis_msg->query_complete.len, 172 rndis_msg->query_complete.rid, 173 rndis_msg->query_complete.status, 174 rndis_msg->query_complete.infobuflen, 175 rndis_msg->query_complete.infobufoffset); 176 break; 177 178 case RNDIS_SET_MSG: 179 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 180 "RNDIS_SET (len %u, id %#x, oid %#x, info %u:%u)\n", 181 rndis_msg->set_request.len, 182 rndis_msg->set_request.rid, 183 rndis_msg->set_request.oid, 184 rndis_msg->set_request.infobuflen, 185 rndis_msg->set_request.infobufoffset); 186 break; 187 188 case RNDIS_SET_CMPLT: 189 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 190 "RNDIS_MSG_SET_C (len %u, id 0x%x, status 0x%x)\n", 191 rndis_msg->set_complete.len, 192 rndis_msg->set_complete.rid, 193 rndis_msg->set_complete.status); 194 break; 195 196 case RNDIS_INDICATE_STATUS_MSG: 197 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 198 "RNDIS_MSG_INDICATE (len %u, status %#x, buf len %u, buf offset %u)\n", 199 rndis_msg->indicate_status.len, 200 rndis_msg->indicate_status.status, 201 rndis_msg->indicate_status.stbuflen, 202 rndis_msg->indicate_status.stbufoffset); 203 break; 204 205 case RNDIS_RESET_MSG: 206 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 207 "RNDIS_RESET (len %u, id %#x)\n", 208 rndis_msg->reset_request.len, 209 rndis_msg->reset_request.rid); 210 break; 211 212 case RNDIS_RESET_CMPLT: 213 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 214 "RNDIS_RESET_C (len %u, status %#x address %#x)\n", 215 rndis_msg->reset_complete.len, 216 rndis_msg->reset_complete.status, 217 rndis_msg->reset_complete.adrreset); 218 break; 219 220 case RNDIS_KEEPALIVE_MSG: 221 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 222 "RNDIS_KEEPALIVE (len %u, id %#x)\n", 223 rndis_msg->keepalive_request.len, 224 rndis_msg->keepalive_request.rid); 225 break; 226 227 case RNDIS_KEEPALIVE_CMPLT: 228 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 229 "RNDIS_KEEPALIVE_C (len %u, id %#x address %#x)\n", 230 rndis_msg->keepalive_complete.len, 231 rndis_msg->keepalive_complete.rid, 232 rndis_msg->keepalive_complete.status); 233 break; 234 235 default: 236 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 237 "RNDIS type %#x len %u\n", 238 rndis_msg->hdr.type, 239 rndis_msg->hdr.len); 240 break; 241 } 242 } 243 #endif 244 245 static int hn_nvs_send_rndis_ctrl(struct vmbus_channel *chan, 246 const void *req, uint32_t reqlen) 247 248 { 249 struct hn_nvs_rndis nvs_rndis = { 250 .type = NVS_TYPE_RNDIS, 251 .rndis_mtype = NVS_RNDIS_MTYPE_CTRL, 252 .chim_idx = NVS_CHIM_IDX_INVALID, 253 .chim_sz = 0 254 }; 255 struct vmbus_gpa sg; 256 rte_iova_t addr; 257 258 addr = rte_malloc_virt2iova(req); 259 if (unlikely(addr == RTE_BAD_IOVA)) { 260 PMD_DRV_LOG(ERR, "RNDIS send request can not get iova"); 261 return -EINVAL; 262 } 263 264 if (unlikely(reqlen > PAGE_SIZE)) { 265 PMD_DRV_LOG(ERR, "RNDIS request %u greater than page size", 266 reqlen); 267 return -EINVAL; 268 } 269 270 sg.page = addr / PAGE_SIZE; 271 sg.ofs = addr & PAGE_MASK; 272 sg.len = reqlen; 273 274 if (sg.ofs + reqlen > PAGE_SIZE) { 275 PMD_DRV_LOG(ERR, "RNDIS request crosses page bounary"); 276 return -EINVAL; 277 } 278 279 hn_rndis_dump(req); 280 281 return hn_nvs_send_sglist(chan, &sg, 1, 282 &nvs_rndis, sizeof(nvs_rndis), 0U, NULL); 283 } 284 285 void hn_rndis_link_status(struct rte_eth_dev *dev, const void *msg) 286 { 287 const struct rndis_status_msg *indicate = msg; 288 289 hn_rndis_dump(msg); 290 291 PMD_DRV_LOG(DEBUG, "link status %#x", indicate->status); 292 293 switch (indicate->status) { 294 case RNDIS_STATUS_NETWORK_CHANGE: 295 case RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG: 296 /* ignore not in DPDK API */ 297 break; 298 299 case RNDIS_STATUS_LINK_SPEED_CHANGE: 300 case RNDIS_STATUS_MEDIA_CONNECT: 301 case RNDIS_STATUS_MEDIA_DISCONNECT: 302 if (dev->data->dev_conf.intr_conf.lsc && 303 hn_dev_link_update(dev, 0) == 0) 304 _rte_eth_dev_callback_process(dev, 305 RTE_ETH_EVENT_INTR_LSC, 306 NULL); 307 break; 308 default: 309 PMD_DRV_LOG(NOTICE, "unknown RNDIS indication: %#x", 310 indicate->status); 311 } 312 } 313 314 /* Callback from hn_process_events when response is visible */ 315 void hn_rndis_receive_response(struct hn_data *hv, 316 const void *data, uint32_t len) 317 { 318 const struct rndis_init_comp *hdr = data; 319 320 hn_rndis_dump(data); 321 322 if (len < sizeof(3 * sizeof(uint32_t))) { 323 PMD_DRV_LOG(ERR, 324 "missing RNDIS header %u", len); 325 return; 326 } 327 328 if (len < hdr->len) { 329 PMD_DRV_LOG(ERR, 330 "truncated RNDIS response %u", len); 331 return; 332 } 333 334 if (len > sizeof(hv->rndis_resp)) { 335 PMD_DRV_LOG(NOTICE, 336 "RNDIS response exceeds buffer"); 337 len = sizeof(hv->rndis_resp); 338 } 339 340 if (hdr->rid == 0) { 341 PMD_DRV_LOG(NOTICE, 342 "RNDIS response id zero!"); 343 } 344 345 memcpy(hv->rndis_resp, data, len); 346 347 /* make sure response copied before update */ 348 rte_smp_wmb(); 349 350 if (rte_atomic32_cmpset(&hv->rndis_pending, hdr->rid, 0) == 0) { 351 PMD_DRV_LOG(ERR, 352 "received id %#x pending id %#x", 353 hdr->rid, (uint32_t)hv->rndis_pending); 354 } 355 } 356 357 /* Do request/response transaction */ 358 static int hn_rndis_exec1(struct hn_data *hv, 359 const void *req, uint32_t reqlen, 360 void *comp, uint32_t comp_len) 361 { 362 const struct rndis_halt_req *hdr = req; 363 uint32_t rid = hdr->rid; 364 struct vmbus_channel *chan = hn_primary_chan(hv); 365 int error; 366 367 if (comp_len > sizeof(hv->rndis_resp)) { 368 PMD_DRV_LOG(ERR, 369 "Expected completion size %u exceeds buffer %zu", 370 comp_len, sizeof(hv->rndis_resp)); 371 return -EIO; 372 } 373 374 if (comp != NULL && 375 rte_atomic32_cmpset(&hv->rndis_pending, 0, rid) == 0) { 376 PMD_DRV_LOG(ERR, 377 "Request already pending"); 378 return -EBUSY; 379 } 380 381 error = hn_nvs_send_rndis_ctrl(chan, req, reqlen); 382 if (error) { 383 PMD_DRV_LOG(ERR, "RNDIS ctrl send failed: %d", error); 384 return error; 385 } 386 387 if (comp) { 388 /* Poll primary channel until response received */ 389 while (hv->rndis_pending == rid) 390 hn_process_events(hv, 0, 1); 391 392 memcpy(comp, hv->rndis_resp, comp_len); 393 } 394 395 return 0; 396 } 397 398 /* Do transaction and validate response */ 399 static int hn_rndis_execute(struct hn_data *hv, uint32_t rid, 400 const void *req, uint32_t reqlen, 401 void *comp, uint32_t comp_len, uint32_t comp_type) 402 { 403 const struct rndis_comp_hdr *hdr = comp; 404 int ret; 405 406 memset(comp, 0, comp_len); 407 408 ret = hn_rndis_exec1(hv, req, reqlen, comp, comp_len); 409 if (ret < 0) 410 return ret; 411 /* 412 * Check this RNDIS complete message. 413 */ 414 if (unlikely(hdr->type != comp_type)) { 415 PMD_DRV_LOG(ERR, 416 "unexpected RNDIS response complete %#x expect %#x", 417 hdr->type, comp_type); 418 419 return -ENXIO; 420 } 421 if (unlikely(hdr->rid != rid)) { 422 PMD_DRV_LOG(ERR, 423 "RNDIS comp rid mismatch %#x, expect %#x", 424 hdr->rid, rid); 425 return -EINVAL; 426 } 427 428 /* All pass! */ 429 return 0; 430 } 431 432 static int 433 hn_rndis_query(struct hn_data *hv, uint32_t oid, 434 const void *idata, uint32_t idlen, 435 void *odata, uint32_t odlen) 436 { 437 struct rndis_query_req *req; 438 struct rndis_query_comp *comp; 439 uint32_t reqlen, comp_len; 440 int error = -EIO; 441 unsigned int ofs; 442 uint32_t rid; 443 444 reqlen = sizeof(*req) + idlen; 445 req = hn_rndis_alloc(hv, reqlen); 446 if (req == NULL) 447 return -ENOMEM; 448 449 comp_len = sizeof(*comp) + odlen; 450 comp = rte_zmalloc("QUERY", comp_len, PAGE_SIZE); 451 if (!comp) { 452 error = -ENOMEM; 453 goto done; 454 } 455 comp->status = RNDIS_STATUS_PENDING; 456 457 rid = hn_rndis_rid(hv); 458 459 req->type = RNDIS_QUERY_MSG; 460 req->len = reqlen; 461 req->rid = rid; 462 req->oid = oid; 463 req->infobufoffset = RNDIS_QUERY_REQ_INFOBUFOFFSET; 464 req->infobuflen = idlen; 465 466 /* Input data immediately follows RNDIS query. */ 467 memcpy(req + 1, idata, idlen); 468 469 error = hn_rndis_execute(hv, rid, req, reqlen, 470 comp, comp_len, RNDIS_QUERY_CMPLT); 471 472 if (error) 473 goto done; 474 475 if (comp->status != RNDIS_STATUS_SUCCESS) { 476 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x failed: status 0x%08x", 477 oid, comp->status); 478 error = -EINVAL; 479 goto done; 480 } 481 482 if (comp->infobuflen == 0 || comp->infobufoffset == 0) { 483 /* No output data! */ 484 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x, no data", oid); 485 error = 0; 486 goto done; 487 } 488 489 /* 490 * Check output data length and offset. 491 */ 492 /* ofs is the offset from the beginning of comp. */ 493 ofs = RNDIS_QUERY_COMP_INFOBUFOFFSET_ABS(comp->infobufoffset); 494 if (ofs < sizeof(*comp) || ofs + comp->infobuflen > comp_len) { 495 PMD_DRV_LOG(ERR, "RNDIS query invalid comp ib off/len, %u/%u", 496 comp->infobufoffset, comp->infobuflen); 497 error = -EINVAL; 498 goto done; 499 } 500 501 /* Save output data. */ 502 if (comp->infobuflen < odlen) 503 odlen = comp->infobuflen; 504 505 /* ofs is the offset from the beginning of comp. */ 506 memcpy(odata, (const char *)comp + ofs, odlen); 507 508 error = 0; 509 done: 510 rte_free(comp); 511 rte_free(req); 512 return error; 513 } 514 515 static int 516 hn_rndis_halt(struct hn_data *hv) 517 { 518 struct rndis_halt_req *halt; 519 520 halt = hn_rndis_alloc(hv, sizeof(*halt)); 521 if (halt == NULL) 522 return -ENOMEM; 523 524 halt->type = RNDIS_HALT_MSG; 525 halt->len = sizeof(*halt); 526 halt->rid = hn_rndis_rid(hv); 527 528 /* No RNDIS completion; rely on NVS message send completion */ 529 hn_rndis_exec1(hv, halt, sizeof(*halt), NULL, 0); 530 531 rte_free(halt); 532 533 PMD_INIT_LOG(DEBUG, "RNDIS halt done"); 534 return 0; 535 } 536 537 static int 538 hn_rndis_query_hwcaps(struct hn_data *hv, struct ndis_offload *caps) 539 { 540 struct ndis_offload in; 541 uint32_t caps_len, size; 542 int error; 543 544 memset(caps, 0, sizeof(*caps)); 545 memset(&in, 0, sizeof(in)); 546 in.ndis_hdr.ndis_type = NDIS_OBJTYPE_OFFLOAD; 547 548 if (hv->ndis_ver >= NDIS_VERSION_6_30) { 549 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_3; 550 size = NDIS_OFFLOAD_SIZE; 551 } else if (hv->ndis_ver >= NDIS_VERSION_6_1) { 552 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_2; 553 size = NDIS_OFFLOAD_SIZE_6_1; 554 } else { 555 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_1; 556 size = NDIS_OFFLOAD_SIZE_6_0; 557 } 558 in.ndis_hdr.ndis_size = size; 559 560 caps_len = NDIS_OFFLOAD_SIZE; 561 error = hn_rndis_query(hv, OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES, 562 &in, size, caps, caps_len); 563 if (error) 564 return error; 565 566 /* Preliminary verification. */ 567 if (caps->ndis_hdr.ndis_type != NDIS_OBJTYPE_OFFLOAD) { 568 PMD_DRV_LOG(NOTICE, "invalid NDIS objtype 0x%02x", 569 caps->ndis_hdr.ndis_type); 570 return -EINVAL; 571 } 572 if (caps->ndis_hdr.ndis_rev < NDIS_OFFLOAD_REV_1) { 573 PMD_DRV_LOG(NOTICE, "invalid NDIS objrev 0x%02x", 574 caps->ndis_hdr.ndis_rev); 575 return -EINVAL; 576 } 577 if (caps->ndis_hdr.ndis_size > caps_len) { 578 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u, data size %u", 579 caps->ndis_hdr.ndis_size, caps_len); 580 return -EINVAL; 581 } else if (caps->ndis_hdr.ndis_size < NDIS_OFFLOAD_SIZE_6_0) { 582 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u", 583 caps->ndis_hdr.ndis_size); 584 return -EINVAL; 585 } 586 587 return 0; 588 } 589 590 int 591 hn_rndis_query_rsscaps(struct hn_data *hv, 592 unsigned int *rxr_cnt0) 593 { 594 struct ndis_rss_caps in, caps; 595 unsigned int indsz, rxr_cnt; 596 uint32_t caps_len; 597 int error; 598 599 *rxr_cnt0 = 0; 600 601 if (hv->ndis_ver < NDIS_VERSION_6_20) { 602 PMD_DRV_LOG(DEBUG, "RSS not supported on this host"); 603 return -EOPNOTSUPP; 604 } 605 606 memset(&in, 0, sizeof(in)); 607 in.ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_CAPS; 608 in.ndis_hdr.ndis_rev = NDIS_RSS_CAPS_REV_2; 609 in.ndis_hdr.ndis_size = NDIS_RSS_CAPS_SIZE; 610 611 caps_len = NDIS_RSS_CAPS_SIZE; 612 error = hn_rndis_query(hv, OID_GEN_RECEIVE_SCALE_CAPABILITIES, 613 &in, NDIS_RSS_CAPS_SIZE, 614 &caps, caps_len); 615 if (error) 616 return error; 617 618 PMD_INIT_LOG(DEBUG, "RX rings %u indirect %u caps %#x", 619 caps.ndis_nrxr, caps.ndis_nind, caps.ndis_caps); 620 /* 621 * Preliminary verification. 622 */ 623 if (caps.ndis_hdr.ndis_type != NDIS_OBJTYPE_RSS_CAPS) { 624 PMD_DRV_LOG(ERR, "invalid NDIS objtype 0x%02x", 625 caps.ndis_hdr.ndis_type); 626 return -EINVAL; 627 } 628 if (caps.ndis_hdr.ndis_rev < NDIS_RSS_CAPS_REV_1) { 629 PMD_DRV_LOG(ERR, "invalid NDIS objrev 0x%02x", 630 caps.ndis_hdr.ndis_rev); 631 return -EINVAL; 632 } 633 if (caps.ndis_hdr.ndis_size > caps_len) { 634 PMD_DRV_LOG(ERR, 635 "invalid NDIS objsize %u, data size %u", 636 caps.ndis_hdr.ndis_size, caps_len); 637 return -EINVAL; 638 } else if (caps.ndis_hdr.ndis_size < NDIS_RSS_CAPS_SIZE_6_0) { 639 PMD_DRV_LOG(ERR, "invalid NDIS objsize %u", 640 caps.ndis_hdr.ndis_size); 641 return -EINVAL; 642 } 643 644 /* 645 * Save information for later RSS configuration. 646 */ 647 if (caps.ndis_nrxr == 0) { 648 PMD_DRV_LOG(ERR, "0 RX rings!?"); 649 return -EINVAL; 650 } 651 rxr_cnt = caps.ndis_nrxr; 652 653 if (caps.ndis_hdr.ndis_size == NDIS_RSS_CAPS_SIZE && 654 caps.ndis_hdr.ndis_rev >= NDIS_RSS_CAPS_REV_2) { 655 if (caps.ndis_nind > NDIS_HASH_INDCNT) { 656 PMD_DRV_LOG(ERR, 657 "too many RSS indirect table entries %u", 658 caps.ndis_nind); 659 return -EOPNOTSUPP; 660 } 661 if (!rte_is_power_of_2(caps.ndis_nind)) { 662 PMD_DRV_LOG(ERR, 663 "RSS indirect table size is not power-of-2 %u", 664 caps.ndis_nind); 665 } 666 667 indsz = caps.ndis_nind; 668 } else { 669 indsz = NDIS_HASH_INDCNT; 670 } 671 672 if (indsz < rxr_cnt) { 673 PMD_DRV_LOG(NOTICE, 674 "# of RX rings (%d) > RSS indirect table size %d", 675 rxr_cnt, indsz); 676 rxr_cnt = indsz; 677 } 678 679 hv->rss_offloads = 0; 680 if (caps.ndis_caps & NDIS_RSS_CAP_IPV4) 681 hv->rss_offloads |= ETH_RSS_IPV4 682 | ETH_RSS_NONFRAG_IPV4_TCP 683 | ETH_RSS_NONFRAG_IPV4_UDP; 684 if (caps.ndis_caps & NDIS_RSS_CAP_IPV6) 685 hv->rss_offloads |= ETH_RSS_IPV6 686 | ETH_RSS_NONFRAG_IPV6_TCP; 687 if (caps.ndis_caps & NDIS_RSS_CAP_IPV6_EX) 688 hv->rss_offloads |= ETH_RSS_IPV6_EX 689 | ETH_RSS_IPV6_TCP_EX; 690 691 /* Commit! */ 692 *rxr_cnt0 = rxr_cnt; 693 694 return 0; 695 } 696 697 static int 698 hn_rndis_set(struct hn_data *hv, uint32_t oid, const void *data, uint32_t dlen) 699 { 700 struct rndis_set_req *req; 701 struct rndis_set_comp comp; 702 uint32_t reqlen, comp_len; 703 uint32_t rid; 704 int error; 705 706 reqlen = sizeof(*req) + dlen; 707 req = rte_zmalloc("RNDIS_SET", reqlen, PAGE_SIZE); 708 if (!req) 709 return -ENOMEM; 710 711 rid = hn_rndis_rid(hv); 712 req->type = RNDIS_SET_MSG; 713 req->len = reqlen; 714 req->rid = rid; 715 req->oid = oid; 716 req->infobuflen = dlen; 717 req->infobufoffset = RNDIS_SET_REQ_INFOBUFOFFSET; 718 719 /* Data immediately follows RNDIS set. */ 720 memcpy(req + 1, data, dlen); 721 722 comp_len = sizeof(comp); 723 error = hn_rndis_execute(hv, rid, req, reqlen, 724 &comp, comp_len, 725 RNDIS_SET_CMPLT); 726 if (error) { 727 PMD_DRV_LOG(ERR, "exec RNDIS set %#" PRIx32 " failed", 728 oid); 729 error = EIO; 730 goto done; 731 } 732 733 if (comp.status != RNDIS_STATUS_SUCCESS) { 734 PMD_DRV_LOG(ERR, 735 "RNDIS set %#" PRIx32 " failed: status %#" PRIx32, 736 oid, comp.status); 737 error = EIO; 738 goto done; 739 } 740 741 done: 742 rte_free(req); 743 return error; 744 } 745 746 int hn_rndis_conf_offload(struct hn_data *hv, 747 uint64_t tx_offloads, uint64_t rx_offloads) 748 { 749 struct ndis_offload_params params; 750 struct ndis_offload hwcaps; 751 int error; 752 753 error = hn_rndis_query_hwcaps(hv, &hwcaps); 754 if (error) { 755 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 756 return error; 757 } 758 759 /* NOTE: 0 means "no change" */ 760 memset(¶ms, 0, sizeof(params)); 761 762 params.ndis_hdr.ndis_type = NDIS_OBJTYPE_DEFAULT; 763 if (hv->ndis_ver < NDIS_VERSION_6_30) { 764 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_2; 765 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE_6_1; 766 } else { 767 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_3; 768 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE; 769 } 770 771 if (tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM) { 772 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_TCP4) 773 params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_TX; 774 else 775 goto unsupported; 776 777 if (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_TCP6) 778 params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_TX; 779 else 780 goto unsupported; 781 } 782 783 if (rx_offloads & DEV_RX_OFFLOAD_TCP_CKSUM) { 784 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) 785 == NDIS_RXCSUM_CAP_TCP4) 786 params.ndis_tcp4csum |= NDIS_OFFLOAD_PARAM_RX; 787 else 788 goto unsupported; 789 790 if ((hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6) 791 == NDIS_RXCSUM_CAP_TCP6) 792 params.ndis_tcp6csum |= NDIS_OFFLOAD_PARAM_RX; 793 else 794 goto unsupported; 795 } 796 797 if (tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) { 798 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) 799 params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_TX; 800 else 801 goto unsupported; 802 803 if ((hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6) 804 == NDIS_TXCSUM_CAP_UDP6) 805 params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_TX; 806 else 807 goto unsupported; 808 } 809 810 if (rx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) { 811 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) 812 params.ndis_udp4csum |= NDIS_OFFLOAD_PARAM_RX; 813 else 814 goto unsupported; 815 816 if (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6) 817 params.ndis_udp6csum |= NDIS_OFFLOAD_PARAM_RX; 818 else 819 goto unsupported; 820 } 821 822 if (tx_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) { 823 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_IP4) 824 == NDIS_TXCSUM_CAP_IP4) 825 params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_TX; 826 else 827 goto unsupported; 828 } 829 if (rx_offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) { 830 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 831 params.ndis_ip4csum |= NDIS_OFFLOAD_PARAM_RX; 832 else 833 goto unsupported; 834 } 835 836 if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO) { 837 if (hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) 838 params.ndis_lsov2_ip4 = NDIS_OFFLOAD_LSOV2_ON; 839 else 840 goto unsupported; 841 842 if ((hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6) 843 == HN_NDIS_LSOV2_CAP_IP6) 844 params.ndis_lsov2_ip6 = NDIS_OFFLOAD_LSOV2_ON; 845 else 846 goto unsupported; 847 } 848 849 error = hn_rndis_set(hv, OID_TCP_OFFLOAD_PARAMETERS, ¶ms, 850 params.ndis_hdr.ndis_size); 851 if (error) { 852 PMD_DRV_LOG(ERR, "offload config failed"); 853 return error; 854 } 855 856 return 0; 857 unsupported: 858 PMD_DRV_LOG(NOTICE, 859 "offload tx:%" PRIx64 " rx:%" PRIx64 " not supported by this version", 860 tx_offloads, rx_offloads); 861 return -EINVAL; 862 } 863 864 int hn_rndis_get_offload(struct hn_data *hv, 865 struct rte_eth_dev_info *dev_info) 866 { 867 struct ndis_offload hwcaps; 868 int error; 869 870 memset(&hwcaps, 0, sizeof(hwcaps)); 871 872 error = hn_rndis_query_hwcaps(hv, &hwcaps); 873 if (error) { 874 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 875 return error; 876 } 877 878 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS | 879 DEV_TX_OFFLOAD_VLAN_INSERT; 880 881 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_IP4) 882 == HN_NDIS_TXCSUM_CAP_IP4) 883 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM; 884 885 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_TCP4) 886 == HN_NDIS_TXCSUM_CAP_TCP4 && 887 (hwcaps.ndis_csum.ndis_ip6_txcsum & HN_NDIS_TXCSUM_CAP_TCP6) 888 == HN_NDIS_TXCSUM_CAP_TCP6) 889 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_CKSUM; 890 891 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) && 892 (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6)) 893 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_UDP_CKSUM; 894 895 if ((hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) && 896 (hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6) 897 == HN_NDIS_LSOV2_CAP_IP6) 898 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; 899 900 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP; 901 902 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 903 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM; 904 905 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) && 906 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)) 907 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_CKSUM; 908 909 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) && 910 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)) 911 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_UDP_CKSUM; 912 913 return 0; 914 } 915 916 uint32_t 917 hn_rndis_get_ptypes(struct hn_data *hv) 918 { 919 struct ndis_offload hwcaps; 920 uint32_t ptypes; 921 int error; 922 923 memset(&hwcaps, 0, sizeof(hwcaps)); 924 925 error = hn_rndis_query_hwcaps(hv, &hwcaps); 926 if (error) { 927 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 928 return RTE_PTYPE_L2_ETHER; 929 } 930 931 ptypes = RTE_PTYPE_L2_ETHER; 932 933 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 934 ptypes |= RTE_PTYPE_L3_IPV4; 935 936 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) || 937 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)) 938 ptypes |= RTE_PTYPE_L4_TCP; 939 940 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) || 941 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)) 942 ptypes |= RTE_PTYPE_L4_UDP; 943 944 return ptypes; 945 } 946 947 int 948 hn_rndis_set_rxfilter(struct hn_data *hv, uint32_t filter) 949 { 950 int error; 951 952 error = hn_rndis_set(hv, OID_GEN_CURRENT_PACKET_FILTER, 953 &filter, sizeof(filter)); 954 if (error) { 955 PMD_DRV_LOG(ERR, "set RX filter %#" PRIx32 " failed: %d", 956 filter, error); 957 } else { 958 PMD_DRV_LOG(DEBUG, "set RX filter %#" PRIx32 " done", filter); 959 } 960 961 return error; 962 } 963 964 int hn_rndis_conf_rss(struct hn_data *hv, uint32_t flags) 965 { 966 struct ndis_rssprm_toeplitz rssp; 967 struct ndis_rss_params *prm = &rssp.rss_params; 968 unsigned int i; 969 int error; 970 971 memset(&rssp, 0, sizeof(rssp)); 972 973 prm->ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_PARAMS; 974 prm->ndis_hdr.ndis_rev = NDIS_RSS_PARAMS_REV_2; 975 prm->ndis_hdr.ndis_size = sizeof(*prm); 976 prm->ndis_flags = flags; 977 prm->ndis_hash = hv->rss_hash; 978 prm->ndis_indsize = sizeof(rssp.rss_ind[0]) * NDIS_HASH_INDCNT; 979 prm->ndis_indoffset = offsetof(struct ndis_rssprm_toeplitz, rss_ind[0]); 980 prm->ndis_keysize = NDIS_HASH_KEYSIZE_TOEPLITZ; 981 prm->ndis_keyoffset = offsetof(struct ndis_rssprm_toeplitz, rss_key[0]); 982 983 for (i = 0; i < NDIS_HASH_INDCNT; i++) 984 rssp.rss_ind[i] = hv->rss_ind[i]; 985 986 /* Set hask key values */ 987 memcpy(&rssp.rss_key, hv->rss_key, NDIS_HASH_KEYSIZE_TOEPLITZ); 988 989 error = hn_rndis_set(hv, OID_GEN_RECEIVE_SCALE_PARAMETERS, 990 &rssp, sizeof(rssp)); 991 if (error != 0) { 992 PMD_DRV_LOG(ERR, 993 "RSS config num queues=%u failed: %d", 994 hv->num_queues, error); 995 } 996 return error; 997 } 998 999 static int hn_rndis_init(struct hn_data *hv) 1000 { 1001 struct rndis_init_req *req; 1002 struct rndis_init_comp comp; 1003 uint32_t comp_len, rid; 1004 int error; 1005 1006 req = hn_rndis_alloc(hv, sizeof(*req)); 1007 if (!req) { 1008 PMD_DRV_LOG(ERR, "no memory for RNDIS init"); 1009 return -ENXIO; 1010 } 1011 1012 rid = hn_rndis_rid(hv); 1013 req->type = RNDIS_INITIALIZE_MSG; 1014 req->len = sizeof(*req); 1015 req->rid = rid; 1016 req->ver_major = RNDIS_VERSION_MAJOR; 1017 req->ver_minor = RNDIS_VERSION_MINOR; 1018 req->max_xfersz = HN_RNDIS_XFER_SIZE; 1019 1020 comp_len = RNDIS_INIT_COMP_SIZE_MIN; 1021 error = hn_rndis_execute(hv, rid, req, sizeof(*req), 1022 &comp, comp_len, 1023 RNDIS_INITIALIZE_CMPLT); 1024 if (error) 1025 goto done; 1026 1027 if (comp.status != RNDIS_STATUS_SUCCESS) { 1028 PMD_DRV_LOG(ERR, "RNDIS init failed: status 0x%08x", 1029 comp.status); 1030 error = -EIO; 1031 goto done; 1032 } 1033 1034 hv->rndis_agg_size = comp.pktmaxsz; 1035 hv->rndis_agg_pkts = comp.pktmaxcnt; 1036 hv->rndis_agg_align = 1U << comp.align; 1037 1038 if (hv->rndis_agg_align < sizeof(uint32_t)) { 1039 /* 1040 * The RNDIS packet message encap assumes that the RNDIS 1041 * packet message is at least 4 bytes aligned. Fix up the 1042 * alignment here, if the remote side sets the alignment 1043 * too low. 1044 */ 1045 PMD_DRV_LOG(NOTICE, 1046 "fixup RNDIS aggpkt align: %u -> %zu", 1047 hv->rndis_agg_align, sizeof(uint32_t)); 1048 hv->rndis_agg_align = sizeof(uint32_t); 1049 } 1050 1051 PMD_INIT_LOG(INFO, 1052 "RNDIS ver %u.%u, aggpkt size %u, aggpkt cnt %u, aggpkt align %u", 1053 comp.ver_major, comp.ver_minor, 1054 hv->rndis_agg_size, hv->rndis_agg_pkts, 1055 hv->rndis_agg_align); 1056 error = 0; 1057 done: 1058 rte_free(req); 1059 return error; 1060 } 1061 1062 int 1063 hn_rndis_get_eaddr(struct hn_data *hv, uint8_t *eaddr) 1064 { 1065 uint32_t eaddr_len; 1066 int error; 1067 1068 eaddr_len = RTE_ETHER_ADDR_LEN; 1069 error = hn_rndis_query(hv, OID_802_3_PERMANENT_ADDRESS, NULL, 0, 1070 eaddr, eaddr_len); 1071 if (error) 1072 return error; 1073 1074 PMD_DRV_LOG(INFO, "MAC address %02x:%02x:%02x:%02x:%02x:%02x", 1075 eaddr[0], eaddr[1], eaddr[2], 1076 eaddr[3], eaddr[4], eaddr[5]); 1077 return 0; 1078 } 1079 1080 int 1081 hn_rndis_get_linkstatus(struct hn_data *hv) 1082 { 1083 return hn_rndis_query(hv, OID_GEN_MEDIA_CONNECT_STATUS, NULL, 0, 1084 &hv->link_status, sizeof(uint32_t)); 1085 } 1086 1087 int 1088 hn_rndis_get_linkspeed(struct hn_data *hv) 1089 { 1090 return hn_rndis_query(hv, OID_GEN_LINK_SPEED, NULL, 0, 1091 &hv->link_speed, sizeof(uint32_t)); 1092 } 1093 1094 int 1095 hn_rndis_attach(struct hn_data *hv) 1096 { 1097 /* Initialize RNDIS. */ 1098 return hn_rndis_init(hv); 1099 } 1100 1101 void 1102 hn_rndis_detach(struct hn_data *hv) 1103 { 1104 /* Halt the RNDIS. */ 1105 hn_rndis_halt(hv); 1106 } 1107