1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2009-2018 Microsoft Corp. 3 * Copyright (c) 2010-2012 Citrix Inc. 4 * Copyright (c) 2012 NetApp Inc. 5 * All rights reserved. 6 */ 7 8 #include <stdint.h> 9 #include <string.h> 10 #include <stdio.h> 11 #include <errno.h> 12 #include <unistd.h> 13 #include <time.h> 14 15 #include <ethdev_driver.h> 16 #include <rte_ethdev.h> 17 #include <rte_string_fns.h> 18 #include <rte_memzone.h> 19 #include <rte_malloc.h> 20 #include <rte_atomic.h> 21 #include <rte_alarm.h> 22 #include <rte_branch_prediction.h> 23 #include <rte_ether.h> 24 #include <rte_common.h> 25 #include <rte_errno.h> 26 #include <rte_cycles.h> 27 #include <rte_memory.h> 28 #include <rte_eal.h> 29 #include <rte_dev.h> 30 #include <rte_bus_vmbus.h> 31 32 #include "hn_logs.h" 33 #include "hn_var.h" 34 #include "hn_nvs.h" 35 #include "hn_rndis.h" 36 #include "ndis.h" 37 38 #define RNDIS_TIMEOUT_SEC 5 39 #define RNDIS_DELAY_MS 10 40 41 #define HN_RNDIS_XFER_SIZE 0x4000 42 43 #define HN_NDIS_TXCSUM_CAP_IP4 \ 44 (NDIS_TXCSUM_CAP_IP4 | NDIS_TXCSUM_CAP_IP4OPT) 45 #define HN_NDIS_TXCSUM_CAP_TCP4 \ 46 (NDIS_TXCSUM_CAP_TCP4 | NDIS_TXCSUM_CAP_TCP4OPT) 47 #define HN_NDIS_TXCSUM_CAP_TCP6 \ 48 (NDIS_TXCSUM_CAP_TCP6 | NDIS_TXCSUM_CAP_TCP6OPT | \ 49 NDIS_TXCSUM_CAP_IP6EXT) 50 #define HN_NDIS_TXCSUM_CAP_UDP6 \ 51 (NDIS_TXCSUM_CAP_UDP6 | NDIS_TXCSUM_CAP_IP6EXT) 52 #define HN_NDIS_LSOV2_CAP_IP6 \ 53 (NDIS_LSOV2_CAP_IP6EXT | NDIS_LSOV2_CAP_TCP6OPT) 54 55 /* Get unique request id */ 56 static inline uint32_t 57 hn_rndis_rid(struct hn_data *hv) 58 { 59 uint32_t rid; 60 61 do { 62 rid = rte_atomic32_add_return(&hv->rndis_req_id, 1); 63 } while (rid == 0); 64 65 return rid; 66 } 67 68 static void *hn_rndis_alloc(size_t size) 69 { 70 return rte_zmalloc("RNDIS", size, rte_mem_page_size()); 71 } 72 73 #ifdef RTE_LIBRTE_NETVSC_DEBUG_DUMP 74 void hn_rndis_dump(const void *buf) 75 { 76 const union { 77 struct rndis_msghdr hdr; 78 struct rndis_packet_msg pkt; 79 struct rndis_init_req init_request; 80 struct rndis_init_comp init_complete; 81 struct rndis_halt_req halt; 82 struct rndis_query_req query_request; 83 struct rndis_query_comp query_complete; 84 struct rndis_set_req set_request; 85 struct rndis_set_comp set_complete; 86 struct rndis_reset_req reset_request; 87 struct rndis_reset_comp reset_complete; 88 struct rndis_keepalive_req keepalive_request; 89 struct rndis_keepalive_comp keepalive_complete; 90 struct rndis_status_msg indicate_status; 91 } *rndis_msg = buf; 92 93 switch (rndis_msg->hdr.type) { 94 case RNDIS_PACKET_MSG: { 95 const struct rndis_pktinfo *ppi; 96 unsigned int ppi_len; 97 98 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 99 "RNDIS_MSG_PACKET (len %u, data %u:%u, # oob %u %u:%u, pkt %u:%u)\n", 100 rndis_msg->pkt.len, 101 rndis_msg->pkt.dataoffset, 102 rndis_msg->pkt.datalen, 103 rndis_msg->pkt.oobdataelements, 104 rndis_msg->pkt.oobdataoffset, 105 rndis_msg->pkt.oobdatalen, 106 rndis_msg->pkt.pktinfooffset, 107 rndis_msg->pkt.pktinfolen); 108 109 ppi = (const struct rndis_pktinfo *) 110 ((const char *)buf 111 + RNDIS_PACKET_MSG_OFFSET_ABS(rndis_msg->pkt.pktinfooffset)); 112 113 ppi_len = rndis_msg->pkt.pktinfolen; 114 while (ppi_len > 0) { 115 const void *ppi_data; 116 117 ppi_data = ppi->data; 118 119 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 120 " PPI (size %u, type %u, offs %u data %#x)\n", 121 ppi->size, ppi->type, ppi->offset, 122 *(const uint32_t *)ppi_data); 123 if (ppi->size == 0) 124 break; 125 ppi_len -= ppi->size; 126 ppi = (const struct rndis_pktinfo *) 127 ((const char *)ppi + ppi->size); 128 } 129 break; 130 } 131 case RNDIS_INITIALIZE_MSG: 132 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 133 "RNDIS_MSG_INIT (len %u id %#x, ver %u.%u max xfer %u)\n", 134 rndis_msg->init_request.len, 135 rndis_msg->init_request.rid, 136 rndis_msg->init_request.ver_major, 137 rndis_msg->init_request.ver_minor, 138 rndis_msg->init_request.max_xfersz); 139 break; 140 141 case RNDIS_INITIALIZE_CMPLT: 142 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 143 "RNDIS_MSG_INIT_C (len %u, id %#x, status 0x%x, vers %u.%u, " 144 "flags %d, max xfer %u, max pkts %u, aligned %u)\n", 145 rndis_msg->init_complete.len, 146 rndis_msg->init_complete.rid, 147 rndis_msg->init_complete.status, 148 rndis_msg->init_complete.ver_major, 149 rndis_msg->init_complete.ver_minor, 150 rndis_msg->init_complete.devflags, 151 rndis_msg->init_complete.pktmaxsz, 152 rndis_msg->init_complete.pktmaxcnt, 153 rndis_msg->init_complete.align); 154 break; 155 156 case RNDIS_HALT_MSG: 157 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 158 "RNDIS_HALT (len %u id %#x)\n", 159 rndis_msg->halt.len, rndis_msg->halt.rid); 160 break; 161 162 case RNDIS_QUERY_MSG: 163 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 164 "RNDIS_QUERY (len %u, id %#x, oid %#x, info %u:%u)\n", 165 rndis_msg->query_request.len, 166 rndis_msg->query_request.rid, 167 rndis_msg->query_request.oid, 168 rndis_msg->query_request.infobuflen, 169 rndis_msg->query_request.infobufoffset); 170 break; 171 172 case RNDIS_QUERY_CMPLT: 173 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 174 "RNDIS_MSG_QUERY_C (len %u, id %#x, status 0x%x, buf %u:%u)\n", 175 rndis_msg->query_complete.len, 176 rndis_msg->query_complete.rid, 177 rndis_msg->query_complete.status, 178 rndis_msg->query_complete.infobuflen, 179 rndis_msg->query_complete.infobufoffset); 180 break; 181 182 case RNDIS_SET_MSG: 183 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 184 "RNDIS_SET (len %u, id %#x, oid %#x, info %u:%u)\n", 185 rndis_msg->set_request.len, 186 rndis_msg->set_request.rid, 187 rndis_msg->set_request.oid, 188 rndis_msg->set_request.infobuflen, 189 rndis_msg->set_request.infobufoffset); 190 break; 191 192 case RNDIS_SET_CMPLT: 193 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 194 "RNDIS_MSG_SET_C (len %u, id 0x%x, status 0x%x)\n", 195 rndis_msg->set_complete.len, 196 rndis_msg->set_complete.rid, 197 rndis_msg->set_complete.status); 198 break; 199 200 case RNDIS_INDICATE_STATUS_MSG: 201 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 202 "RNDIS_MSG_INDICATE (len %u, status %#x, buf len %u, buf offset %u)\n", 203 rndis_msg->indicate_status.len, 204 rndis_msg->indicate_status.status, 205 rndis_msg->indicate_status.stbuflen, 206 rndis_msg->indicate_status.stbufoffset); 207 break; 208 209 case RNDIS_RESET_MSG: 210 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 211 "RNDIS_RESET (len %u, id %#x)\n", 212 rndis_msg->reset_request.len, 213 rndis_msg->reset_request.rid); 214 break; 215 216 case RNDIS_RESET_CMPLT: 217 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 218 "RNDIS_RESET_C (len %u, status %#x address %#x)\n", 219 rndis_msg->reset_complete.len, 220 rndis_msg->reset_complete.status, 221 rndis_msg->reset_complete.adrreset); 222 break; 223 224 case RNDIS_KEEPALIVE_MSG: 225 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 226 "RNDIS_KEEPALIVE (len %u, id %#x)\n", 227 rndis_msg->keepalive_request.len, 228 rndis_msg->keepalive_request.rid); 229 break; 230 231 case RNDIS_KEEPALIVE_CMPLT: 232 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 233 "RNDIS_KEEPALIVE_C (len %u, id %#x address %#x)\n", 234 rndis_msg->keepalive_complete.len, 235 rndis_msg->keepalive_complete.rid, 236 rndis_msg->keepalive_complete.status); 237 break; 238 239 default: 240 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 241 "RNDIS type %#x len %u\n", 242 rndis_msg->hdr.type, 243 rndis_msg->hdr.len); 244 break; 245 } 246 } 247 #endif 248 249 static int hn_nvs_send_rndis_ctrl(struct vmbus_channel *chan, 250 const void *req, uint32_t reqlen) 251 252 { 253 struct hn_nvs_rndis nvs_rndis = { 254 .type = NVS_TYPE_RNDIS, 255 .rndis_mtype = NVS_RNDIS_MTYPE_CTRL, 256 .chim_idx = NVS_CHIM_IDX_INVALID, 257 .chim_sz = 0 258 }; 259 struct vmbus_gpa sg; 260 rte_iova_t addr; 261 262 addr = rte_malloc_virt2iova(req); 263 if (unlikely(addr == RTE_BAD_IOVA)) { 264 PMD_DRV_LOG(ERR, "RNDIS send request can not get iova"); 265 return -EINVAL; 266 } 267 268 if (unlikely(reqlen > rte_mem_page_size())) { 269 PMD_DRV_LOG(ERR, "RNDIS request %u greater than page size", 270 reqlen); 271 return -EINVAL; 272 } 273 274 sg.page = addr / rte_mem_page_size(); 275 sg.ofs = addr & PAGE_MASK; 276 sg.len = reqlen; 277 278 if (sg.ofs + reqlen > rte_mem_page_size()) { 279 PMD_DRV_LOG(ERR, "RNDIS request crosses page boundary"); 280 return -EINVAL; 281 } 282 283 hn_rndis_dump(req); 284 285 return hn_nvs_send_sglist(chan, &sg, 1, 286 &nvs_rndis, sizeof(nvs_rndis), 0U, NULL); 287 } 288 289 /* 290 * Alarm callback to process link changed notifications. 291 * Can not directly since link_status is discovered while reading ring 292 */ 293 static void hn_rndis_link_alarm(void *arg) 294 { 295 rte_eth_dev_callback_process(arg, RTE_ETH_EVENT_INTR_LSC, NULL); 296 } 297 298 void hn_rndis_link_status(struct rte_eth_dev *dev, const void *msg) 299 { 300 const struct rndis_status_msg *indicate = msg; 301 302 hn_rndis_dump(msg); 303 304 PMD_DRV_LOG(DEBUG, "link status %#x", indicate->status); 305 306 switch (indicate->status) { 307 case RNDIS_STATUS_NETWORK_CHANGE: 308 case RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG: 309 /* ignore not in DPDK API */ 310 break; 311 312 case RNDIS_STATUS_LINK_SPEED_CHANGE: 313 case RNDIS_STATUS_MEDIA_CONNECT: 314 case RNDIS_STATUS_MEDIA_DISCONNECT: 315 if (dev->data->dev_conf.intr_conf.lsc) 316 rte_eal_alarm_set(10, hn_rndis_link_alarm, dev); 317 break; 318 default: 319 PMD_DRV_LOG(NOTICE, "unknown RNDIS indication: %#x", 320 indicate->status); 321 } 322 } 323 324 /* Callback from hn_process_events when response is visible */ 325 void hn_rndis_receive_response(struct hn_data *hv, 326 const void *data, uint32_t len) 327 { 328 const struct rndis_init_comp *hdr = data; 329 330 hn_rndis_dump(data); 331 332 if (len < sizeof(3 * sizeof(uint32_t))) { 333 PMD_DRV_LOG(ERR, 334 "missing RNDIS header %u", len); 335 return; 336 } 337 338 if (len < hdr->len) { 339 PMD_DRV_LOG(ERR, 340 "truncated RNDIS response %u", len); 341 return; 342 } 343 344 if (len > sizeof(hv->rndis_resp)) { 345 PMD_DRV_LOG(NOTICE, 346 "RNDIS response exceeds buffer"); 347 len = sizeof(hv->rndis_resp); 348 } 349 350 if (hdr->rid == 0) { 351 PMD_DRV_LOG(NOTICE, 352 "RNDIS response id zero!"); 353 } 354 355 memcpy(hv->rndis_resp, data, len); 356 357 /* make sure response copied before update */ 358 rte_smp_wmb(); 359 360 if (rte_atomic32_cmpset(&hv->rndis_pending, hdr->rid, 0) == 0) { 361 PMD_DRV_LOG(NOTICE, 362 "received id %#x pending id %#x", 363 hdr->rid, (uint32_t)hv->rndis_pending); 364 } 365 } 366 367 /* Do request/response transaction */ 368 static int hn_rndis_exec1(struct hn_data *hv, 369 const void *req, uint32_t reqlen, 370 void *comp, uint32_t comp_len) 371 { 372 const struct rndis_halt_req *hdr = req; 373 uint32_t rid = hdr->rid; 374 struct vmbus_channel *chan = hn_primary_chan(hv); 375 int error; 376 377 if (comp_len > sizeof(hv->rndis_resp)) { 378 PMD_DRV_LOG(ERR, 379 "Expected completion size %u exceeds buffer %zu", 380 comp_len, sizeof(hv->rndis_resp)); 381 return -EIO; 382 } 383 384 if (rid == 0) { 385 PMD_DRV_LOG(ERR, "Invalid request id"); 386 return -EINVAL; 387 } 388 389 if (comp != NULL && 390 rte_atomic32_cmpset(&hv->rndis_pending, 0, rid) == 0) { 391 PMD_DRV_LOG(ERR, 392 "Request already pending"); 393 return -EBUSY; 394 } 395 396 error = hn_nvs_send_rndis_ctrl(chan, req, reqlen); 397 if (error) { 398 PMD_DRV_LOG(ERR, "RNDIS ctrl send failed: %d", error); 399 return error; 400 } 401 402 if (comp) { 403 time_t start = time(NULL); 404 405 /* Poll primary channel until response received */ 406 while (hv->rndis_pending == rid) { 407 if (hv->closed) 408 return -ENETDOWN; 409 410 if (time(NULL) - start > RNDIS_TIMEOUT_SEC) { 411 PMD_DRV_LOG(ERR, 412 "RNDIS response timed out"); 413 414 rte_atomic32_cmpset(&hv->rndis_pending, rid, 0); 415 return -ETIMEDOUT; 416 } 417 418 if (rte_vmbus_chan_rx_empty(hv->primary->chan)) 419 rte_delay_ms(RNDIS_DELAY_MS); 420 421 hn_process_events(hv, 0, 1); 422 } 423 424 memcpy(comp, hv->rndis_resp, comp_len); 425 } 426 427 return 0; 428 } 429 430 /* Do transaction and validate response */ 431 static int hn_rndis_execute(struct hn_data *hv, uint32_t rid, 432 const void *req, uint32_t reqlen, 433 void *comp, uint32_t comp_len, uint32_t comp_type) 434 { 435 const struct rndis_comp_hdr *hdr = comp; 436 int ret; 437 438 memset(comp, 0, comp_len); 439 440 ret = hn_rndis_exec1(hv, req, reqlen, comp, comp_len); 441 if (ret < 0) 442 return ret; 443 /* 444 * Check this RNDIS complete message. 445 */ 446 if (unlikely(hdr->type != comp_type)) { 447 PMD_DRV_LOG(ERR, 448 "unexpected RNDIS response complete %#x expect %#x", 449 hdr->type, comp_type); 450 451 return -ENXIO; 452 } 453 if (unlikely(hdr->rid != rid)) { 454 PMD_DRV_LOG(ERR, 455 "RNDIS comp rid mismatch %#x, expect %#x", 456 hdr->rid, rid); 457 return -EINVAL; 458 } 459 460 /* All pass! */ 461 return 0; 462 } 463 464 static int 465 hn_rndis_query(struct hn_data *hv, uint32_t oid, 466 const void *idata, uint32_t idlen, 467 void *odata, uint32_t odlen) 468 { 469 struct rndis_query_req *req; 470 struct rndis_query_comp *comp; 471 uint32_t reqlen, comp_len; 472 int error = -EIO; 473 unsigned int ofs; 474 uint32_t rid; 475 476 reqlen = sizeof(*req) + idlen; 477 req = hn_rndis_alloc(reqlen); 478 if (req == NULL) 479 return -ENOMEM; 480 481 comp_len = sizeof(*comp) + odlen; 482 comp = rte_zmalloc("QUERY", comp_len, rte_mem_page_size()); 483 if (!comp) { 484 error = -ENOMEM; 485 goto done; 486 } 487 comp->status = RNDIS_STATUS_PENDING; 488 489 rid = hn_rndis_rid(hv); 490 491 req->type = RNDIS_QUERY_MSG; 492 req->len = reqlen; 493 req->rid = rid; 494 req->oid = oid; 495 req->infobufoffset = RNDIS_QUERY_REQ_INFOBUFOFFSET; 496 req->infobuflen = idlen; 497 498 /* Input data immediately follows RNDIS query. */ 499 memcpy(req + 1, idata, idlen); 500 501 error = hn_rndis_execute(hv, rid, req, reqlen, 502 comp, comp_len, RNDIS_QUERY_CMPLT); 503 504 if (error) 505 goto done; 506 507 if (comp->status != RNDIS_STATUS_SUCCESS) { 508 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x failed: status 0x%08x", 509 oid, comp->status); 510 error = -EINVAL; 511 goto done; 512 } 513 514 if (comp->infobuflen == 0 || comp->infobufoffset == 0) { 515 /* No output data! */ 516 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x, no data", oid); 517 error = 0; 518 goto done; 519 } 520 521 /* 522 * Check output data length and offset. 523 */ 524 /* ofs is the offset from the beginning of comp. */ 525 ofs = RNDIS_QUERY_COMP_INFOBUFOFFSET_ABS(comp->infobufoffset); 526 if (ofs < sizeof(*comp) || ofs + comp->infobuflen > comp_len) { 527 PMD_DRV_LOG(ERR, "RNDIS query invalid comp ib off/len, %u/%u", 528 comp->infobufoffset, comp->infobuflen); 529 error = -EINVAL; 530 goto done; 531 } 532 533 /* Save output data. */ 534 if (comp->infobuflen < odlen) 535 odlen = comp->infobuflen; 536 537 /* ofs is the offset from the beginning of comp. */ 538 memcpy(odata, (const char *)comp + ofs, odlen); 539 540 error = 0; 541 done: 542 rte_free(comp); 543 rte_free(req); 544 return error; 545 } 546 547 static int 548 hn_rndis_halt(struct hn_data *hv) 549 { 550 struct rndis_halt_req *halt; 551 552 halt = hn_rndis_alloc(sizeof(*halt)); 553 if (halt == NULL) 554 return -ENOMEM; 555 556 halt->type = RNDIS_HALT_MSG; 557 halt->len = sizeof(*halt); 558 halt->rid = hn_rndis_rid(hv); 559 560 /* No RNDIS completion; rely on NVS message send completion */ 561 hn_rndis_exec1(hv, halt, sizeof(*halt), NULL, 0); 562 563 rte_free(halt); 564 565 PMD_INIT_LOG(DEBUG, "RNDIS halt done"); 566 return 0; 567 } 568 569 static int 570 hn_rndis_query_hwcaps(struct hn_data *hv, struct ndis_offload *caps) 571 { 572 struct ndis_offload in; 573 uint32_t caps_len, size; 574 int error; 575 576 memset(caps, 0, sizeof(*caps)); 577 memset(&in, 0, sizeof(in)); 578 in.ndis_hdr.ndis_type = NDIS_OBJTYPE_OFFLOAD; 579 580 if (hv->ndis_ver >= NDIS_VERSION_6_30) { 581 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_3; 582 size = NDIS_OFFLOAD_SIZE; 583 } else if (hv->ndis_ver >= NDIS_VERSION_6_1) { 584 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_2; 585 size = NDIS_OFFLOAD_SIZE_6_1; 586 } else { 587 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_1; 588 size = NDIS_OFFLOAD_SIZE_6_0; 589 } 590 in.ndis_hdr.ndis_size = size; 591 592 caps_len = NDIS_OFFLOAD_SIZE; 593 error = hn_rndis_query(hv, OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES, 594 &in, size, caps, caps_len); 595 if (error) 596 return error; 597 598 /* Preliminary verification. */ 599 if (caps->ndis_hdr.ndis_type != NDIS_OBJTYPE_OFFLOAD) { 600 PMD_DRV_LOG(NOTICE, "invalid NDIS objtype 0x%02x", 601 caps->ndis_hdr.ndis_type); 602 return -EINVAL; 603 } 604 if (caps->ndis_hdr.ndis_rev < NDIS_OFFLOAD_REV_1) { 605 PMD_DRV_LOG(NOTICE, "invalid NDIS objrev 0x%02x", 606 caps->ndis_hdr.ndis_rev); 607 return -EINVAL; 608 } 609 if (caps->ndis_hdr.ndis_size > caps_len) { 610 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u, data size %u", 611 caps->ndis_hdr.ndis_size, caps_len); 612 return -EINVAL; 613 } else if (caps->ndis_hdr.ndis_size < NDIS_OFFLOAD_SIZE_6_0) { 614 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u", 615 caps->ndis_hdr.ndis_size); 616 return -EINVAL; 617 } 618 619 return 0; 620 } 621 622 int 623 hn_rndis_query_rsscaps(struct hn_data *hv, 624 unsigned int *rxr_cnt0) 625 { 626 struct ndis_rss_caps in, caps; 627 unsigned int indsz, rxr_cnt; 628 uint32_t caps_len; 629 int error; 630 631 *rxr_cnt0 = 0; 632 633 if (hv->ndis_ver < NDIS_VERSION_6_20) { 634 PMD_DRV_LOG(DEBUG, "RSS not supported on this host"); 635 return -EOPNOTSUPP; 636 } 637 638 memset(&in, 0, sizeof(in)); 639 in.ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_CAPS; 640 in.ndis_hdr.ndis_rev = NDIS_RSS_CAPS_REV_2; 641 in.ndis_hdr.ndis_size = NDIS_RSS_CAPS_SIZE; 642 643 caps_len = NDIS_RSS_CAPS_SIZE; 644 error = hn_rndis_query(hv, OID_GEN_RECEIVE_SCALE_CAPABILITIES, 645 &in, NDIS_RSS_CAPS_SIZE, 646 &caps, caps_len); 647 if (error) 648 return error; 649 650 PMD_INIT_LOG(DEBUG, "RX rings %u indirect %u caps %#x", 651 caps.ndis_nrxr, caps.ndis_nind, caps.ndis_caps); 652 /* 653 * Preliminary verification. 654 */ 655 if (caps.ndis_hdr.ndis_type != NDIS_OBJTYPE_RSS_CAPS) { 656 PMD_DRV_LOG(ERR, "invalid NDIS objtype 0x%02x", 657 caps.ndis_hdr.ndis_type); 658 return -EINVAL; 659 } 660 if (caps.ndis_hdr.ndis_rev < NDIS_RSS_CAPS_REV_1) { 661 PMD_DRV_LOG(ERR, "invalid NDIS objrev 0x%02x", 662 caps.ndis_hdr.ndis_rev); 663 return -EINVAL; 664 } 665 if (caps.ndis_hdr.ndis_size > caps_len) { 666 PMD_DRV_LOG(ERR, 667 "invalid NDIS objsize %u, data size %u", 668 caps.ndis_hdr.ndis_size, caps_len); 669 return -EINVAL; 670 } else if (caps.ndis_hdr.ndis_size < NDIS_RSS_CAPS_SIZE_6_0) { 671 PMD_DRV_LOG(ERR, "invalid NDIS objsize %u", 672 caps.ndis_hdr.ndis_size); 673 return -EINVAL; 674 } 675 676 /* 677 * Save information for later RSS configuration. 678 */ 679 if (caps.ndis_nrxr == 0) { 680 PMD_DRV_LOG(ERR, "0 RX rings!?"); 681 return -EINVAL; 682 } 683 rxr_cnt = caps.ndis_nrxr; 684 685 if (caps.ndis_hdr.ndis_size == NDIS_RSS_CAPS_SIZE && 686 caps.ndis_hdr.ndis_rev >= NDIS_RSS_CAPS_REV_2) { 687 if (caps.ndis_nind > NDIS_HASH_INDCNT) { 688 PMD_DRV_LOG(ERR, 689 "too many RSS indirect table entries %u", 690 caps.ndis_nind); 691 return -EOPNOTSUPP; 692 } 693 if (!rte_is_power_of_2(caps.ndis_nind)) { 694 PMD_DRV_LOG(ERR, 695 "RSS indirect table size is not power-of-2 %u", 696 caps.ndis_nind); 697 } 698 699 indsz = caps.ndis_nind; 700 } else { 701 indsz = NDIS_HASH_INDCNT; 702 } 703 704 if (indsz < rxr_cnt) { 705 PMD_DRV_LOG(NOTICE, 706 "# of RX rings (%d) > RSS indirect table size %d", 707 rxr_cnt, indsz); 708 rxr_cnt = indsz; 709 } 710 711 hv->rss_offloads = 0; 712 if (caps.ndis_caps & NDIS_RSS_CAP_IPV4) 713 hv->rss_offloads |= ETH_RSS_IPV4 714 | ETH_RSS_NONFRAG_IPV4_TCP 715 | ETH_RSS_NONFRAG_IPV4_UDP; 716 if (caps.ndis_caps & NDIS_RSS_CAP_IPV6) 717 hv->rss_offloads |= ETH_RSS_IPV6 718 | ETH_RSS_NONFRAG_IPV6_TCP; 719 if (caps.ndis_caps & NDIS_RSS_CAP_IPV6_EX) 720 hv->rss_offloads |= ETH_RSS_IPV6_EX 721 | ETH_RSS_IPV6_TCP_EX; 722 723 /* Commit! */ 724 *rxr_cnt0 = rxr_cnt; 725 726 return 0; 727 } 728 729 static int 730 hn_rndis_set(struct hn_data *hv, uint32_t oid, const void *data, uint32_t dlen) 731 { 732 struct rndis_set_req *req; 733 struct rndis_set_comp comp; 734 uint32_t reqlen, comp_len; 735 uint32_t rid; 736 int error; 737 738 reqlen = sizeof(*req) + dlen; 739 req = rte_zmalloc("RNDIS_SET", reqlen, rte_mem_page_size()); 740 if (!req) 741 return -ENOMEM; 742 743 rid = hn_rndis_rid(hv); 744 req->type = RNDIS_SET_MSG; 745 req->len = reqlen; 746 req->rid = rid; 747 req->oid = oid; 748 req->infobuflen = dlen; 749 req->infobufoffset = RNDIS_SET_REQ_INFOBUFOFFSET; 750 751 /* Data immediately follows RNDIS set. */ 752 memcpy(req + 1, data, dlen); 753 754 comp_len = sizeof(comp); 755 error = hn_rndis_execute(hv, rid, req, reqlen, 756 &comp, comp_len, 757 RNDIS_SET_CMPLT); 758 if (error) { 759 PMD_DRV_LOG(ERR, "exec RNDIS set %#" PRIx32 " failed", 760 oid); 761 error = EIO; 762 goto done; 763 } 764 765 if (comp.status != RNDIS_STATUS_SUCCESS) { 766 PMD_DRV_LOG(ERR, 767 "RNDIS set %#" PRIx32 " failed: status %#" PRIx32, 768 oid, comp.status); 769 error = EIO; 770 goto done; 771 } 772 773 done: 774 rte_free(req); 775 return error; 776 } 777 778 int hn_rndis_conf_offload(struct hn_data *hv, 779 uint64_t tx_offloads, uint64_t rx_offloads) 780 { 781 struct ndis_offload_params params; 782 struct ndis_offload hwcaps; 783 int error; 784 785 error = hn_rndis_query_hwcaps(hv, &hwcaps); 786 if (error) { 787 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 788 return error; 789 } 790 791 /* NOTE: 0 means "no change" */ 792 memset(¶ms, 0, sizeof(params)); 793 794 params.ndis_hdr.ndis_type = NDIS_OBJTYPE_DEFAULT; 795 if (hv->ndis_ver < NDIS_VERSION_6_30) { 796 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_2; 797 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE_6_1; 798 } else { 799 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_3; 800 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE; 801 } 802 803 if (tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM) { 804 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_TCP4) 805 params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_TX; 806 else 807 goto unsupported; 808 809 if (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_TCP6) 810 params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_TX; 811 else 812 goto unsupported; 813 } 814 815 if (rx_offloads & DEV_RX_OFFLOAD_TCP_CKSUM) { 816 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) 817 == NDIS_RXCSUM_CAP_TCP4) 818 params.ndis_tcp4csum |= NDIS_OFFLOAD_PARAM_RX; 819 else 820 goto unsupported; 821 822 if ((hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6) 823 == NDIS_RXCSUM_CAP_TCP6) 824 params.ndis_tcp6csum |= NDIS_OFFLOAD_PARAM_RX; 825 else 826 goto unsupported; 827 } 828 829 if (tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) { 830 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) 831 params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_TX; 832 else 833 goto unsupported; 834 835 if ((hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6) 836 == NDIS_TXCSUM_CAP_UDP6) 837 params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_TX; 838 else 839 goto unsupported; 840 } 841 842 if (rx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) { 843 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) 844 params.ndis_udp4csum |= NDIS_OFFLOAD_PARAM_RX; 845 else 846 goto unsupported; 847 848 if (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6) 849 params.ndis_udp6csum |= NDIS_OFFLOAD_PARAM_RX; 850 else 851 goto unsupported; 852 } 853 854 if (tx_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) { 855 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_IP4) 856 == NDIS_TXCSUM_CAP_IP4) 857 params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_TX; 858 else 859 goto unsupported; 860 } 861 if (rx_offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) { 862 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 863 params.ndis_ip4csum |= NDIS_OFFLOAD_PARAM_RX; 864 else 865 goto unsupported; 866 } 867 868 if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO) { 869 if (hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) 870 params.ndis_lsov2_ip4 = NDIS_OFFLOAD_LSOV2_ON; 871 else 872 goto unsupported; 873 874 if ((hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6) 875 == HN_NDIS_LSOV2_CAP_IP6) 876 params.ndis_lsov2_ip6 = NDIS_OFFLOAD_LSOV2_ON; 877 else 878 goto unsupported; 879 } 880 881 error = hn_rndis_set(hv, OID_TCP_OFFLOAD_PARAMETERS, ¶ms, 882 params.ndis_hdr.ndis_size); 883 if (error) { 884 PMD_DRV_LOG(ERR, "offload config failed"); 885 return error; 886 } 887 888 return 0; 889 unsupported: 890 PMD_DRV_LOG(NOTICE, 891 "offload tx:%" PRIx64 " rx:%" PRIx64 " not supported by this version", 892 tx_offloads, rx_offloads); 893 return -EINVAL; 894 } 895 896 int hn_rndis_get_offload(struct hn_data *hv, 897 struct rte_eth_dev_info *dev_info) 898 { 899 struct ndis_offload hwcaps; 900 int error; 901 902 memset(&hwcaps, 0, sizeof(hwcaps)); 903 904 error = hn_rndis_query_hwcaps(hv, &hwcaps); 905 if (error) { 906 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 907 return error; 908 } 909 910 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS | 911 DEV_TX_OFFLOAD_VLAN_INSERT; 912 913 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_IP4) 914 == HN_NDIS_TXCSUM_CAP_IP4) 915 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM; 916 917 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_TCP4) 918 == HN_NDIS_TXCSUM_CAP_TCP4 && 919 (hwcaps.ndis_csum.ndis_ip6_txcsum & HN_NDIS_TXCSUM_CAP_TCP6) 920 == HN_NDIS_TXCSUM_CAP_TCP6) 921 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_CKSUM; 922 923 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) && 924 (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6)) 925 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_UDP_CKSUM; 926 927 if ((hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) && 928 (hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6) 929 == HN_NDIS_LSOV2_CAP_IP6) 930 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; 931 932 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP | 933 DEV_RX_OFFLOAD_RSS_HASH; 934 935 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 936 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM; 937 938 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) && 939 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)) 940 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_CKSUM; 941 942 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) && 943 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)) 944 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_UDP_CKSUM; 945 946 return 0; 947 } 948 949 uint32_t 950 hn_rndis_get_ptypes(struct hn_data *hv) 951 { 952 struct ndis_offload hwcaps; 953 uint32_t ptypes; 954 int error; 955 956 memset(&hwcaps, 0, sizeof(hwcaps)); 957 958 error = hn_rndis_query_hwcaps(hv, &hwcaps); 959 if (error) { 960 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 961 return RTE_PTYPE_L2_ETHER; 962 } 963 964 ptypes = RTE_PTYPE_L2_ETHER; 965 966 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 967 ptypes |= RTE_PTYPE_L3_IPV4; 968 969 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) || 970 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)) 971 ptypes |= RTE_PTYPE_L4_TCP; 972 973 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) || 974 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)) 975 ptypes |= RTE_PTYPE_L4_UDP; 976 977 return ptypes; 978 } 979 980 int 981 hn_rndis_set_rxfilter(struct hn_data *hv, uint32_t filter) 982 { 983 int error; 984 985 error = hn_rndis_set(hv, OID_GEN_CURRENT_PACKET_FILTER, 986 &filter, sizeof(filter)); 987 if (error) { 988 PMD_DRV_LOG(ERR, "set RX filter %#" PRIx32 " failed: %d", 989 filter, error); 990 } else { 991 PMD_DRV_LOG(DEBUG, "set RX filter %#" PRIx32 " done", filter); 992 } 993 994 return error; 995 } 996 997 int hn_rndis_conf_rss(struct hn_data *hv, uint32_t flags) 998 { 999 struct ndis_rssprm_toeplitz rssp; 1000 struct ndis_rss_params *prm = &rssp.rss_params; 1001 unsigned int i; 1002 int error; 1003 1004 memset(&rssp, 0, sizeof(rssp)); 1005 1006 prm->ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_PARAMS; 1007 prm->ndis_hdr.ndis_rev = NDIS_RSS_PARAMS_REV_2; 1008 prm->ndis_hdr.ndis_size = sizeof(*prm); 1009 prm->ndis_flags = flags; 1010 prm->ndis_hash = hv->rss_hash; 1011 prm->ndis_indsize = sizeof(rssp.rss_ind[0]) * NDIS_HASH_INDCNT; 1012 prm->ndis_indoffset = offsetof(struct ndis_rssprm_toeplitz, rss_ind[0]); 1013 prm->ndis_keysize = NDIS_HASH_KEYSIZE_TOEPLITZ; 1014 prm->ndis_keyoffset = offsetof(struct ndis_rssprm_toeplitz, rss_key[0]); 1015 1016 for (i = 0; i < NDIS_HASH_INDCNT; i++) 1017 rssp.rss_ind[i] = hv->rss_ind[i]; 1018 1019 /* Set hask key values */ 1020 memcpy(&rssp.rss_key, hv->rss_key, NDIS_HASH_KEYSIZE_TOEPLITZ); 1021 1022 error = hn_rndis_set(hv, OID_GEN_RECEIVE_SCALE_PARAMETERS, 1023 &rssp, sizeof(rssp)); 1024 if (error != 0) { 1025 PMD_DRV_LOG(ERR, 1026 "RSS config num queues=%u failed: %d", 1027 hv->num_queues, error); 1028 } 1029 return error; 1030 } 1031 1032 static int hn_rndis_init(struct hn_data *hv) 1033 { 1034 struct rndis_init_req *req; 1035 struct rndis_init_comp comp; 1036 uint32_t comp_len, rid; 1037 int error; 1038 1039 req = hn_rndis_alloc(sizeof(*req)); 1040 if (!req) { 1041 PMD_DRV_LOG(ERR, "no memory for RNDIS init"); 1042 return -ENXIO; 1043 } 1044 1045 rid = hn_rndis_rid(hv); 1046 req->type = RNDIS_INITIALIZE_MSG; 1047 req->len = sizeof(*req); 1048 req->rid = rid; 1049 req->ver_major = RNDIS_VERSION_MAJOR; 1050 req->ver_minor = RNDIS_VERSION_MINOR; 1051 req->max_xfersz = HN_RNDIS_XFER_SIZE; 1052 1053 comp_len = RNDIS_INIT_COMP_SIZE_MIN; 1054 error = hn_rndis_execute(hv, rid, req, sizeof(*req), 1055 &comp, comp_len, 1056 RNDIS_INITIALIZE_CMPLT); 1057 if (error) 1058 goto done; 1059 1060 if (comp.status != RNDIS_STATUS_SUCCESS) { 1061 PMD_DRV_LOG(ERR, "RNDIS init failed: status 0x%08x", 1062 comp.status); 1063 error = -EIO; 1064 goto done; 1065 } 1066 1067 hv->rndis_agg_size = comp.pktmaxsz; 1068 hv->rndis_agg_pkts = comp.pktmaxcnt; 1069 hv->rndis_agg_align = 1U << comp.align; 1070 1071 if (hv->rndis_agg_align < sizeof(uint32_t)) { 1072 /* 1073 * The RNDIS packet message encap assumes that the RNDIS 1074 * packet message is at least 4 bytes aligned. Fix up the 1075 * alignment here, if the remote side sets the alignment 1076 * too low. 1077 */ 1078 PMD_DRV_LOG(NOTICE, 1079 "fixup RNDIS aggpkt align: %u -> %zu", 1080 hv->rndis_agg_align, sizeof(uint32_t)); 1081 hv->rndis_agg_align = sizeof(uint32_t); 1082 } 1083 1084 PMD_INIT_LOG(INFO, 1085 "RNDIS ver %u.%u, aggpkt size %u, aggpkt cnt %u, aggpkt align %u", 1086 comp.ver_major, comp.ver_minor, 1087 hv->rndis_agg_size, hv->rndis_agg_pkts, 1088 hv->rndis_agg_align); 1089 error = 0; 1090 done: 1091 rte_free(req); 1092 return error; 1093 } 1094 1095 int 1096 hn_rndis_get_eaddr(struct hn_data *hv, uint8_t *eaddr) 1097 { 1098 uint32_t eaddr_len; 1099 int error; 1100 1101 eaddr_len = RTE_ETHER_ADDR_LEN; 1102 error = hn_rndis_query(hv, OID_802_3_PERMANENT_ADDRESS, NULL, 0, 1103 eaddr, eaddr_len); 1104 if (error) 1105 return error; 1106 1107 PMD_DRV_LOG(INFO, "MAC address " RTE_ETHER_ADDR_PRT_FMT, 1108 eaddr[0], eaddr[1], eaddr[2], 1109 eaddr[3], eaddr[4], eaddr[5]); 1110 return 0; 1111 } 1112 1113 int 1114 hn_rndis_get_linkstatus(struct hn_data *hv) 1115 { 1116 return hn_rndis_query(hv, OID_GEN_MEDIA_CONNECT_STATUS, NULL, 0, 1117 &hv->link_status, sizeof(uint32_t)); 1118 } 1119 1120 int 1121 hn_rndis_get_linkspeed(struct hn_data *hv) 1122 { 1123 return hn_rndis_query(hv, OID_GEN_LINK_SPEED, NULL, 0, 1124 &hv->link_speed, sizeof(uint32_t)); 1125 } 1126 1127 int 1128 hn_rndis_attach(struct hn_data *hv) 1129 { 1130 /* Initialize RNDIS. */ 1131 return hn_rndis_init(hv); 1132 } 1133 1134 void 1135 hn_rndis_detach(struct hn_data *hv) 1136 { 1137 struct rte_eth_dev *dev = &rte_eth_devices[hv->port_id]; 1138 1139 rte_eal_alarm_cancel(hn_rndis_link_alarm, dev); 1140 1141 /* Halt the RNDIS. */ 1142 hn_rndis_halt(hv); 1143 } 1144