1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2009-2018 Microsoft Corp. 3 * Copyright (c) 2010-2012 Citrix Inc. 4 * Copyright (c) 2012 NetApp Inc. 5 * All rights reserved. 6 */ 7 8 #include <stdint.h> 9 #include <string.h> 10 #include <stdio.h> 11 #include <errno.h> 12 #include <unistd.h> 13 14 #include <rte_ethdev.h> 15 #include <rte_string_fns.h> 16 #include <rte_memzone.h> 17 #include <rte_malloc.h> 18 #include <rte_atomic.h> 19 #include <rte_branch_prediction.h> 20 #include <rte_ether.h> 21 #include <rte_common.h> 22 #include <rte_errno.h> 23 #include <rte_cycles.h> 24 #include <rte_memory.h> 25 #include <rte_eal.h> 26 #include <rte_dev.h> 27 #include <rte_bus_vmbus.h> 28 29 #include "hn_logs.h" 30 #include "hn_var.h" 31 #include "hn_nvs.h" 32 #include "hn_rndis.h" 33 #include "ndis.h" 34 35 #define HN_RNDIS_XFER_SIZE 0x4000 36 37 #define HN_NDIS_TXCSUM_CAP_IP4 \ 38 (NDIS_TXCSUM_CAP_IP4 | NDIS_TXCSUM_CAP_IP4OPT) 39 #define HN_NDIS_TXCSUM_CAP_TCP4 \ 40 (NDIS_TXCSUM_CAP_TCP4 | NDIS_TXCSUM_CAP_TCP4OPT) 41 #define HN_NDIS_TXCSUM_CAP_TCP6 \ 42 (NDIS_TXCSUM_CAP_TCP6 | NDIS_TXCSUM_CAP_TCP6OPT | \ 43 NDIS_TXCSUM_CAP_IP6EXT) 44 #define HN_NDIS_TXCSUM_CAP_UDP6 \ 45 (NDIS_TXCSUM_CAP_UDP6 | NDIS_TXCSUM_CAP_IP6EXT) 46 #define HN_NDIS_LSOV2_CAP_IP6 \ 47 (NDIS_LSOV2_CAP_IP6EXT | NDIS_LSOV2_CAP_TCP6OPT) 48 49 /* Get unique request id */ 50 static inline uint32_t 51 hn_rndis_rid(struct hn_data *hv) 52 { 53 uint32_t rid; 54 55 do { 56 rid = rte_atomic32_add_return(&hv->rndis_req_id, 1); 57 } while (rid == 0); 58 59 return rid; 60 } 61 62 static void *hn_rndis_alloc(struct hn_data *hv, size_t size) 63 { 64 return rte_zmalloc_socket("RNDIS", size, PAGE_SIZE, 65 hv->vmbus->device.numa_node); 66 } 67 68 #ifdef RTE_LIBRTE_NETVSC_DEBUG_DUMP 69 void hn_rndis_dump(const void *buf) 70 { 71 const union { 72 struct rndis_msghdr hdr; 73 struct rndis_packet_msg pkt; 74 struct rndis_init_req init_request; 75 struct rndis_init_comp init_complete; 76 struct rndis_halt_req halt; 77 struct rndis_query_req query_request; 78 struct rndis_query_comp query_complete; 79 struct rndis_set_req set_request; 80 struct rndis_set_comp set_complete; 81 struct rndis_reset_req reset_request; 82 struct rndis_reset_comp reset_complete; 83 struct rndis_keepalive_req keepalive_request; 84 struct rndis_keepalive_comp keepalive_complete; 85 struct rndis_status_msg indicate_status; 86 } *rndis_msg = buf; 87 88 switch (rndis_msg->hdr.type) { 89 case RNDIS_PACKET_MSG: { 90 const struct rndis_pktinfo *ppi; 91 unsigned int ppi_len; 92 93 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 94 "RNDIS_MSG_PACKET (len %u, data %u:%u, # oob %u %u:%u, pkt %u:%u)\n", 95 rndis_msg->pkt.len, 96 rndis_msg->pkt.dataoffset, 97 rndis_msg->pkt.datalen, 98 rndis_msg->pkt.oobdataelements, 99 rndis_msg->pkt.oobdataoffset, 100 rndis_msg->pkt.oobdatalen, 101 rndis_msg->pkt.pktinfooffset, 102 rndis_msg->pkt.pktinfolen); 103 104 ppi = (const struct rndis_pktinfo *) 105 ((const char *)buf 106 + RNDIS_PACKET_MSG_OFFSET_ABS(rndis_msg->pkt.pktinfooffset)); 107 108 ppi_len = rndis_msg->pkt.pktinfolen; 109 while (ppi_len > 0) { 110 const void *ppi_data; 111 112 ppi_data = ppi->data; 113 114 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 115 " PPI (size %u, type %u, offs %u data %#x)\n", 116 ppi->size, ppi->type, ppi->offset, 117 *(const uint32_t *)ppi_data); 118 if (ppi->size == 0) 119 break; 120 ppi_len -= ppi->size; 121 ppi = (const struct rndis_pktinfo *) 122 ((const char *)ppi + ppi->size); 123 } 124 break; 125 } 126 case RNDIS_INITIALIZE_MSG: 127 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 128 "RNDIS_MSG_INIT (len %u id %#x, ver %u.%u max xfer %u)\n", 129 rndis_msg->init_request.len, 130 rndis_msg->init_request.rid, 131 rndis_msg->init_request.ver_major, 132 rndis_msg->init_request.ver_minor, 133 rndis_msg->init_request.max_xfersz); 134 break; 135 136 case RNDIS_INITIALIZE_CMPLT: 137 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 138 "RNDIS_MSG_INIT_C (len %u, id %#x, status 0x%x, vers %u.%u, " 139 "flags %d, max xfer %u, max pkts %u, aligned %u)\n", 140 rndis_msg->init_complete.len, 141 rndis_msg->init_complete.rid, 142 rndis_msg->init_complete.status, 143 rndis_msg->init_complete.ver_major, 144 rndis_msg->init_complete.ver_minor, 145 rndis_msg->init_complete.devflags, 146 rndis_msg->init_complete.pktmaxsz, 147 rndis_msg->init_complete.pktmaxcnt, 148 rndis_msg->init_complete.align); 149 break; 150 151 case RNDIS_HALT_MSG: 152 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 153 "RNDIS_HALT (len %u id %#x)\n", 154 rndis_msg->halt.len, rndis_msg->halt.rid); 155 break; 156 157 case RNDIS_QUERY_MSG: 158 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 159 "RNDIS_QUERY (len %u, id %#x, oid %#x, info %u:%u)\n", 160 rndis_msg->query_request.len, 161 rndis_msg->query_request.rid, 162 rndis_msg->query_request.oid, 163 rndis_msg->query_request.infobuflen, 164 rndis_msg->query_request.infobufoffset); 165 break; 166 167 case RNDIS_QUERY_CMPLT: 168 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 169 "RNDIS_MSG_QUERY_C (len %u, id %#x, status 0x%x, buf %u:%u)\n", 170 rndis_msg->query_complete.len, 171 rndis_msg->query_complete.rid, 172 rndis_msg->query_complete.status, 173 rndis_msg->query_complete.infobuflen, 174 rndis_msg->query_complete.infobufoffset); 175 break; 176 177 case RNDIS_SET_MSG: 178 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 179 "RNDIS_SET (len %u, id %#x, oid %#x, info %u:%u)\n", 180 rndis_msg->set_request.len, 181 rndis_msg->set_request.rid, 182 rndis_msg->set_request.oid, 183 rndis_msg->set_request.infobuflen, 184 rndis_msg->set_request.infobufoffset); 185 break; 186 187 case RNDIS_SET_CMPLT: 188 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 189 "RNDIS_MSG_SET_C (len %u, id 0x%x, status 0x%x)\n", 190 rndis_msg->set_complete.len, 191 rndis_msg->set_complete.rid, 192 rndis_msg->set_complete.status); 193 break; 194 195 case RNDIS_INDICATE_STATUS_MSG: 196 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 197 "RNDIS_MSG_INDICATE (len %u, status %#x, buf len %u, buf offset %u)\n", 198 rndis_msg->indicate_status.len, 199 rndis_msg->indicate_status.status, 200 rndis_msg->indicate_status.stbuflen, 201 rndis_msg->indicate_status.stbufoffset); 202 break; 203 204 case RNDIS_RESET_MSG: 205 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 206 "RNDIS_RESET (len %u, id %#x)\n", 207 rndis_msg->reset_request.len, 208 rndis_msg->reset_request.rid); 209 break; 210 211 case RNDIS_RESET_CMPLT: 212 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 213 "RNDIS_RESET_C (len %u, status %#x address %#x)\n", 214 rndis_msg->reset_complete.len, 215 rndis_msg->reset_complete.status, 216 rndis_msg->reset_complete.adrreset); 217 break; 218 219 case RNDIS_KEEPALIVE_MSG: 220 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 221 "RNDIS_KEEPALIVE (len %u, id %#x)\n", 222 rndis_msg->keepalive_request.len, 223 rndis_msg->keepalive_request.rid); 224 break; 225 226 case RNDIS_KEEPALIVE_CMPLT: 227 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 228 "RNDIS_KEEPALIVE_C (len %u, id %#x address %#x)\n", 229 rndis_msg->keepalive_complete.len, 230 rndis_msg->keepalive_complete.rid, 231 rndis_msg->keepalive_complete.status); 232 break; 233 234 default: 235 rte_log(RTE_LOG_DEBUG, hn_logtype_driver, 236 "RNDIS type %#x len %u\n", 237 rndis_msg->hdr.type, 238 rndis_msg->hdr.len); 239 break; 240 } 241 } 242 #endif 243 244 static int hn_nvs_send_rndis_ctrl(struct vmbus_channel *chan, 245 const void *req, uint32_t reqlen) 246 247 { 248 struct hn_nvs_rndis nvs_rndis = { 249 .type = NVS_TYPE_RNDIS, 250 .rndis_mtype = NVS_RNDIS_MTYPE_CTRL, 251 .chim_idx = NVS_CHIM_IDX_INVALID, 252 .chim_sz = 0 253 }; 254 struct vmbus_gpa sg; 255 rte_iova_t addr; 256 257 addr = rte_malloc_virt2iova(req); 258 if (unlikely(addr == RTE_BAD_IOVA)) { 259 PMD_DRV_LOG(ERR, "RNDIS send request can not get iova"); 260 return -EINVAL; 261 } 262 263 if (unlikely(reqlen > PAGE_SIZE)) { 264 PMD_DRV_LOG(ERR, "RNDIS request %u greater than page size", 265 reqlen); 266 return -EINVAL; 267 } 268 269 sg.page = addr / PAGE_SIZE; 270 sg.ofs = addr & PAGE_MASK; 271 sg.len = reqlen; 272 273 if (sg.ofs + reqlen > PAGE_SIZE) { 274 PMD_DRV_LOG(ERR, "RNDIS request crosses page bounary"); 275 return -EINVAL; 276 } 277 278 hn_rndis_dump(req); 279 280 return hn_nvs_send_sglist(chan, &sg, 1, 281 &nvs_rndis, sizeof(nvs_rndis), 0U, NULL); 282 } 283 284 void hn_rndis_link_status(struct hn_data *hv __rte_unused, const void *msg) 285 { 286 const struct rndis_status_msg *indicate = msg; 287 288 hn_rndis_dump(msg); 289 290 PMD_DRV_LOG(DEBUG, "link status %#x", indicate->status); 291 292 switch (indicate->status) { 293 case RNDIS_STATUS_LINK_SPEED_CHANGE: 294 case RNDIS_STATUS_NETWORK_CHANGE: 295 case RNDIS_STATUS_TASK_OFFLOAD_CURRENT_CONFIG: 296 /* ignore not in DPDK API */ 297 break; 298 299 case RNDIS_STATUS_MEDIA_CONNECT: 300 case RNDIS_STATUS_MEDIA_DISCONNECT: 301 /* TODO handle as LSC interrupt */ 302 break; 303 default: 304 PMD_DRV_LOG(NOTICE, "unknown RNDIS indication: %#x", 305 indicate->status); 306 } 307 } 308 309 /* Callback from hn_process_events when response is visible */ 310 void hn_rndis_receive_response(struct hn_data *hv, 311 const void *data, uint32_t len) 312 { 313 const struct rndis_init_comp *hdr = data; 314 315 hn_rndis_dump(data); 316 317 if (len < sizeof(3 * sizeof(uint32_t))) { 318 PMD_DRV_LOG(ERR, 319 "missing RNDIS header %u", len); 320 return; 321 } 322 323 if (len < hdr->len) { 324 PMD_DRV_LOG(ERR, 325 "truncated RNDIS response %u", len); 326 return; 327 } 328 329 if (len > sizeof(hv->rndis_resp)) { 330 PMD_DRV_LOG(NOTICE, 331 "RNDIS response exceeds buffer"); 332 len = sizeof(hv->rndis_resp); 333 } 334 335 if (hdr->rid == 0) { 336 PMD_DRV_LOG(NOTICE, 337 "RNDIS response id zero!"); 338 } 339 340 memcpy(hv->rndis_resp, data, len); 341 342 /* make sure response copied before update */ 343 rte_smp_wmb(); 344 345 if (rte_atomic32_cmpset(&hv->rndis_pending, hdr->rid, 0) == 0) { 346 PMD_DRV_LOG(ERR, 347 "received id %#x pending id %#x", 348 hdr->rid, (uint32_t)hv->rndis_pending); 349 } 350 } 351 352 /* Do request/response transaction */ 353 static int hn_rndis_exec1(struct hn_data *hv, 354 const void *req, uint32_t reqlen, 355 void *comp, uint32_t comp_len) 356 { 357 const struct rndis_halt_req *hdr = req; 358 uint32_t rid = hdr->rid; 359 struct vmbus_channel *chan = hn_primary_chan(hv); 360 int error; 361 362 if (comp_len > sizeof(hv->rndis_resp)) { 363 PMD_DRV_LOG(ERR, 364 "Expected completion size %u exceeds buffer %zu", 365 comp_len, sizeof(hv->rndis_resp)); 366 return -EIO; 367 } 368 369 if (comp != NULL && 370 rte_atomic32_cmpset(&hv->rndis_pending, 0, rid) == 0) { 371 PMD_DRV_LOG(ERR, 372 "Request already pending"); 373 return -EBUSY; 374 } 375 376 error = hn_nvs_send_rndis_ctrl(chan, req, reqlen); 377 if (error) { 378 PMD_DRV_LOG(ERR, "RNDIS ctrl send failed: %d", error); 379 return error; 380 } 381 382 if (comp) { 383 /* Poll primary channel until response received */ 384 while (hv->rndis_pending == rid) 385 hn_process_events(hv, 0, 1); 386 387 memcpy(comp, hv->rndis_resp, comp_len); 388 } 389 390 return 0; 391 } 392 393 /* Do transaction and validate response */ 394 static int hn_rndis_execute(struct hn_data *hv, uint32_t rid, 395 const void *req, uint32_t reqlen, 396 void *comp, uint32_t comp_len, uint32_t comp_type) 397 { 398 const struct rndis_comp_hdr *hdr = comp; 399 int ret; 400 401 memset(comp, 0, comp_len); 402 403 ret = hn_rndis_exec1(hv, req, reqlen, comp, comp_len); 404 if (ret < 0) 405 return ret; 406 /* 407 * Check this RNDIS complete message. 408 */ 409 if (unlikely(hdr->type != comp_type)) { 410 PMD_DRV_LOG(ERR, 411 "unexpected RNDIS response complete %#x expect %#x", 412 hdr->type, comp_type); 413 414 return -ENXIO; 415 } 416 if (unlikely(hdr->rid != rid)) { 417 PMD_DRV_LOG(ERR, 418 "RNDIS comp rid mismatch %#x, expect %#x", 419 hdr->rid, rid); 420 return -EINVAL; 421 } 422 423 /* All pass! */ 424 return 0; 425 } 426 427 static int 428 hn_rndis_query(struct hn_data *hv, uint32_t oid, 429 const void *idata, uint32_t idlen, 430 void *odata, uint32_t odlen) 431 { 432 struct rndis_query_req *req; 433 struct rndis_query_comp *comp; 434 uint32_t reqlen, comp_len; 435 int error = -EIO; 436 unsigned int ofs; 437 uint32_t rid; 438 439 reqlen = sizeof(*req) + idlen; 440 req = hn_rndis_alloc(hv, reqlen); 441 if (req == NULL) 442 return -ENOMEM; 443 444 comp_len = sizeof(*comp) + odlen; 445 comp = rte_zmalloc("QUERY", comp_len, PAGE_SIZE); 446 if (!comp) { 447 error = -ENOMEM; 448 goto done; 449 } 450 comp->status = RNDIS_STATUS_PENDING; 451 452 rid = hn_rndis_rid(hv); 453 454 req->type = RNDIS_QUERY_MSG; 455 req->len = reqlen; 456 req->rid = rid; 457 req->oid = oid; 458 req->infobufoffset = RNDIS_QUERY_REQ_INFOBUFOFFSET; 459 req->infobuflen = idlen; 460 461 /* Input data immediately follows RNDIS query. */ 462 memcpy(req + 1, idata, idlen); 463 464 error = hn_rndis_execute(hv, rid, req, reqlen, 465 comp, comp_len, RNDIS_QUERY_CMPLT); 466 467 if (error) 468 goto done; 469 470 if (comp->status != RNDIS_STATUS_SUCCESS) { 471 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x failed: status 0x%08x", 472 oid, comp->status); 473 error = -EINVAL; 474 goto done; 475 } 476 477 if (comp->infobuflen == 0 || comp->infobufoffset == 0) { 478 /* No output data! */ 479 PMD_DRV_LOG(ERR, "RNDIS query 0x%08x, no data", oid); 480 error = 0; 481 goto done; 482 } 483 484 /* 485 * Check output data length and offset. 486 */ 487 /* ofs is the offset from the beginning of comp. */ 488 ofs = RNDIS_QUERY_COMP_INFOBUFOFFSET_ABS(comp->infobufoffset); 489 if (ofs < sizeof(*comp) || ofs + comp->infobuflen > comp_len) { 490 PMD_DRV_LOG(ERR, "RNDIS query invalid comp ib off/len, %u/%u", 491 comp->infobufoffset, comp->infobuflen); 492 error = -EINVAL; 493 goto done; 494 } 495 496 /* Save output data. */ 497 if (comp->infobuflen < odlen) 498 odlen = comp->infobuflen; 499 500 /* ofs is the offset from the beginning of comp. */ 501 memcpy(odata, (const char *)comp + ofs, odlen); 502 503 error = 0; 504 done: 505 rte_free(comp); 506 rte_free(req); 507 return error; 508 } 509 510 static int 511 hn_rndis_halt(struct hn_data *hv) 512 { 513 struct rndis_halt_req *halt; 514 515 halt = hn_rndis_alloc(hv, sizeof(*halt)); 516 if (halt == NULL) 517 return -ENOMEM; 518 519 halt->type = RNDIS_HALT_MSG; 520 halt->len = sizeof(*halt); 521 halt->rid = hn_rndis_rid(hv); 522 523 /* No RNDIS completion; rely on NVS message send completion */ 524 hn_rndis_exec1(hv, halt, sizeof(*halt), NULL, 0); 525 526 rte_free(halt); 527 528 PMD_INIT_LOG(DEBUG, "RNDIS halt done"); 529 return 0; 530 } 531 532 static int 533 hn_rndis_query_hwcaps(struct hn_data *hv, struct ndis_offload *caps) 534 { 535 struct ndis_offload in; 536 uint32_t caps_len, size; 537 int error; 538 539 memset(caps, 0, sizeof(*caps)); 540 memset(&in, 0, sizeof(in)); 541 in.ndis_hdr.ndis_type = NDIS_OBJTYPE_OFFLOAD; 542 543 if (hv->ndis_ver >= NDIS_VERSION_6_30) { 544 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_3; 545 size = NDIS_OFFLOAD_SIZE; 546 } else if (hv->ndis_ver >= NDIS_VERSION_6_1) { 547 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_2; 548 size = NDIS_OFFLOAD_SIZE_6_1; 549 } else { 550 in.ndis_hdr.ndis_rev = NDIS_OFFLOAD_REV_1; 551 size = NDIS_OFFLOAD_SIZE_6_0; 552 } 553 in.ndis_hdr.ndis_size = size; 554 555 caps_len = NDIS_OFFLOAD_SIZE; 556 error = hn_rndis_query(hv, OID_TCP_OFFLOAD_HARDWARE_CAPABILITIES, 557 &in, size, caps, caps_len); 558 if (error) 559 return error; 560 561 /* Preliminary verification. */ 562 if (caps->ndis_hdr.ndis_type != NDIS_OBJTYPE_OFFLOAD) { 563 PMD_DRV_LOG(NOTICE, "invalid NDIS objtype 0x%02x", 564 caps->ndis_hdr.ndis_type); 565 return -EINVAL; 566 } 567 if (caps->ndis_hdr.ndis_rev < NDIS_OFFLOAD_REV_1) { 568 PMD_DRV_LOG(NOTICE, "invalid NDIS objrev 0x%02x", 569 caps->ndis_hdr.ndis_rev); 570 return -EINVAL; 571 } 572 if (caps->ndis_hdr.ndis_size > caps_len) { 573 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u, data size %u", 574 caps->ndis_hdr.ndis_size, caps_len); 575 return -EINVAL; 576 } else if (caps->ndis_hdr.ndis_size < NDIS_OFFLOAD_SIZE_6_0) { 577 PMD_DRV_LOG(NOTICE, "invalid NDIS objsize %u", 578 caps->ndis_hdr.ndis_size); 579 return -EINVAL; 580 } 581 582 return 0; 583 } 584 585 int 586 hn_rndis_query_rsscaps(struct hn_data *hv, 587 unsigned int *rxr_cnt0) 588 { 589 struct ndis_rss_caps in, caps; 590 unsigned int indsz, rxr_cnt; 591 uint32_t caps_len; 592 int error; 593 594 *rxr_cnt0 = 0; 595 596 if (hv->ndis_ver < NDIS_VERSION_6_20) { 597 PMD_DRV_LOG(DEBUG, "RSS not supported on this host"); 598 return -EOPNOTSUPP; 599 } 600 601 memset(&in, 0, sizeof(in)); 602 in.ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_CAPS; 603 in.ndis_hdr.ndis_rev = NDIS_RSS_CAPS_REV_2; 604 in.ndis_hdr.ndis_size = NDIS_RSS_CAPS_SIZE; 605 606 caps_len = NDIS_RSS_CAPS_SIZE; 607 error = hn_rndis_query(hv, OID_GEN_RECEIVE_SCALE_CAPABILITIES, 608 &in, NDIS_RSS_CAPS_SIZE, 609 &caps, caps_len); 610 if (error) 611 return error; 612 613 PMD_INIT_LOG(DEBUG, "RX rings %u indirect %u caps %#x", 614 caps.ndis_nrxr, caps.ndis_nind, caps.ndis_caps); 615 /* 616 * Preliminary verification. 617 */ 618 if (caps.ndis_hdr.ndis_type != NDIS_OBJTYPE_RSS_CAPS) { 619 PMD_DRV_LOG(ERR, "invalid NDIS objtype 0x%02x", 620 caps.ndis_hdr.ndis_type); 621 return -EINVAL; 622 } 623 if (caps.ndis_hdr.ndis_rev < NDIS_RSS_CAPS_REV_1) { 624 PMD_DRV_LOG(ERR, "invalid NDIS objrev 0x%02x", 625 caps.ndis_hdr.ndis_rev); 626 return -EINVAL; 627 } 628 if (caps.ndis_hdr.ndis_size > caps_len) { 629 PMD_DRV_LOG(ERR, 630 "invalid NDIS objsize %u, data size %u", 631 caps.ndis_hdr.ndis_size, caps_len); 632 return -EINVAL; 633 } else if (caps.ndis_hdr.ndis_size < NDIS_RSS_CAPS_SIZE_6_0) { 634 PMD_DRV_LOG(ERR, "invalid NDIS objsize %u", 635 caps.ndis_hdr.ndis_size); 636 return -EINVAL; 637 } 638 639 /* 640 * Save information for later RSS configuration. 641 */ 642 if (caps.ndis_nrxr == 0) { 643 PMD_DRV_LOG(ERR, "0 RX rings!?"); 644 return -EINVAL; 645 } 646 rxr_cnt = caps.ndis_nrxr; 647 648 if (caps.ndis_hdr.ndis_size == NDIS_RSS_CAPS_SIZE && 649 caps.ndis_hdr.ndis_rev >= NDIS_RSS_CAPS_REV_2) { 650 if (caps.ndis_nind > NDIS_HASH_INDCNT) { 651 PMD_DRV_LOG(ERR, 652 "too many RSS indirect table entries %u", 653 caps.ndis_nind); 654 return -EOPNOTSUPP; 655 } 656 if (!rte_is_power_of_2(caps.ndis_nind)) { 657 PMD_DRV_LOG(ERR, 658 "RSS indirect table size is not power-of-2 %u", 659 caps.ndis_nind); 660 } 661 662 indsz = caps.ndis_nind; 663 } else { 664 indsz = NDIS_HASH_INDCNT; 665 } 666 667 if (indsz < rxr_cnt) { 668 PMD_DRV_LOG(NOTICE, 669 "# of RX rings (%d) > RSS indirect table size %d", 670 rxr_cnt, indsz); 671 rxr_cnt = indsz; 672 } 673 674 hv->rss_offloads = 0; 675 if (caps.ndis_caps & NDIS_RSS_CAP_IPV4) 676 hv->rss_offloads |= ETH_RSS_IPV4 677 | ETH_RSS_NONFRAG_IPV4_TCP 678 | ETH_RSS_NONFRAG_IPV4_UDP; 679 if (caps.ndis_caps & NDIS_RSS_CAP_IPV6) 680 hv->rss_offloads |= ETH_RSS_IPV6 681 | ETH_RSS_NONFRAG_IPV6_TCP; 682 if (caps.ndis_caps & NDIS_RSS_CAP_IPV6_EX) 683 hv->rss_offloads |= ETH_RSS_IPV6_EX 684 | ETH_RSS_IPV6_TCP_EX; 685 686 /* Commit! */ 687 *rxr_cnt0 = rxr_cnt; 688 689 return 0; 690 } 691 692 static int 693 hn_rndis_set(struct hn_data *hv, uint32_t oid, const void *data, uint32_t dlen) 694 { 695 struct rndis_set_req *req; 696 struct rndis_set_comp comp; 697 uint32_t reqlen, comp_len; 698 uint32_t rid; 699 int error; 700 701 reqlen = sizeof(*req) + dlen; 702 req = rte_zmalloc("RNDIS_SET", reqlen, PAGE_SIZE); 703 if (!req) 704 return -ENOMEM; 705 706 rid = hn_rndis_rid(hv); 707 req->type = RNDIS_SET_MSG; 708 req->len = reqlen; 709 req->rid = rid; 710 req->oid = oid; 711 req->infobuflen = dlen; 712 req->infobufoffset = RNDIS_SET_REQ_INFOBUFOFFSET; 713 714 /* Data immediately follows RNDIS set. */ 715 memcpy(req + 1, data, dlen); 716 717 comp_len = sizeof(comp); 718 error = hn_rndis_execute(hv, rid, req, reqlen, 719 &comp, comp_len, 720 RNDIS_SET_CMPLT); 721 if (error) { 722 PMD_DRV_LOG(ERR, "exec RNDIS set %#" PRIx32 " failed", 723 oid); 724 error = EIO; 725 goto done; 726 } 727 728 if (comp.status != RNDIS_STATUS_SUCCESS) { 729 PMD_DRV_LOG(ERR, 730 "RNDIS set %#" PRIx32 " failed: status %#" PRIx32, 731 oid, comp.status); 732 error = EIO; 733 goto done; 734 } 735 736 done: 737 rte_free(req); 738 return error; 739 } 740 741 int hn_rndis_conf_offload(struct hn_data *hv, 742 uint64_t tx_offloads, uint64_t rx_offloads) 743 { 744 struct ndis_offload_params params; 745 struct ndis_offload hwcaps; 746 int error; 747 748 error = hn_rndis_query_hwcaps(hv, &hwcaps); 749 if (error) { 750 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 751 return error; 752 } 753 754 /* NOTE: 0 means "no change" */ 755 memset(¶ms, 0, sizeof(params)); 756 757 params.ndis_hdr.ndis_type = NDIS_OBJTYPE_DEFAULT; 758 if (hv->ndis_ver < NDIS_VERSION_6_30) { 759 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_2; 760 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE_6_1; 761 } else { 762 params.ndis_hdr.ndis_rev = NDIS_OFFLOAD_PARAMS_REV_3; 763 params.ndis_hdr.ndis_size = NDIS_OFFLOAD_PARAMS_SIZE; 764 } 765 766 if (tx_offloads & DEV_TX_OFFLOAD_TCP_CKSUM) { 767 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_TCP4) 768 params.ndis_tcp4csum = NDIS_OFFLOAD_PARAM_TX; 769 else 770 goto unsupported; 771 772 if (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_TCP6) 773 params.ndis_tcp6csum = NDIS_OFFLOAD_PARAM_TX; 774 else 775 goto unsupported; 776 } 777 778 if (rx_offloads & DEV_RX_OFFLOAD_TCP_CKSUM) { 779 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) 780 == NDIS_RXCSUM_CAP_TCP4) 781 params.ndis_tcp4csum |= NDIS_OFFLOAD_PARAM_RX; 782 else 783 goto unsupported; 784 785 if ((hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6) 786 == NDIS_RXCSUM_CAP_TCP6) 787 params.ndis_tcp6csum |= NDIS_OFFLOAD_PARAM_RX; 788 else 789 goto unsupported; 790 } 791 792 if (tx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) { 793 if (hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) 794 params.ndis_udp4csum = NDIS_OFFLOAD_PARAM_TX; 795 else 796 goto unsupported; 797 798 if ((hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6) 799 == NDIS_TXCSUM_CAP_UDP6) 800 params.ndis_udp6csum = NDIS_OFFLOAD_PARAM_TX; 801 else 802 goto unsupported; 803 } 804 805 if (rx_offloads & DEV_TX_OFFLOAD_UDP_CKSUM) { 806 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) 807 params.ndis_udp4csum |= NDIS_OFFLOAD_PARAM_RX; 808 else 809 goto unsupported; 810 811 if (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6) 812 params.ndis_udp6csum |= NDIS_OFFLOAD_PARAM_RX; 813 else 814 goto unsupported; 815 } 816 817 if (tx_offloads & DEV_TX_OFFLOAD_IPV4_CKSUM) { 818 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_IP4) 819 == NDIS_TXCSUM_CAP_IP4) 820 params.ndis_ip4csum = NDIS_OFFLOAD_PARAM_TX; 821 else 822 goto unsupported; 823 } 824 if (rx_offloads & DEV_RX_OFFLOAD_IPV4_CKSUM) { 825 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 826 params.ndis_ip4csum |= NDIS_OFFLOAD_PARAM_RX; 827 else 828 goto unsupported; 829 } 830 831 if (tx_offloads & DEV_TX_OFFLOAD_TCP_TSO) { 832 if (hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) 833 params.ndis_lsov2_ip4 = NDIS_OFFLOAD_LSOV2_ON; 834 else 835 goto unsupported; 836 837 if ((hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6) 838 == HN_NDIS_LSOV2_CAP_IP6) 839 params.ndis_lsov2_ip6 = NDIS_OFFLOAD_LSOV2_ON; 840 else 841 goto unsupported; 842 } 843 844 error = hn_rndis_set(hv, OID_TCP_OFFLOAD_PARAMETERS, ¶ms, 845 params.ndis_hdr.ndis_size); 846 if (error) { 847 PMD_DRV_LOG(ERR, "offload config failed"); 848 return error; 849 } 850 851 return 0; 852 unsupported: 853 PMD_DRV_LOG(NOTICE, 854 "offload tx:%" PRIx64 " rx:%" PRIx64 " not supported by this version", 855 tx_offloads, rx_offloads); 856 return -EINVAL; 857 } 858 859 int hn_rndis_get_offload(struct hn_data *hv, 860 struct rte_eth_dev_info *dev_info) 861 { 862 struct ndis_offload hwcaps; 863 int error; 864 865 memset(&hwcaps, 0, sizeof(hwcaps)); 866 867 error = hn_rndis_query_hwcaps(hv, &hwcaps); 868 if (error) { 869 PMD_DRV_LOG(ERR, "hwcaps query failed: %d", error); 870 return error; 871 } 872 873 dev_info->tx_offload_capa = DEV_TX_OFFLOAD_MULTI_SEGS | 874 DEV_TX_OFFLOAD_VLAN_INSERT; 875 876 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_IP4) 877 == HN_NDIS_TXCSUM_CAP_IP4) 878 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_IPV4_CKSUM; 879 880 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & HN_NDIS_TXCSUM_CAP_TCP4) 881 == HN_NDIS_TXCSUM_CAP_TCP4 && 882 (hwcaps.ndis_csum.ndis_ip6_txcsum & HN_NDIS_TXCSUM_CAP_TCP6) 883 == HN_NDIS_TXCSUM_CAP_TCP6) 884 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_CKSUM; 885 886 if ((hwcaps.ndis_csum.ndis_ip4_txcsum & NDIS_TXCSUM_CAP_UDP4) && 887 (hwcaps.ndis_csum.ndis_ip6_txcsum & NDIS_TXCSUM_CAP_UDP6)) 888 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_UDP_CKSUM; 889 890 if ((hwcaps.ndis_lsov2.ndis_ip4_encap & NDIS_OFFLOAD_ENCAP_8023) && 891 (hwcaps.ndis_lsov2.ndis_ip6_opts & HN_NDIS_LSOV2_CAP_IP6) 892 == HN_NDIS_LSOV2_CAP_IP6) 893 dev_info->tx_offload_capa |= DEV_TX_OFFLOAD_TCP_TSO; 894 895 dev_info->rx_offload_capa = DEV_RX_OFFLOAD_VLAN_STRIP | 896 DEV_RX_OFFLOAD_CRC_STRIP; 897 898 if (hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_IP4) 899 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_IPV4_CKSUM; 900 901 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_TCP4) && 902 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_TCP6)) 903 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_TCP_CKSUM; 904 905 if ((hwcaps.ndis_csum.ndis_ip4_rxcsum & NDIS_RXCSUM_CAP_UDP4) && 906 (hwcaps.ndis_csum.ndis_ip6_rxcsum & NDIS_RXCSUM_CAP_UDP6)) 907 dev_info->rx_offload_capa |= DEV_RX_OFFLOAD_UDP_CKSUM; 908 909 return 0; 910 } 911 912 int 913 hn_rndis_set_rxfilter(struct hn_data *hv, uint32_t filter) 914 { 915 int error; 916 917 error = hn_rndis_set(hv, OID_GEN_CURRENT_PACKET_FILTER, 918 &filter, sizeof(filter)); 919 if (error) { 920 PMD_DRV_LOG(ERR, "set RX filter %#" PRIx32 " failed: %d", 921 filter, error); 922 } else { 923 PMD_DRV_LOG(DEBUG, "set RX filter %#" PRIx32 " done", filter); 924 } 925 926 return error; 927 } 928 929 /* The default RSS key. 930 * This value is the same as MLX5 so that flows will be 931 * received on same path for both VF ans synthetic NIC. 932 */ 933 static const uint8_t rss_default_key[NDIS_HASH_KEYSIZE_TOEPLITZ] = { 934 0x2c, 0xc6, 0x81, 0xd1, 0x5b, 0xdb, 0xf4, 0xf7, 935 0xfc, 0xa2, 0x83, 0x19, 0xdb, 0x1a, 0x3e, 0x94, 936 0x6b, 0x9e, 0x38, 0xd9, 0x2c, 0x9c, 0x03, 0xd1, 937 0xad, 0x99, 0x44, 0xa7, 0xd9, 0x56, 0x3d, 0x59, 938 0x06, 0x3c, 0x25, 0xf3, 0xfc, 0x1f, 0xdc, 0x2a, 939 }; 940 941 int hn_rndis_conf_rss(struct hn_data *hv, 942 const struct rte_eth_rss_conf *rss_conf) 943 { 944 struct ndis_rssprm_toeplitz rssp; 945 struct ndis_rss_params *prm = &rssp.rss_params; 946 const uint8_t *rss_key = rss_conf->rss_key ? : rss_default_key; 947 uint32_t rss_hash; 948 unsigned int i; 949 int error; 950 951 PMD_INIT_FUNC_TRACE(); 952 953 memset(&rssp, 0, sizeof(rssp)); 954 955 prm->ndis_hdr.ndis_type = NDIS_OBJTYPE_RSS_PARAMS; 956 prm->ndis_hdr.ndis_rev = NDIS_RSS_PARAMS_REV_2; 957 prm->ndis_hdr.ndis_size = sizeof(*prm); 958 prm->ndis_flags = 0; 959 960 rss_hash = NDIS_HASH_FUNCTION_TOEPLITZ; 961 if (rss_conf->rss_hf & ETH_RSS_IPV4) 962 rss_hash |= NDIS_HASH_IPV4; 963 if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV4_TCP) 964 rss_hash |= NDIS_HASH_TCP_IPV4; 965 if (rss_conf->rss_hf & ETH_RSS_IPV6) 966 rss_hash |= NDIS_HASH_IPV6; 967 if (rss_conf->rss_hf & ETH_RSS_NONFRAG_IPV6_TCP) 968 rss_hash |= NDIS_HASH_TCP_IPV6; 969 970 prm->ndis_hash = rss_hash; 971 prm->ndis_indsize = sizeof(rssp.rss_ind[0]) * NDIS_HASH_INDCNT; 972 prm->ndis_indoffset = offsetof(struct ndis_rssprm_toeplitz, rss_ind[0]); 973 prm->ndis_keysize = NDIS_HASH_KEYSIZE_TOEPLITZ; 974 prm->ndis_keyoffset = offsetof(struct ndis_rssprm_toeplitz, rss_key[0]); 975 976 for (i = 0; i < NDIS_HASH_INDCNT; i++) 977 rssp.rss_ind[i] = i % hv->num_queues; 978 979 /* Set hask key values */ 980 memcpy(&rssp.rss_key, rss_key, NDIS_HASH_KEYSIZE_TOEPLITZ); 981 982 error = hn_rndis_set(hv, OID_GEN_RECEIVE_SCALE_PARAMETERS, 983 &rssp, sizeof(rssp)); 984 if (error) { 985 PMD_DRV_LOG(ERR, 986 "RSS config num queues=%u failed: %d", 987 hv->num_queues, error); 988 } 989 return error; 990 } 991 992 static int hn_rndis_init(struct hn_data *hv) 993 { 994 struct rndis_init_req *req; 995 struct rndis_init_comp comp; 996 uint32_t comp_len, rid; 997 int error; 998 999 req = hn_rndis_alloc(hv, sizeof(*req)); 1000 if (!req) { 1001 PMD_DRV_LOG(ERR, "no memory for RNDIS init"); 1002 return -ENXIO; 1003 } 1004 1005 rid = hn_rndis_rid(hv); 1006 req->type = RNDIS_INITIALIZE_MSG; 1007 req->len = sizeof(*req); 1008 req->rid = rid; 1009 req->ver_major = RNDIS_VERSION_MAJOR; 1010 req->ver_minor = RNDIS_VERSION_MINOR; 1011 req->max_xfersz = HN_RNDIS_XFER_SIZE; 1012 1013 comp_len = RNDIS_INIT_COMP_SIZE_MIN; 1014 error = hn_rndis_execute(hv, rid, req, sizeof(*req), 1015 &comp, comp_len, 1016 RNDIS_INITIALIZE_CMPLT); 1017 if (error) 1018 goto done; 1019 1020 if (comp.status != RNDIS_STATUS_SUCCESS) { 1021 PMD_DRV_LOG(ERR, "RNDIS init failed: status 0x%08x", 1022 comp.status); 1023 error = -EIO; 1024 goto done; 1025 } 1026 1027 hv->rndis_agg_size = comp.pktmaxsz; 1028 hv->rndis_agg_pkts = comp.pktmaxcnt; 1029 hv->rndis_agg_align = 1U << comp.align; 1030 1031 if (hv->rndis_agg_align < sizeof(uint32_t)) { 1032 /* 1033 * The RNDIS packet message encap assumes that the RNDIS 1034 * packet message is at least 4 bytes aligned. Fix up the 1035 * alignment here, if the remote side sets the alignment 1036 * too low. 1037 */ 1038 PMD_DRV_LOG(NOTICE, 1039 "fixup RNDIS aggpkt align: %u -> %zu", 1040 hv->rndis_agg_align, sizeof(uint32_t)); 1041 hv->rndis_agg_align = sizeof(uint32_t); 1042 } 1043 1044 PMD_INIT_LOG(INFO, 1045 "RNDIS ver %u.%u, aggpkt size %u, aggpkt cnt %u, aggpkt align %u", 1046 comp.ver_major, comp.ver_minor, 1047 hv->rndis_agg_size, hv->rndis_agg_pkts, 1048 hv->rndis_agg_align); 1049 error = 0; 1050 done: 1051 rte_free(req); 1052 return error; 1053 } 1054 1055 int 1056 hn_rndis_get_eaddr(struct hn_data *hv, uint8_t *eaddr) 1057 { 1058 uint32_t eaddr_len; 1059 int error; 1060 1061 eaddr_len = ETHER_ADDR_LEN; 1062 error = hn_rndis_query(hv, OID_802_3_PERMANENT_ADDRESS, NULL, 0, 1063 eaddr, eaddr_len); 1064 if (error) 1065 return error; 1066 1067 PMD_DRV_LOG(INFO, "MAC address %02x:%02x:%02x:%02x:%02x:%02x", 1068 eaddr[0], eaddr[1], eaddr[2], 1069 eaddr[3], eaddr[4], eaddr[5]); 1070 return 0; 1071 } 1072 1073 int 1074 hn_rndis_get_linkstatus(struct hn_data *hv) 1075 { 1076 return hn_rndis_query(hv, OID_GEN_MEDIA_CONNECT_STATUS, NULL, 0, 1077 &hv->link_status, sizeof(uint32_t)); 1078 } 1079 1080 int 1081 hn_rndis_get_linkspeed(struct hn_data *hv) 1082 { 1083 return hn_rndis_query(hv, OID_GEN_LINK_SPEED, NULL, 0, 1084 &hv->link_speed, sizeof(uint32_t)); 1085 } 1086 1087 int 1088 hn_rndis_attach(struct hn_data *hv) 1089 { 1090 /* Initialize RNDIS. */ 1091 return hn_rndis_init(hv); 1092 } 1093 1094 void 1095 hn_rndis_detach(struct hn_data *hv) 1096 { 1097 /* Halt the RNDIS. */ 1098 hn_rndis_halt(hv); 1099 } 1100