1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2018 Microsoft Corp. 3 * Copyright (c) 2010-2012 Citrix Inc. 4 * Copyright (c) 2012 NetApp Inc. 5 * All rights reserved. 6 */ 7 8 /* 9 * Network Virtualization Service. 10 */ 11 12 13 #include <stdint.h> 14 #include <string.h> 15 #include <stdio.h> 16 #include <errno.h> 17 #include <unistd.h> 18 19 #include <rte_ethdev.h> 20 #include <rte_string_fns.h> 21 #include <rte_memzone.h> 22 #include <rte_malloc.h> 23 #include <rte_atomic.h> 24 #include <rte_branch_prediction.h> 25 #include <rte_ether.h> 26 #include <rte_common.h> 27 #include <rte_errno.h> 28 #include <rte_cycles.h> 29 #include <rte_memory.h> 30 #include <rte_eal.h> 31 #include <rte_dev.h> 32 #include <rte_bus_vmbus.h> 33 34 #include "hn_logs.h" 35 #include "hn_var.h" 36 #include "hn_nvs.h" 37 38 static const uint32_t hn_nvs_version[] = { 39 NVS_VERSION_61, 40 NVS_VERSION_6, 41 NVS_VERSION_5, 42 NVS_VERSION_4, 43 NVS_VERSION_2, 44 NVS_VERSION_1 45 }; 46 47 static int hn_nvs_req_send(struct hn_data *hv, 48 void *req, uint32_t reqlen) 49 { 50 return rte_vmbus_chan_send(hn_primary_chan(hv), 51 VMBUS_CHANPKT_TYPE_INBAND, 52 req, reqlen, 0, 53 VMBUS_CHANPKT_FLAG_NONE, NULL); 54 } 55 56 static int 57 __hn_nvs_execute(struct hn_data *hv, 58 void *req, uint32_t reqlen, 59 void *resp, uint32_t resplen, 60 uint32_t type) 61 { 62 struct vmbus_channel *chan = hn_primary_chan(hv); 63 char buffer[NVS_RESPSIZE_MAX]; 64 const struct hn_nvs_hdr *hdr; 65 uint64_t xactid; 66 uint32_t len; 67 int ret; 68 69 /* Send request to ring buffer */ 70 ret = rte_vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_INBAND, 71 req, reqlen, 0, 72 VMBUS_CHANPKT_FLAG_RC, NULL); 73 74 if (ret) { 75 PMD_DRV_LOG(ERR, "send request failed: %d", ret); 76 return ret; 77 } 78 79 retry: 80 len = sizeof(buffer); 81 ret = rte_vmbus_chan_recv(chan, buffer, &len, &xactid); 82 if (ret == -EAGAIN) { 83 rte_delay_us(HN_CHAN_INTERVAL_US); 84 goto retry; 85 } 86 87 if (ret < 0) { 88 PMD_DRV_LOG(ERR, "recv response failed: %d", ret); 89 return ret; 90 } 91 92 if (len < sizeof(*hdr)) { 93 PMD_DRV_LOG(ERR, "response missing NVS header"); 94 return -EINVAL; 95 } 96 97 hdr = (struct hn_nvs_hdr *)buffer; 98 99 /* Silently drop received packets while waiting for response */ 100 switch (hdr->type) { 101 case NVS_TYPE_RNDIS: 102 hn_nvs_ack_rxbuf(chan, xactid); 103 /* fallthrough */ 104 105 case NVS_TYPE_TXTBL_NOTE: 106 PMD_DRV_LOG(DEBUG, "discard packet type 0x%x", hdr->type); 107 goto retry; 108 } 109 110 if (hdr->type != type) { 111 PMD_DRV_LOG(ERR, "unexpected NVS resp %#x, expect %#x", 112 hdr->type, type); 113 return -EINVAL; 114 } 115 116 if (len < resplen) { 117 PMD_DRV_LOG(ERR, 118 "invalid NVS resp len %u (expect %u)", 119 len, resplen); 120 return -EINVAL; 121 } 122 123 memcpy(resp, buffer, resplen); 124 125 /* All pass! */ 126 return 0; 127 } 128 129 130 /* 131 * Execute one control command and get the response. 132 * Only one command can be active on a channel at once 133 * Unlike BSD, DPDK does not have an interrupt context 134 * so the polling is required to wait for response. 135 */ 136 static int 137 hn_nvs_execute(struct hn_data *hv, 138 void *req, uint32_t reqlen, 139 void *resp, uint32_t resplen, 140 uint32_t type) 141 { 142 struct hn_rx_queue *rxq = hv->primary; 143 int ret; 144 145 rte_spinlock_lock(&rxq->ring_lock); 146 ret = __hn_nvs_execute(hv, req, reqlen, resp, resplen, type); 147 rte_spinlock_unlock(&rxq->ring_lock); 148 149 return ret; 150 } 151 152 static int 153 hn_nvs_doinit(struct hn_data *hv, uint32_t nvs_ver) 154 { 155 struct hn_nvs_init init; 156 struct hn_nvs_init_resp resp; 157 uint32_t status; 158 int error; 159 160 memset(&init, 0, sizeof(init)); 161 init.type = NVS_TYPE_INIT; 162 init.ver_min = nvs_ver; 163 init.ver_max = nvs_ver; 164 165 error = hn_nvs_execute(hv, &init, sizeof(init), 166 &resp, sizeof(resp), 167 NVS_TYPE_INIT_RESP); 168 if (error) 169 return error; 170 171 status = resp.status; 172 if (status != NVS_STATUS_OK) { 173 /* Not fatal, try other versions */ 174 PMD_INIT_LOG(DEBUG, "nvs init failed for ver 0x%x", 175 nvs_ver); 176 return -EINVAL; 177 } 178 179 return 0; 180 } 181 182 static int 183 hn_nvs_conn_rxbuf(struct hn_data *hv) 184 { 185 struct hn_nvs_rxbuf_conn conn; 186 struct hn_nvs_rxbuf_connresp resp; 187 uint32_t status; 188 int error; 189 190 /* Kernel has already setup RXBUF on primary channel. */ 191 192 /* 193 * Connect RXBUF to NVS. 194 */ 195 conn.type = NVS_TYPE_RXBUF_CONN; 196 conn.gpadl = hv->rxbuf_res->phys_addr; 197 conn.sig = NVS_RXBUF_SIG; 198 PMD_DRV_LOG(DEBUG, "connect rxbuff va=%p gpad=%#" PRIx64, 199 hv->rxbuf_res->addr, 200 hv->rxbuf_res->phys_addr); 201 202 error = hn_nvs_execute(hv, &conn, sizeof(conn), 203 &resp, sizeof(resp), 204 NVS_TYPE_RXBUF_CONNRESP); 205 if (error) { 206 PMD_DRV_LOG(ERR, 207 "exec nvs rxbuf conn failed: %d", 208 error); 209 return error; 210 } 211 212 status = resp.status; 213 if (status != NVS_STATUS_OK) { 214 PMD_DRV_LOG(ERR, 215 "nvs rxbuf conn failed: %x", status); 216 return -EIO; 217 } 218 if (resp.nsect != 1) { 219 PMD_DRV_LOG(ERR, 220 "nvs rxbuf response num sections %u != 1", 221 resp.nsect); 222 return -EIO; 223 } 224 225 PMD_DRV_LOG(INFO, 226 "receive buffer size %u count %u", 227 resp.nvs_sect[0].slotsz, 228 resp.nvs_sect[0].slotcnt); 229 hv->rxbuf_section_cnt = resp.nvs_sect[0].slotcnt; 230 231 /* 232 * Pimary queue's rxbuf_info is not allocated at creation time. 233 * Now we can allocate it after we figure out the slotcnt. 234 */ 235 hv->primary->rxbuf_info = rte_calloc("HN_RXBUF_INFO", 236 hv->rxbuf_section_cnt, 237 sizeof(*hv->primary->rxbuf_info), 238 RTE_CACHE_LINE_SIZE); 239 if (!hv->primary->rxbuf_info) { 240 PMD_DRV_LOG(ERR, 241 "could not allocate rxbuf info"); 242 return -ENOMEM; 243 } 244 245 return 0; 246 } 247 248 static void 249 hn_nvs_disconn_rxbuf(struct hn_data *hv) 250 { 251 struct hn_nvs_rxbuf_disconn disconn; 252 int error; 253 254 /* 255 * Disconnect RXBUF from NVS. 256 */ 257 memset(&disconn, 0, sizeof(disconn)); 258 disconn.type = NVS_TYPE_RXBUF_DISCONN; 259 disconn.sig = NVS_RXBUF_SIG; 260 261 /* NOTE: No response. */ 262 error = hn_nvs_req_send(hv, &disconn, sizeof(disconn)); 263 if (error) { 264 PMD_DRV_LOG(ERR, 265 "send nvs rxbuf disconn failed: %d", 266 error); 267 } 268 269 /* 270 * Linger long enough for NVS to disconnect RXBUF. 271 */ 272 rte_delay_ms(200); 273 } 274 275 static void 276 hn_nvs_disconn_chim(struct hn_data *hv) 277 { 278 int error; 279 280 if (hv->chim_cnt != 0) { 281 struct hn_nvs_chim_disconn disconn; 282 283 /* Disconnect chimney sending buffer from NVS. */ 284 memset(&disconn, 0, sizeof(disconn)); 285 disconn.type = NVS_TYPE_CHIM_DISCONN; 286 disconn.sig = NVS_CHIM_SIG; 287 288 /* NOTE: No response. */ 289 error = hn_nvs_req_send(hv, &disconn, sizeof(disconn)); 290 291 if (error) { 292 PMD_DRV_LOG(ERR, 293 "send nvs chim disconn failed: %d", error); 294 } 295 296 hv->chim_cnt = 0; 297 /* 298 * Linger long enough for NVS to disconnect chimney 299 * sending buffer. 300 */ 301 rte_delay_ms(200); 302 } 303 } 304 305 static int 306 hn_nvs_conn_chim(struct hn_data *hv) 307 { 308 struct hn_nvs_chim_conn chim; 309 struct hn_nvs_chim_connresp resp; 310 uint32_t sectsz; 311 unsigned long len = hv->chim_res->len; 312 int error; 313 314 /* Connect chimney sending buffer to NVS */ 315 memset(&chim, 0, sizeof(chim)); 316 chim.type = NVS_TYPE_CHIM_CONN; 317 chim.gpadl = hv->chim_res->phys_addr; 318 chim.sig = NVS_CHIM_SIG; 319 PMD_DRV_LOG(DEBUG, "connect send buf va=%p gpad=%#" PRIx64, 320 hv->chim_res->addr, 321 hv->chim_res->phys_addr); 322 323 error = hn_nvs_execute(hv, &chim, sizeof(chim), 324 &resp, sizeof(resp), 325 NVS_TYPE_CHIM_CONNRESP); 326 if (error) { 327 PMD_DRV_LOG(ERR, "exec nvs chim conn failed"); 328 return error; 329 } 330 331 if (resp.status != NVS_STATUS_OK) { 332 PMD_DRV_LOG(ERR, "nvs chim conn failed: %x", 333 resp.status); 334 return -EIO; 335 } 336 337 sectsz = resp.sectsz; 338 if (sectsz == 0 || sectsz & (sizeof(uint32_t) - 1)) { 339 /* Can't use chimney sending buffer; done! */ 340 PMD_DRV_LOG(NOTICE, 341 "invalid chimney sending buffer section size: %u", 342 sectsz); 343 error = -EINVAL; 344 goto cleanup; 345 } 346 347 hv->chim_szmax = sectsz; 348 hv->chim_cnt = len / sectsz; 349 350 PMD_DRV_LOG(INFO, "send buffer %lu section size:%u, count:%u", 351 len, hv->chim_szmax, hv->chim_cnt); 352 353 /* Done! */ 354 return 0; 355 356 cleanup: 357 hn_nvs_disconn_chim(hv); 358 return error; 359 } 360 361 /* 362 * Configure MTU and enable VLAN. 363 */ 364 static int 365 hn_nvs_conf_ndis(struct hn_data *hv, unsigned int mtu) 366 { 367 struct hn_nvs_ndis_conf conf; 368 int error; 369 370 memset(&conf, 0, sizeof(conf)); 371 conf.type = NVS_TYPE_NDIS_CONF; 372 conf.mtu = mtu + RTE_ETHER_HDR_LEN; 373 conf.caps = NVS_NDIS_CONF_VLAN; 374 375 /* enable SRIOV */ 376 if (hv->nvs_ver >= NVS_VERSION_5) 377 conf.caps |= NVS_NDIS_CONF_SRIOV; 378 379 /* NOTE: No response. */ 380 error = hn_nvs_req_send(hv, &conf, sizeof(conf)); 381 if (error) { 382 PMD_DRV_LOG(ERR, 383 "send nvs ndis conf failed: %d", error); 384 return error; 385 } 386 387 return 0; 388 } 389 390 static int 391 hn_nvs_init_ndis(struct hn_data *hv) 392 { 393 struct hn_nvs_ndis_init ndis; 394 int error; 395 396 memset(&ndis, 0, sizeof(ndis)); 397 ndis.type = NVS_TYPE_NDIS_INIT; 398 ndis.ndis_major = NDIS_VERSION_MAJOR(hv->ndis_ver); 399 ndis.ndis_minor = NDIS_VERSION_MINOR(hv->ndis_ver); 400 401 /* NOTE: No response. */ 402 error = hn_nvs_req_send(hv, &ndis, sizeof(ndis)); 403 if (error) 404 PMD_DRV_LOG(ERR, 405 "send nvs ndis init failed: %d", error); 406 407 return error; 408 } 409 410 static int 411 hn_nvs_init(struct hn_data *hv) 412 { 413 unsigned int i; 414 int error; 415 416 /* 417 * Find the supported NVS version and set NDIS version accordingly. 418 */ 419 for (i = 0; i < RTE_DIM(hn_nvs_version); ++i) { 420 error = hn_nvs_doinit(hv, hn_nvs_version[i]); 421 if (error) { 422 PMD_INIT_LOG(DEBUG, "version %#x error %d", 423 hn_nvs_version[i], error); 424 continue; 425 } 426 427 hv->nvs_ver = hn_nvs_version[i]; 428 429 /* Set NDIS version according to NVS version. */ 430 hv->ndis_ver = NDIS_VERSION_6_30; 431 if (hv->nvs_ver <= NVS_VERSION_4) 432 hv->ndis_ver = NDIS_VERSION_6_1; 433 434 PMD_INIT_LOG(DEBUG, 435 "NVS version %#x, NDIS version %u.%u", 436 hv->nvs_ver, NDIS_VERSION_MAJOR(hv->ndis_ver), 437 NDIS_VERSION_MINOR(hv->ndis_ver)); 438 return 0; 439 } 440 441 PMD_DRV_LOG(ERR, 442 "no NVS compatible version available"); 443 return -ENXIO; 444 } 445 446 int 447 hn_nvs_attach(struct hn_data *hv, unsigned int mtu) 448 { 449 int error; 450 451 /* 452 * Initialize NVS. 453 */ 454 error = hn_nvs_init(hv); 455 if (error) 456 return error; 457 458 /** Configure NDIS before initializing it. */ 459 if (hv->nvs_ver >= NVS_VERSION_2) { 460 error = hn_nvs_conf_ndis(hv, mtu); 461 if (error) 462 return error; 463 } 464 465 /* 466 * Initialize NDIS. 467 */ 468 error = hn_nvs_init_ndis(hv); 469 if (error) 470 return error; 471 472 /* 473 * Connect RXBUF. 474 */ 475 error = hn_nvs_conn_rxbuf(hv); 476 if (error) 477 return error; 478 479 /* 480 * Connect chimney sending buffer. 481 */ 482 error = hn_nvs_conn_chim(hv); 483 if (error) { 484 hn_nvs_disconn_rxbuf(hv); 485 return error; 486 } 487 488 return 0; 489 } 490 491 void 492 hn_nvs_detach(struct hn_data *hv __rte_unused) 493 { 494 PMD_INIT_FUNC_TRACE(); 495 496 /* NOTE: there are no requests to stop the NVS. */ 497 hn_nvs_disconn_rxbuf(hv); 498 hn_nvs_disconn_chim(hv); 499 } 500 501 /* 502 * Ack the consumed RXBUF associated w/ this channel packet, 503 * so that this RXBUF can be recycled by the hypervisor. 504 */ 505 void 506 hn_nvs_ack_rxbuf(struct vmbus_channel *chan, uint64_t tid) 507 { 508 unsigned int retries = 0; 509 struct hn_nvs_rndis_ack ack = { 510 .type = NVS_TYPE_RNDIS_ACK, 511 .status = NVS_STATUS_OK, 512 }; 513 int error; 514 515 PMD_RX_LOG(DEBUG, "ack RX id %" PRIu64, tid); 516 517 again: 518 error = rte_vmbus_chan_send(chan, VMBUS_CHANPKT_TYPE_COMP, 519 &ack, sizeof(ack), tid, 520 VMBUS_CHANPKT_FLAG_NONE, NULL); 521 522 if (error == 0) 523 return; 524 525 if (error == -EAGAIN) { 526 /* 527 * NOTE: 528 * This should _not_ happen in real world, since the 529 * consumption of the TX bufring from the TX path is 530 * controlled. 531 */ 532 PMD_RX_LOG(NOTICE, "RXBUF ack retry"); 533 if (++retries < 10) { 534 rte_delay_ms(1); 535 goto again; 536 } 537 } 538 /* RXBUF leaks! */ 539 PMD_DRV_LOG(ERR, "RXBUF ack failed"); 540 } 541 542 int 543 hn_nvs_alloc_subchans(struct hn_data *hv, uint32_t *nsubch) 544 { 545 struct hn_nvs_subch_req req; 546 struct hn_nvs_subch_resp resp; 547 int error; 548 549 memset(&req, 0, sizeof(req)); 550 req.type = NVS_TYPE_SUBCH_REQ; 551 req.op = NVS_SUBCH_OP_ALLOC; 552 req.nsubch = *nsubch; 553 554 error = hn_nvs_execute(hv, &req, sizeof(req), 555 &resp, sizeof(resp), 556 NVS_TYPE_SUBCH_RESP); 557 if (error) 558 return error; 559 560 if (resp.status != NVS_STATUS_OK) { 561 PMD_INIT_LOG(ERR, 562 "nvs subch alloc failed: %#x", 563 resp.status); 564 return -EIO; 565 } 566 567 if (resp.nsubch > *nsubch) { 568 PMD_INIT_LOG(NOTICE, 569 "%u subchans are allocated, requested %u", 570 resp.nsubch, *nsubch); 571 } 572 *nsubch = resp.nsubch; 573 574 return 0; 575 } 576 577 int 578 hn_nvs_set_datapath(struct hn_data *hv, uint32_t path) 579 { 580 struct hn_nvs_datapath dp; 581 int error; 582 583 PMD_DRV_LOG(DEBUG, "set datapath %s", 584 path ? "VF" : "Synthetic"); 585 586 memset(&dp, 0, sizeof(dp)); 587 dp.type = NVS_TYPE_SET_DATAPATH; 588 dp.active_path = path; 589 590 error = hn_nvs_req_send(hv, &dp, sizeof(dp)); 591 if (error) { 592 PMD_DRV_LOG(ERR, 593 "send set datapath failed: %d", 594 error); 595 } 596 597 return error; 598 } 599