1 /* $NetBSD: vmbus.c,v 1.13 2021/04/24 23:36:54 thorpej Exp $ */ 2 /* $OpenBSD: hyperv.c,v 1.43 2017/06/27 13:56:15 mikeb Exp $ */ 3 4 /*- 5 * Copyright (c) 2009-2012 Microsoft Corp. 6 * Copyright (c) 2012 NetApp Inc. 7 * Copyright (c) 2012 Citrix Inc. 8 * Copyright (c) 2016 Mike Belopuhov <mike@esdenera.com> 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice unmodified, this list of conditions, and the following 16 * disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 22 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 23 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 24 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 26 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 30 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * The OpenBSD port was done under funding by Esdenera Networks GmbH. 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: vmbus.c,v 1.13 2021/04/24 23:36:54 thorpej Exp $"); 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/device.h> 43 #include <sys/atomic.h> 44 #include <sys/bitops.h> 45 #include <sys/bus.h> 46 #include <sys/cpu.h> 47 #include <sys/intr.h> 48 #include <sys/kmem.h> 49 #include <sys/kthread.h> 50 #include <sys/module.h> 51 #include <sys/mutex.h> 52 #include <sys/xcall.h> 53 54 #include <uvm/uvm_extern.h> 55 56 #include <dev/hyperv/vmbusvar.h> 57 58 #define VMBUS_GPADL_START 0xffff /* 0x10000 effectively */ 59 60 /* Command submission flags */ 61 #define HCF_SLEEPOK 0x0000 62 #define HCF_NOSLEEP 0x0002 /* M_NOWAIT */ 63 #define HCF_NOREPLY 0x0004 64 65 static void vmbus_attach_deferred(device_t); 66 static int vmbus_attach_print(void *, const char *); 67 static int vmbus_alloc_dma(struct vmbus_softc *); 68 static void vmbus_free_dma(struct vmbus_softc *); 69 static int vmbus_init_interrupts(struct vmbus_softc *); 70 static void vmbus_deinit_interrupts(struct vmbus_softc *); 71 static void vmbus_init_synic(void *, void *); 72 static void vmbus_deinit_synic(void *, void *); 73 74 static int vmbus_connect(struct vmbus_softc *); 75 static int vmbus_cmd(struct vmbus_softc *, void *, size_t, void *, size_t, 76 int); 77 static int vmbus_start(struct vmbus_softc *, struct vmbus_msg *, paddr_t); 78 static int vmbus_reply(struct vmbus_softc *, struct vmbus_msg *); 79 static uint16_t vmbus_intr_signal(struct vmbus_softc *, paddr_t); 80 static void vmbus_event_proc(void *, struct cpu_info *); 81 static void vmbus_event_proc_compat(void *, struct cpu_info *); 82 static void vmbus_message_proc(void *, struct cpu_info *); 83 static void vmbus_message_softintr(void *); 84 static void vmbus_channel_response(struct vmbus_softc *, 85 struct vmbus_chanmsg_hdr *); 86 static void vmbus_channel_offer(struct vmbus_softc *, 87 struct vmbus_chanmsg_hdr *); 88 static void vmbus_channel_rescind(struct vmbus_softc *, 89 struct vmbus_chanmsg_hdr *); 90 static void vmbus_channel_delivered(struct vmbus_softc *, 91 struct vmbus_chanmsg_hdr *); 92 static int vmbus_channel_scan(struct vmbus_softc *); 93 static void vmbus_channel_cpu_default(struct vmbus_channel *); 94 static void vmbus_process_offer(struct vmbus_softc *, 95 struct vmbus_chanmsg_choffer *); 96 static void vmbus_process_rescind(struct vmbus_softc *, 97 struct vmbus_chanmsg_chrescind *); 98 static struct vmbus_channel * 99 vmbus_channel_lookup(struct vmbus_softc *, uint32_t); 100 static int vmbus_channel_ring_create(struct vmbus_channel *, uint32_t); 101 static void vmbus_channel_ring_destroy(struct vmbus_channel *); 102 static void vmbus_channel_detach(struct vmbus_channel *); 103 static void vmbus_channel_pause(struct vmbus_channel *); 104 static uint32_t vmbus_channel_unpause(struct vmbus_channel *); 105 static uint32_t vmbus_channel_ready(struct vmbus_channel *); 106 static void vmbus_chevq_enqueue(struct vmbus_softc *, int, void *); 107 static void vmbus_process_chevq(void *); 108 static void vmbus_chevq_thread(void *); 109 static void vmbus_devq_enqueue(struct vmbus_softc *, int, 110 struct vmbus_channel *); 111 static void vmbus_process_devq(void *); 112 static void vmbus_devq_thread(void *); 113 static void vmbus_subchannel_devq_thread(void *); 114 115 static struct vmbus_softc *vmbus_sc; 116 117 static const struct { 118 int hmd_response; 119 int hmd_request; 120 void (*hmd_handler)(struct vmbus_softc *, 121 struct vmbus_chanmsg_hdr *); 122 } vmbus_msg_dispatch[] = { 123 { 0, 0, NULL }, 124 { VMBUS_CHANMSG_CHOFFER, 0, vmbus_channel_offer }, 125 { VMBUS_CHANMSG_CHRESCIND, 0, vmbus_channel_rescind }, 126 { VMBUS_CHANMSG_CHREQUEST, VMBUS_CHANMSG_CHOFFER, NULL }, 127 { VMBUS_CHANMSG_CHOFFER_DONE, 0, vmbus_channel_delivered }, 128 { VMBUS_CHANMSG_CHOPEN, 0, NULL }, 129 { VMBUS_CHANMSG_CHOPEN_RESP, VMBUS_CHANMSG_CHOPEN, 130 vmbus_channel_response }, 131 { VMBUS_CHANMSG_CHCLOSE, 0, NULL }, 132 { VMBUS_CHANMSG_GPADL_CONN, 0, NULL }, 133 { VMBUS_CHANMSG_GPADL_SUBCONN, 0, NULL }, 134 { VMBUS_CHANMSG_GPADL_CONNRESP, VMBUS_CHANMSG_GPADL_CONN, 135 vmbus_channel_response }, 136 { VMBUS_CHANMSG_GPADL_DISCONN, 0, NULL }, 137 { VMBUS_CHANMSG_GPADL_DISCONNRESP, VMBUS_CHANMSG_GPADL_DISCONN, 138 vmbus_channel_response }, 139 { VMBUS_CHANMSG_CHFREE, 0, NULL }, 140 { VMBUS_CHANMSG_CONNECT, 0, NULL }, 141 { VMBUS_CHANMSG_CONNECT_RESP, VMBUS_CHANMSG_CONNECT, 142 vmbus_channel_response }, 143 { VMBUS_CHANMSG_DISCONNECT, 0, NULL }, 144 }; 145 146 const struct hyperv_guid hyperv_guid_network = { 147 { 0x63, 0x51, 0x61, 0xf8, 0x3e, 0xdf, 0xc5, 0x46, 148 0x91, 0x3f, 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0x0e } 149 }; 150 151 const struct hyperv_guid hyperv_guid_ide = { 152 { 0x32, 0x26, 0x41, 0x32, 0xcb, 0x86, 0xa2, 0x44, 153 0x9b, 0x5c, 0x50, 0xd1, 0x41, 0x73, 0x54, 0xf5 } 154 }; 155 156 const struct hyperv_guid hyperv_guid_scsi = { 157 { 0xd9, 0x63, 0x61, 0xba, 0xa1, 0x04, 0x29, 0x4d, 158 0xb6, 0x05, 0x72, 0xe2, 0xff, 0xb1, 0xdc, 0x7f } 159 }; 160 161 const struct hyperv_guid hyperv_guid_shutdown = { 162 { 0x31, 0x60, 0x0b, 0x0e, 0x13, 0x52, 0x34, 0x49, 163 0x81, 0x8b, 0x38, 0xd9, 0x0c, 0xed, 0x39, 0xdb } 164 }; 165 166 const struct hyperv_guid hyperv_guid_timesync = { 167 { 0x30, 0xe6, 0x27, 0x95, 0xae, 0xd0, 0x7b, 0x49, 168 0xad, 0xce, 0xe8, 0x0a, 0xb0, 0x17, 0x5c, 0xaf } 169 }; 170 171 const struct hyperv_guid hyperv_guid_heartbeat = { 172 { 0x39, 0x4f, 0x16, 0x57, 0x15, 0x91, 0x78, 0x4e, 173 0xab, 0x55, 0x38, 0x2f, 0x3b, 0xd5, 0x42, 0x2d } 174 }; 175 176 const struct hyperv_guid hyperv_guid_kvp = { 177 { 0xe7, 0xf4, 0xa0, 0xa9, 0x45, 0x5a, 0x96, 0x4d, 178 0xb8, 0x27, 0x8a, 0x84, 0x1e, 0x8c, 0x03, 0xe6 } 179 }; 180 181 const struct hyperv_guid hyperv_guid_vss = { 182 { 0x29, 0x2e, 0xfa, 0x35, 0x23, 0xea, 0x36, 0x42, 183 0x96, 0xae, 0x3a, 0x6e, 0xba, 0xcb, 0xa4, 0x40 } 184 }; 185 186 const struct hyperv_guid hyperv_guid_dynmem = { 187 { 0xdc, 0x74, 0x50, 0x52, 0x85, 0x89, 0xe2, 0x46, 188 0x80, 0x57, 0xa3, 0x07, 0xdc, 0x18, 0xa5, 0x02 } 189 }; 190 191 const struct hyperv_guid hyperv_guid_mouse = { 192 { 0x9e, 0xb6, 0xa8, 0xcf, 0x4a, 0x5b, 0xc0, 0x4c, 193 0xb9, 0x8b, 0x8b, 0xa1, 0xa1, 0xf3, 0xf9, 0x5a } 194 }; 195 196 const struct hyperv_guid hyperv_guid_kbd = { 197 { 0x6d, 0xad, 0x12, 0xf9, 0x17, 0x2b, 0xea, 0x48, 198 0xbd, 0x65, 0xf9, 0x27, 0xa6, 0x1c, 0x76, 0x84 } 199 }; 200 201 const struct hyperv_guid hyperv_guid_video = { 202 { 0x02, 0x78, 0x0a, 0xda, 0x77, 0xe3, 0xac, 0x4a, 203 0x8e, 0x77, 0x05, 0x58, 0xeb, 0x10, 0x73, 0xf8 } 204 }; 205 206 const struct hyperv_guid hyperv_guid_fc = { 207 { 0x4a, 0xcc, 0x9b, 0x2f, 0x69, 0x00, 0xf3, 0x4a, 208 0xb7, 0x6b, 0x6f, 0xd0, 0xbe, 0x52, 0x8c, 0xda } 209 }; 210 211 const struct hyperv_guid hyperv_guid_fcopy = { 212 { 0xe3, 0x4b, 0xd1, 0x34, 0xe4, 0xde, 0xc8, 0x41, 213 0x9a, 0xe7, 0x6b, 0x17, 0x49, 0x77, 0xc1, 0x92 } 214 }; 215 216 const struct hyperv_guid hyperv_guid_pcie = { 217 { 0x1d, 0xf6, 0xc4, 0x44, 0x44, 0x44, 0x00, 0x44, 218 0x9d, 0x52, 0x80, 0x2e, 0x27, 0xed, 0xe1, 0x9f } 219 }; 220 221 const struct hyperv_guid hyperv_guid_netdir = { 222 { 0x3d, 0xaf, 0x2e, 0x8c, 0xa7, 0x32, 0x09, 0x4b, 223 0xab, 0x99, 0xbd, 0x1f, 0x1c, 0x86, 0xb5, 0x01 } 224 }; 225 226 const struct hyperv_guid hyperv_guid_rdesktop = { 227 { 0xf4, 0xac, 0x6a, 0x27, 0x15, 0xac, 0x6c, 0x42, 228 0x98, 0xdd, 0x75, 0x21, 0xad, 0x3f, 0x01, 0xfe } 229 }; 230 231 /* Automatic Virtual Machine Activation (AVMA) Services */ 232 const struct hyperv_guid hyperv_guid_avma1 = { 233 { 0x55, 0xb2, 0x87, 0x44, 0x8c, 0xb8, 0x3f, 0x40, 234 0xbb, 0x51, 0xd1, 0xf6, 0x9c, 0xf1, 0x7f, 0x87 } 235 }; 236 237 const struct hyperv_guid hyperv_guid_avma2 = { 238 { 0xf4, 0xba, 0x75, 0x33, 0x15, 0x9e, 0x30, 0x4b, 239 0xb7, 0x65, 0x67, 0xac, 0xb1, 0x0d, 0x60, 0x7b } 240 }; 241 242 const struct hyperv_guid hyperv_guid_avma3 = { 243 { 0xa0, 0x1f, 0x22, 0x99, 0xad, 0x24, 0xe2, 0x11, 244 0xbe, 0x98, 0x00, 0x1a, 0xa0, 0x1b, 0xbf, 0x6e } 245 }; 246 247 const struct hyperv_guid hyperv_guid_avma4 = { 248 { 0x16, 0x57, 0xe6, 0xf8, 0xb3, 0x3c, 0x06, 0x4a, 249 0x9a, 0x60, 0x18, 0x89, 0xc5, 0xcc, 0xca, 0xb5 } 250 }; 251 252 int 253 vmbus_match(device_t parent, cfdata_t cf, void *aux) 254 { 255 256 if (cf->cf_unit != 0 || 257 !hyperv_hypercall_enabled() || 258 !hyperv_synic_supported()) 259 return 0; 260 261 return 1; 262 } 263 264 int 265 vmbus_attach(struct vmbus_softc *sc) 266 { 267 268 aprint_naive("\n"); 269 aprint_normal(": Hyper-V VMBus\n"); 270 271 vmbus_sc = sc; 272 273 sc->sc_msgpool = pool_cache_init(sizeof(struct vmbus_msg), 8, 0, 0, 274 "hvmsg", NULL, IPL_NET, NULL, NULL, NULL); 275 hyperv_set_message_proc(vmbus_message_proc, sc); 276 277 if (vmbus_alloc_dma(sc)) 278 goto cleanup; 279 280 if (vmbus_init_interrupts(sc)) 281 goto cleanup; 282 283 if (vmbus_connect(sc)) 284 goto cleanup; 285 286 aprint_normal_dev(sc->sc_dev, "protocol %d.%d\n", 287 VMBUS_VERSION_MAJOR(sc->sc_proto), 288 VMBUS_VERSION_MINOR(sc->sc_proto)); 289 290 if (sc->sc_proto == VMBUS_VERSION_WS2008 || 291 sc->sc_proto == VMBUS_VERSION_WIN7) { 292 hyperv_set_event_proc(vmbus_event_proc_compat, sc); 293 sc->sc_channel_max = VMBUS_CHAN_MAX_COMPAT; 294 } else { 295 hyperv_set_event_proc(vmbus_event_proc, sc); 296 sc->sc_channel_max = VMBUS_CHAN_MAX; 297 } 298 299 if (vmbus_channel_scan(sc)) 300 goto cleanup; 301 302 config_interrupts(sc->sc_dev, vmbus_attach_deferred); 303 304 return 0; 305 306 cleanup: 307 vmbus_deinit_interrupts(sc); 308 vmbus_free_dma(sc); 309 return -1; 310 } 311 312 static void 313 vmbus_attach_deferred(device_t self) 314 { 315 struct vmbus_softc *sc = device_private(self); 316 317 xc_wait(xc_broadcast(0, vmbus_init_synic, sc, NULL)); 318 } 319 320 int 321 vmbus_detach(struct vmbus_softc *sc, int flags) 322 { 323 324 vmbus_deinit_interrupts(sc); 325 vmbus_free_dma(sc); 326 327 return 0; 328 } 329 330 static int 331 vmbus_alloc_dma(struct vmbus_softc *sc) 332 { 333 CPU_INFO_ITERATOR cii; 334 struct cpu_info *ci; 335 struct vmbus_percpu_data *pd; 336 int i; 337 338 /* 339 * Per-CPU messages and event flags. 340 */ 341 for (CPU_INFO_FOREACH(cii, ci)) { 342 pd = &sc->sc_percpu[cpu_index(ci)]; 343 344 pd->simp = hyperv_dma_alloc(sc->sc_dmat, &pd->simp_dma, 345 PAGE_SIZE, PAGE_SIZE, 0, 1, HYPERV_DMA_SLEEPOK); 346 if (pd->simp == NULL) 347 return ENOMEM; 348 349 pd->siep = hyperv_dma_alloc(sc->sc_dmat, &pd->siep_dma, 350 PAGE_SIZE, PAGE_SIZE, 0, 1, HYPERV_DMA_SLEEPOK); 351 if (pd->siep == NULL) 352 return ENOMEM; 353 } 354 355 sc->sc_events = hyperv_dma_alloc(sc->sc_dmat, &sc->sc_events_dma, 356 PAGE_SIZE, PAGE_SIZE, 0, 1, HYPERV_DMA_SLEEPOK); 357 if (sc->sc_events == NULL) 358 return ENOMEM; 359 sc->sc_wevents = (u_long *)sc->sc_events; 360 sc->sc_revents = (u_long *)((uint8_t *)sc->sc_events + (PAGE_SIZE / 2)); 361 362 for (i = 0; i < __arraycount(sc->sc_monitor); i++) { 363 sc->sc_monitor[i] = hyperv_dma_alloc(sc->sc_dmat, 364 &sc->sc_monitor_dma[i], PAGE_SIZE, PAGE_SIZE, 0, 1, 365 HYPERV_DMA_SLEEPOK); 366 if (sc->sc_monitor[i] == NULL) 367 return ENOMEM; 368 } 369 370 return 0; 371 } 372 373 static void 374 vmbus_free_dma(struct vmbus_softc *sc) 375 { 376 CPU_INFO_ITERATOR cii; 377 struct cpu_info *ci; 378 int i; 379 380 if (sc->sc_events != NULL) { 381 sc->sc_events = sc->sc_wevents = sc->sc_revents = NULL; 382 hyperv_dma_free(sc->sc_dmat, &sc->sc_events_dma); 383 } 384 385 for (i = 0; i < __arraycount(sc->sc_monitor); i++) { 386 sc->sc_monitor[i] = NULL; 387 hyperv_dma_free(sc->sc_dmat, &sc->sc_monitor_dma[i]); 388 } 389 390 for (CPU_INFO_FOREACH(cii, ci)) { 391 struct vmbus_percpu_data *pd = &sc->sc_percpu[cpu_index(ci)]; 392 393 if (pd->simp != NULL) { 394 pd->simp = NULL; 395 hyperv_dma_free(sc->sc_dmat, &pd->simp_dma); 396 } 397 if (pd->siep != NULL) { 398 pd->siep = NULL; 399 hyperv_dma_free(sc->sc_dmat, &pd->siep_dma); 400 } 401 } 402 } 403 404 static int 405 vmbus_init_interrupts(struct vmbus_softc *sc) 406 { 407 408 TAILQ_INIT(&sc->sc_reqs); 409 mutex_init(&sc->sc_req_lock, MUTEX_DEFAULT, IPL_NET); 410 411 TAILQ_INIT(&sc->sc_rsps); 412 mutex_init(&sc->sc_rsp_lock, MUTEX_DEFAULT, IPL_NET); 413 414 sc->sc_proto = VMBUS_VERSION_WS2008; 415 416 /* XXX event_tq */ 417 418 sc->sc_msg_sih = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 419 vmbus_message_softintr, sc); 420 if (sc->sc_msg_sih == NULL) 421 return -1; 422 423 vmbus_init_interrupts_md(sc); 424 425 kcpuset_create(&sc->sc_intr_cpuset, true); 426 if (cold) { 427 /* Initialize other CPUs later. */ 428 vmbus_init_synic(sc, NULL); 429 } else 430 xc_wait(xc_broadcast(0, vmbus_init_synic, sc, NULL)); 431 atomic_or_32(&sc->sc_flags, VMBUS_SCFLAG_SYNIC); 432 433 return 0; 434 } 435 436 static void 437 vmbus_deinit_interrupts(struct vmbus_softc *sc) 438 { 439 440 if (ISSET(sc->sc_flags, VMBUS_SCFLAG_SYNIC)) { 441 if (cold) 442 vmbus_deinit_synic(sc, NULL); 443 else 444 xc_wait(xc_broadcast(0, vmbus_deinit_synic, sc, NULL)); 445 atomic_and_32(&sc->sc_flags, (uint32_t)~VMBUS_SCFLAG_SYNIC); 446 } 447 448 /* XXX event_tq */ 449 450 if (sc->sc_msg_sih != NULL) { 451 softint_disestablish(sc->sc_msg_sih); 452 sc->sc_msg_sih = NULL; 453 } 454 455 vmbus_deinit_interrupts_md(sc); 456 } 457 458 static void 459 vmbus_init_synic(void *arg1, void *arg2) 460 { 461 struct vmbus_softc *sc = arg1; 462 cpuid_t cpu; 463 int s; 464 465 s = splhigh(); 466 467 cpu = cpu_index(curcpu()); 468 if (!kcpuset_isset(sc->sc_intr_cpuset, cpu)) { 469 kcpuset_atomic_set(sc->sc_intr_cpuset, cpu); 470 vmbus_init_synic_md(sc, cpu); 471 } 472 473 splx(s); 474 } 475 476 static void 477 vmbus_deinit_synic(void *arg1, void *arg2) 478 { 479 struct vmbus_softc *sc = arg1; 480 cpuid_t cpu; 481 int s; 482 483 s = splhigh(); 484 485 cpu = cpu_index(curcpu()); 486 if (kcpuset_isset(sc->sc_intr_cpuset, cpu)) { 487 vmbus_deinit_synic_md(sc, cpu); 488 kcpuset_atomic_clear(sc->sc_intr_cpuset, cpu); 489 } 490 491 splx(s); 492 } 493 494 static int 495 vmbus_connect(struct vmbus_softc *sc) 496 { 497 static const uint32_t versions[] = { 498 VMBUS_VERSION_WIN8_1, 499 VMBUS_VERSION_WIN8, 500 VMBUS_VERSION_WIN7, 501 VMBUS_VERSION_WS2008 502 }; 503 struct vmbus_chanmsg_connect cmd; 504 struct vmbus_chanmsg_connect_resp rsp; 505 int i, rv; 506 507 memset(&cmd, 0, sizeof(cmd)); 508 cmd.chm_hdr.chm_type = VMBUS_CHANMSG_CONNECT; 509 cmd.chm_evtflags = hyperv_dma_get_paddr(&sc->sc_events_dma); 510 cmd.chm_mnf1 = hyperv_dma_get_paddr(&sc->sc_monitor_dma[0]); 511 cmd.chm_mnf2 = hyperv_dma_get_paddr(&sc->sc_monitor_dma[1]); 512 513 memset(&rsp, 0, sizeof(rsp)); 514 515 for (i = 0; i < __arraycount(versions); i++) { 516 cmd.chm_ver = versions[i]; 517 rv = vmbus_cmd(sc, &cmd, sizeof(cmd), &rsp, sizeof(rsp), 518 cold ? HCF_NOSLEEP : HCF_SLEEPOK); 519 if (rv) { 520 DPRINTF("%s: CONNECT failed\n", 521 device_xname(sc->sc_dev)); 522 return rv; 523 } 524 if (rsp.chm_done) { 525 atomic_or_32(&sc->sc_flags, VMBUS_SCFLAG_CONNECTED); 526 sc->sc_proto = versions[i]; 527 sc->sc_handle = VMBUS_GPADL_START; 528 break; 529 } 530 } 531 if (i == __arraycount(versions)) { 532 device_printf(sc->sc_dev, 533 "failed to negotiate protocol version\n"); 534 return ENXIO; 535 } 536 537 return 0; 538 } 539 540 static int 541 vmbus_cmd(struct vmbus_softc *sc, void *cmd, size_t cmdlen, void *rsp, 542 size_t rsplen, int flags) 543 { 544 const int prflags = cold ? PR_NOWAIT : PR_WAITOK; 545 struct vmbus_msg *msg; 546 paddr_t pa; 547 int rv; 548 549 if (cmdlen > VMBUS_MSG_DSIZE_MAX) { 550 device_printf(sc->sc_dev, "payload too large (%zu)\n", 551 cmdlen); 552 return EMSGSIZE; 553 } 554 555 msg = pool_cache_get_paddr(sc->sc_msgpool, prflags, &pa); 556 if (msg == NULL) { 557 device_printf(sc->sc_dev, "couldn't get msgpool\n"); 558 return ENOMEM; 559 } 560 memset(msg, 0, sizeof(*msg)); 561 msg->msg_req.hc_dsize = cmdlen; 562 memcpy(msg->msg_req.hc_data, cmd, cmdlen); 563 564 if (!(flags & HCF_NOREPLY)) { 565 msg->msg_rsp = rsp; 566 msg->msg_rsplen = rsplen; 567 } else 568 msg->msg_flags |= MSGF_NOQUEUE; 569 570 if (flags & HCF_NOSLEEP) 571 msg->msg_flags |= MSGF_NOSLEEP; 572 573 rv = vmbus_start(sc, msg, pa); 574 if (rv == 0) 575 rv = vmbus_reply(sc, msg); 576 pool_cache_put_paddr(sc->sc_msgpool, msg, pa); 577 return rv; 578 } 579 580 static int 581 vmbus_start(struct vmbus_softc *sc, struct vmbus_msg *msg, paddr_t msg_pa) 582 { 583 static const int delays[] = { 584 100, 100, 100, 500, 500, 5000, 5000, 5000 585 }; 586 const char *wchan = "hvstart"; 587 uint16_t status; 588 int i, s; 589 590 msg->msg_req.hc_connid = VMBUS_CONNID_MESSAGE; 591 msg->msg_req.hc_msgtype = 1; 592 593 if (!(msg->msg_flags & MSGF_NOQUEUE)) { 594 mutex_enter(&sc->sc_req_lock); 595 TAILQ_INSERT_TAIL(&sc->sc_reqs, msg, msg_entry); 596 mutex_exit(&sc->sc_req_lock); 597 } 598 599 for (i = 0; i < __arraycount(delays); i++) { 600 status = hyperv_hypercall_post_message( 601 msg_pa + offsetof(struct vmbus_msg, msg_req)); 602 if (status == HYPERCALL_STATUS_SUCCESS) 603 break; 604 605 if (msg->msg_flags & MSGF_NOSLEEP) { 606 delay(delays[i]); 607 s = splnet(); 608 hyperv_intr(); 609 splx(s); 610 } else 611 tsleep(wchan, PRIBIO, wchan, 612 uimax(1, mstohz(delays[i] / 1000))); 613 } 614 if (status != HYPERCALL_STATUS_SUCCESS) { 615 device_printf(sc->sc_dev, 616 "posting vmbus message failed with %d\n", status); 617 if (!(msg->msg_flags & MSGF_NOQUEUE)) { 618 mutex_enter(&sc->sc_req_lock); 619 TAILQ_REMOVE(&sc->sc_reqs, msg, msg_entry); 620 mutex_exit(&sc->sc_req_lock); 621 } 622 return EIO; 623 } 624 625 return 0; 626 } 627 628 static int 629 vmbus_reply_done(struct vmbus_softc *sc, struct vmbus_msg *msg) 630 { 631 struct vmbus_msg *m; 632 633 mutex_enter(&sc->sc_rsp_lock); 634 TAILQ_FOREACH(m, &sc->sc_rsps, msg_entry) { 635 if (m == msg) { 636 mutex_exit(&sc->sc_rsp_lock); 637 return 1; 638 } 639 } 640 mutex_exit(&sc->sc_rsp_lock); 641 return 0; 642 } 643 644 static int 645 vmbus_reply(struct vmbus_softc *sc, struct vmbus_msg *msg) 646 { 647 int s; 648 649 if (msg->msg_flags & MSGF_NOQUEUE) 650 return 0; 651 652 while (!vmbus_reply_done(sc, msg)) { 653 if (msg->msg_flags & MSGF_NOSLEEP) { 654 delay(1000); 655 s = splnet(); 656 hyperv_intr(); 657 splx(s); 658 } else 659 tsleep(msg, PRIBIO, "hvreply", 1); 660 } 661 662 mutex_enter(&sc->sc_rsp_lock); 663 TAILQ_REMOVE(&sc->sc_rsps, msg, msg_entry); 664 mutex_exit(&sc->sc_rsp_lock); 665 666 return 0; 667 } 668 669 static uint16_t 670 vmbus_intr_signal(struct vmbus_softc *sc, paddr_t con_pa) 671 { 672 uint64_t status; 673 674 status = hyperv_hypercall_signal_event(con_pa); 675 return (uint16_t)status; 676 } 677 678 #if LONG_BIT == 64 679 #define ffsl(v) ffs64(v) 680 #elif LONG_BIT == 32 681 #define ffsl(v) ffs32(v) 682 #else 683 #error unsupport LONG_BIT 684 #endif /* LONG_BIT */ 685 686 static void 687 vmbus_event_flags_proc(struct vmbus_softc *sc, volatile u_long *revents, 688 int maxrow) 689 { 690 struct vmbus_channel *ch; 691 u_long pending; 692 uint32_t chanid, chanid_base; 693 int row, chanid_ofs; 694 695 for (row = 0; row < maxrow; row++) { 696 if (revents[row] == 0) 697 continue; 698 699 pending = atomic_swap_ulong(&revents[row], 0); 700 chanid_base = row * LONG_BIT; 701 702 while ((chanid_ofs = ffsl(pending)) != 0) { 703 chanid_ofs--; /* NOTE: ffs is 1-based */ 704 pending &= ~(1UL << chanid_ofs); 705 706 chanid = chanid_base + chanid_ofs; 707 /* vmbus channel protocol message */ 708 if (chanid == 0) 709 continue; 710 711 ch = vmbus_channel_lookup(sc, chanid); 712 if (ch == NULL) { 713 device_printf(sc->sc_dev, 714 "unhandled event on %d\n", chanid); 715 continue; 716 } 717 if (ch->ch_state != VMBUS_CHANSTATE_OPENED) { 718 device_printf(sc->sc_dev, 719 "channel %d is not active\n", chanid); 720 continue; 721 } 722 ch->ch_evcnt.ev_count++; 723 vmbus_channel_schedule(ch); 724 } 725 } 726 } 727 728 static void 729 vmbus_event_proc(void *arg, struct cpu_info *ci) 730 { 731 struct vmbus_softc *sc = arg; 732 struct vmbus_evtflags *evt; 733 734 /* 735 * On Host with Win8 or above, the event page can be 736 * checked directly to get the id of the channel 737 * that has the pending interrupt. 738 */ 739 evt = (struct vmbus_evtflags *)sc->sc_percpu[cpu_index(ci)].siep + 740 VMBUS_SINT_MESSAGE; 741 742 vmbus_event_flags_proc(sc, evt->evt_flags, 743 __arraycount(evt->evt_flags)); 744 } 745 746 static void 747 vmbus_event_proc_compat(void *arg, struct cpu_info *ci) 748 { 749 struct vmbus_softc *sc = arg; 750 struct vmbus_evtflags *evt; 751 752 evt = (struct vmbus_evtflags *)sc->sc_percpu[cpu_index(ci)].siep + 753 VMBUS_SINT_MESSAGE; 754 755 if (test_bit(0, &evt->evt_flags[0])) { 756 clear_bit(0, &evt->evt_flags[0]); 757 /* 758 * receive size is 1/2 page and divide that by 4 bytes 759 */ 760 vmbus_event_flags_proc(sc, sc->sc_revents, 761 VMBUS_CHAN_MAX_COMPAT / VMBUS_EVTFLAG_LEN); 762 } 763 } 764 765 static void 766 vmbus_message_proc(void *arg, struct cpu_info *ci) 767 { 768 struct vmbus_softc *sc = arg; 769 struct vmbus_message *msg; 770 771 msg = (struct vmbus_message *)sc->sc_percpu[cpu_index(ci)].simp + 772 VMBUS_SINT_MESSAGE; 773 if (__predict_false(msg->msg_type != HYPERV_MSGTYPE_NONE)) { 774 if (__predict_true(!cold)) 775 softint_schedule_cpu(sc->sc_msg_sih, ci); 776 else 777 vmbus_message_softintr(sc); 778 } 779 } 780 781 static void 782 vmbus_message_softintr(void *arg) 783 { 784 struct vmbus_softc *sc = arg; 785 struct vmbus_message *msg; 786 struct vmbus_chanmsg_hdr *hdr; 787 uint32_t type; 788 cpuid_t cpu; 789 790 cpu = cpu_index(curcpu()); 791 792 for (;;) { 793 msg = (struct vmbus_message *)sc->sc_percpu[cpu].simp + 794 VMBUS_SINT_MESSAGE; 795 if (msg->msg_type == HYPERV_MSGTYPE_NONE) 796 break; 797 798 hdr = (struct vmbus_chanmsg_hdr *)msg->msg_data; 799 type = hdr->chm_type; 800 if (type >= VMBUS_CHANMSG_COUNT) { 801 device_printf(sc->sc_dev, 802 "unhandled message type %u flags %#x\n", type, 803 msg->msg_flags); 804 } else { 805 if (vmbus_msg_dispatch[type].hmd_handler) { 806 vmbus_msg_dispatch[type].hmd_handler(sc, hdr); 807 } else { 808 device_printf(sc->sc_dev, 809 "unhandled message type %u\n", type); 810 } 811 } 812 813 msg->msg_type = HYPERV_MSGTYPE_NONE; 814 membar_sync(); 815 if (msg->msg_flags & VMBUS_MSGFLAG_PENDING) 816 hyperv_send_eom(); 817 } 818 } 819 820 static void 821 vmbus_channel_response(struct vmbus_softc *sc, struct vmbus_chanmsg_hdr *rsphdr) 822 { 823 struct vmbus_msg *msg; 824 struct vmbus_chanmsg_hdr *reqhdr; 825 int req; 826 827 req = vmbus_msg_dispatch[rsphdr->chm_type].hmd_request; 828 mutex_enter(&sc->sc_req_lock); 829 TAILQ_FOREACH(msg, &sc->sc_reqs, msg_entry) { 830 reqhdr = (struct vmbus_chanmsg_hdr *)&msg->msg_req.hc_data; 831 if (reqhdr->chm_type == req) { 832 TAILQ_REMOVE(&sc->sc_reqs, msg, msg_entry); 833 break; 834 } 835 } 836 mutex_exit(&sc->sc_req_lock); 837 if (msg != NULL) { 838 memcpy(msg->msg_rsp, rsphdr, msg->msg_rsplen); 839 mutex_enter(&sc->sc_rsp_lock); 840 TAILQ_INSERT_TAIL(&sc->sc_rsps, msg, msg_entry); 841 mutex_exit(&sc->sc_rsp_lock); 842 wakeup(msg); 843 } 844 } 845 846 static void 847 vmbus_channel_offer(struct vmbus_softc *sc, struct vmbus_chanmsg_hdr *hdr) 848 { 849 struct vmbus_chanmsg_choffer *co; 850 851 co = kmem_intr_alloc(sizeof(*co), KM_NOSLEEP); 852 if (co == NULL) { 853 device_printf(sc->sc_dev, 854 "failed to allocate an offer object\n"); 855 return; 856 } 857 858 memcpy(co, hdr, sizeof(*co)); 859 vmbus_chevq_enqueue(sc, VMBUS_CHEV_TYPE_OFFER, co); 860 } 861 862 static void 863 vmbus_channel_rescind(struct vmbus_softc *sc, struct vmbus_chanmsg_hdr *hdr) 864 { 865 struct vmbus_chanmsg_chrescind *cr; 866 867 cr = kmem_intr_alloc(sizeof(*cr), KM_NOSLEEP); 868 if (cr == NULL) { 869 device_printf(sc->sc_dev, 870 "failed to allocate an rescind object\n"); 871 return; 872 } 873 874 memcpy(cr, hdr, sizeof(*cr)); 875 vmbus_chevq_enqueue(sc, VMBUS_CHEV_TYPE_RESCIND, cr); 876 } 877 878 static void 879 vmbus_channel_delivered(struct vmbus_softc *sc, struct vmbus_chanmsg_hdr *hdr) 880 { 881 882 atomic_or_32(&sc->sc_flags, VMBUS_SCFLAG_OFFERS_DELIVERED); 883 wakeup(&sc->sc_devq); 884 } 885 886 static void 887 hyperv_guid_sprint(struct hyperv_guid *guid, char *str, size_t size) 888 { 889 static const struct { 890 const struct hyperv_guid *guid; 891 const char *ident; 892 } map[] = { 893 { &hyperv_guid_network, "network" }, 894 { &hyperv_guid_ide, "ide" }, 895 { &hyperv_guid_scsi, "scsi" }, 896 { &hyperv_guid_shutdown, "shutdown" }, 897 { &hyperv_guid_timesync, "timesync" }, 898 { &hyperv_guid_heartbeat, "heartbeat" }, 899 { &hyperv_guid_kvp, "kvp" }, 900 { &hyperv_guid_vss, "vss" }, 901 { &hyperv_guid_dynmem, "dynamic-memory" }, 902 { &hyperv_guid_mouse, "mouse" }, 903 { &hyperv_guid_kbd, "keyboard" }, 904 { &hyperv_guid_video, "video" }, 905 { &hyperv_guid_fc, "fiber-channel" }, 906 { &hyperv_guid_fcopy, "file-copy" }, 907 { &hyperv_guid_pcie, "pcie-passthrough" }, 908 { &hyperv_guid_netdir, "network-direct" }, 909 { &hyperv_guid_rdesktop, "remote-desktop" }, 910 { &hyperv_guid_avma1, "avma-1" }, 911 { &hyperv_guid_avma2, "avma-2" }, 912 { &hyperv_guid_avma3, "avma-3" }, 913 { &hyperv_guid_avma4, "avma-4" }, 914 }; 915 int i; 916 917 for (i = 0; i < __arraycount(map); i++) { 918 if (memcmp(guid, map[i].guid, sizeof(*guid)) == 0) { 919 strlcpy(str, map[i].ident, size); 920 return; 921 } 922 } 923 hyperv_guid2str(guid, str, size); 924 } 925 926 static int 927 vmbus_channel_scan(struct vmbus_softc *sc) 928 { 929 struct vmbus_chanmsg_hdr hdr; 930 struct vmbus_chanmsg_choffer rsp; 931 932 TAILQ_INIT(&sc->sc_prichans); 933 mutex_init(&sc->sc_prichan_lock, MUTEX_DEFAULT, IPL_NET); 934 TAILQ_INIT(&sc->sc_channels); 935 mutex_init(&sc->sc_channel_lock, MUTEX_DEFAULT, IPL_NET); 936 937 /* 938 * This queue serializes vmbus channel offer and rescind messages. 939 */ 940 SIMPLEQ_INIT(&sc->sc_chevq); 941 mutex_init(&sc->sc_chevq_lock, MUTEX_DEFAULT, IPL_NET); 942 cv_init(&sc->sc_chevq_cv, "hvchevcv"); 943 if (kthread_create(PRI_NONE, KTHREAD_MPSAFE, NULL, 944 vmbus_chevq_thread, sc, NULL, "hvchevq") != 0) { 945 DPRINTF("%s: failed to create prich chevq thread\n", 946 device_xname(sc->sc_dev)); 947 return -1; 948 } 949 950 /* 951 * This queue serializes vmbus devices' attach and detach 952 * for channel offer and rescind messages. 953 */ 954 SIMPLEQ_INIT(&sc->sc_devq); 955 mutex_init(&sc->sc_devq_lock, MUTEX_DEFAULT, IPL_NET); 956 cv_init(&sc->sc_devq_cv, "hvdevqcv"); 957 if (kthread_create(PRI_NONE, KTHREAD_MPSAFE, NULL, 958 vmbus_devq_thread, sc, NULL, "hvdevq") != 0) { 959 DPRINTF("%s: failed to create prich devq thread\n", 960 device_xname(sc->sc_dev)); 961 return -1; 962 } 963 964 /* 965 * This queue handles sub-channel detach, so that vmbus 966 * device's detach running in sc_devq can drain its sub-channels. 967 */ 968 SIMPLEQ_INIT(&sc->sc_subch_devq); 969 mutex_init(&sc->sc_subch_devq_lock, MUTEX_DEFAULT, IPL_NET); 970 cv_init(&sc->sc_subch_devq_cv, "hvsdvqcv"); 971 if (kthread_create(PRI_NONE, KTHREAD_MPSAFE, NULL, 972 vmbus_subchannel_devq_thread, sc, NULL, "hvsdevq") != 0) { 973 DPRINTF("%s: failed to create subch devq thread\n", 974 device_xname(sc->sc_dev)); 975 return -1; 976 } 977 978 memset(&hdr, 0, sizeof(hdr)); 979 hdr.chm_type = VMBUS_CHANMSG_CHREQUEST; 980 981 if (vmbus_cmd(sc, &hdr, sizeof(hdr), &rsp, sizeof(rsp), 982 HCF_NOREPLY | (cold ? HCF_NOSLEEP : HCF_SLEEPOK))) { 983 DPRINTF("%s: CHREQUEST failed\n", device_xname(sc->sc_dev)); 984 return -1; 985 } 986 987 while (!ISSET(sc->sc_flags, VMBUS_SCFLAG_OFFERS_DELIVERED)) 988 tsleep(&sc->sc_devq, PRIBIO, "hvscan", 1); 989 990 mutex_enter(&sc->sc_chevq_lock); 991 vmbus_process_chevq(sc); 992 mutex_exit(&sc->sc_chevq_lock); 993 mutex_enter(&sc->sc_devq_lock); 994 vmbus_process_devq(sc); 995 mutex_exit(&sc->sc_devq_lock); 996 997 return 0; 998 } 999 1000 static struct vmbus_channel * 1001 vmbus_channel_alloc(struct vmbus_softc *sc) 1002 { 1003 struct vmbus_channel *ch; 1004 1005 ch = kmem_zalloc(sizeof(*ch), KM_SLEEP); 1006 1007 ch->ch_monprm = hyperv_dma_alloc(sc->sc_dmat, &ch->ch_monprm_dma, 1008 sizeof(*ch->ch_monprm), 8, 0, 1, HYPERV_DMA_SLEEPOK); 1009 if (ch->ch_monprm == NULL) { 1010 device_printf(sc->sc_dev, "monprm alloc failed\n"); 1011 kmem_free(ch, sizeof(*ch)); 1012 return NULL; 1013 } 1014 1015 ch->ch_refs = 1; 1016 ch->ch_sc = sc; 1017 mutex_init(&ch->ch_subchannel_lock, MUTEX_DEFAULT, IPL_NET); 1018 TAILQ_INIT(&ch->ch_subchannels); 1019 1020 ch->ch_state = VMBUS_CHANSTATE_CLOSED; 1021 1022 return ch; 1023 } 1024 1025 static void 1026 vmbus_channel_free(struct vmbus_channel *ch) 1027 { 1028 struct vmbus_softc *sc = ch->ch_sc; 1029 1030 KASSERTMSG(TAILQ_EMPTY(&ch->ch_subchannels) && 1031 ch->ch_subchannel_count == 0, "still owns sub-channels"); 1032 KASSERTMSG(ch->ch_state == 0 || ch->ch_state == VMBUS_CHANSTATE_CLOSED, 1033 "free busy channel"); 1034 KASSERTMSG(ch->ch_refs == 0, "channel %u: invalid refcnt %d", 1035 ch->ch_id, ch->ch_refs); 1036 1037 hyperv_dma_free(sc->sc_dmat, &ch->ch_monprm_dma); 1038 mutex_destroy(&ch->ch_subchannel_lock); 1039 /* XXX ch_evcnt */ 1040 if (ch->ch_taskq != NULL) 1041 softint_disestablish(ch->ch_taskq); 1042 kmem_free(ch, sizeof(*ch)); 1043 } 1044 1045 static int 1046 vmbus_channel_add(struct vmbus_channel *nch) 1047 { 1048 struct vmbus_softc *sc = nch->ch_sc; 1049 struct vmbus_channel *ch; 1050 u_int refs __diagused; 1051 1052 if (nch->ch_id == 0) { 1053 device_printf(sc->sc_dev, "got channel 0 offer, discard\n"); 1054 return EINVAL; 1055 } else if (nch->ch_id >= sc->sc_channel_max) { 1056 device_printf(sc->sc_dev, "invalid channel %u offer\n", 1057 nch->ch_id); 1058 return EINVAL; 1059 } 1060 1061 mutex_enter(&sc->sc_prichan_lock); 1062 TAILQ_FOREACH(ch, &sc->sc_prichans, ch_prientry) { 1063 if (!memcmp(&ch->ch_type, &nch->ch_type, sizeof(ch->ch_type)) && 1064 !memcmp(&ch->ch_inst, &nch->ch_inst, sizeof(ch->ch_inst))) 1065 break; 1066 } 1067 if (VMBUS_CHAN_ISPRIMARY(nch)) { 1068 if (ch == NULL) { 1069 TAILQ_INSERT_TAIL(&sc->sc_prichans, nch, ch_prientry); 1070 mutex_exit(&sc->sc_prichan_lock); 1071 goto done; 1072 } else { 1073 mutex_exit(&sc->sc_prichan_lock); 1074 device_printf(sc->sc_dev, 1075 "duplicated primary channel%u\n", nch->ch_id); 1076 return EINVAL; 1077 } 1078 } else { 1079 if (ch == NULL) { 1080 mutex_exit(&sc->sc_prichan_lock); 1081 device_printf(sc->sc_dev, "no primary channel%u\n", 1082 nch->ch_id); 1083 return EINVAL; 1084 } 1085 } 1086 mutex_exit(&sc->sc_prichan_lock); 1087 1088 KASSERT(!VMBUS_CHAN_ISPRIMARY(nch)); 1089 KASSERT(ch != NULL); 1090 1091 refs = atomic_inc_uint_nv(&nch->ch_refs); 1092 KASSERT(refs == 2); 1093 1094 nch->ch_primary_channel = ch; 1095 nch->ch_dev = ch->ch_dev; 1096 1097 mutex_enter(&ch->ch_subchannel_lock); 1098 TAILQ_INSERT_TAIL(&ch->ch_subchannels, nch, ch_subentry); 1099 ch->ch_subchannel_count++; 1100 mutex_exit(&ch->ch_subchannel_lock); 1101 wakeup(ch); 1102 1103 done: 1104 mutex_enter(&sc->sc_channel_lock); 1105 TAILQ_INSERT_TAIL(&sc->sc_channels, nch, ch_entry); 1106 mutex_exit(&sc->sc_channel_lock); 1107 1108 vmbus_channel_cpu_default(nch); 1109 1110 return 0; 1111 } 1112 1113 void 1114 vmbus_channel_cpu_set(struct vmbus_channel *ch, int cpu) 1115 { 1116 struct vmbus_softc *sc = ch->ch_sc; 1117 1118 KASSERTMSG(cpu >= 0 && cpu < ncpu, "invalid cpu %d", cpu); 1119 1120 if (sc->sc_proto == VMBUS_VERSION_WS2008 || 1121 sc->sc_proto == VMBUS_VERSION_WIN7) { 1122 /* Only cpu0 is supported */ 1123 cpu = 0; 1124 } 1125 1126 ch->ch_cpuid = cpu; 1127 ch->ch_vcpu = hyperv_get_vcpuid(cpu); 1128 } 1129 1130 void 1131 vmbus_channel_cpu_rr(struct vmbus_channel *ch) 1132 { 1133 static uint32_t vmbus_channel_nextcpu; 1134 int cpu; 1135 1136 cpu = atomic_inc_32_nv(&vmbus_channel_nextcpu) % ncpu; 1137 vmbus_channel_cpu_set(ch, cpu); 1138 } 1139 1140 static void 1141 vmbus_channel_cpu_default(struct vmbus_channel *ch) 1142 { 1143 1144 /* 1145 * By default, pin the channel to cpu0. Devices having 1146 * special channel-cpu mapping requirement should call 1147 * vmbus_channel_cpu_{set,rr}(). 1148 */ 1149 vmbus_channel_cpu_set(ch, 0); 1150 } 1151 1152 bool 1153 vmbus_channel_is_revoked(struct vmbus_channel *ch) 1154 { 1155 1156 return (ch->ch_flags & CHF_REVOKED) ? true : false; 1157 } 1158 1159 static void 1160 vmbus_process_offer(struct vmbus_softc *sc, struct vmbus_chanmsg_choffer *co) 1161 { 1162 struct vmbus_channel *ch; 1163 1164 ch = vmbus_channel_alloc(sc); 1165 if (ch == NULL) { 1166 device_printf(sc->sc_dev, "allocate channel %u failed\n", 1167 co->chm_chanid); 1168 return; 1169 } 1170 1171 /* 1172 * By default we setup state to enable batched reading. 1173 * A specific service can choose to disable this prior 1174 * to opening the channel. 1175 */ 1176 ch->ch_flags |= CHF_BATCHED; 1177 1178 hyperv_guid_sprint(&co->chm_chtype, ch->ch_ident, 1179 sizeof(ch->ch_ident)); 1180 1181 ch->ch_monprm->mp_connid = VMBUS_CONNID_EVENT; 1182 if (sc->sc_proto > VMBUS_VERSION_WS2008) 1183 ch->ch_monprm->mp_connid = co->chm_connid; 1184 1185 if (co->chm_flags1 & VMBUS_CHOFFER_FLAG1_HASMNF) { 1186 ch->ch_mgroup = co->chm_montrig / VMBUS_MONTRIG_LEN; 1187 ch->ch_mindex = co->chm_montrig % VMBUS_MONTRIG_LEN; 1188 ch->ch_flags |= CHF_MONITOR; 1189 } 1190 1191 ch->ch_id = co->chm_chanid; 1192 ch->ch_subidx = co->chm_subidx; 1193 1194 memcpy(&ch->ch_type, &co->chm_chtype, sizeof(ch->ch_type)); 1195 memcpy(&ch->ch_inst, &co->chm_chinst, sizeof(ch->ch_inst)); 1196 1197 if (vmbus_channel_add(ch) != 0) { 1198 atomic_dec_uint(&ch->ch_refs); 1199 vmbus_channel_free(ch); 1200 return; 1201 } 1202 1203 ch->ch_state = VMBUS_CHANSTATE_OFFERED; 1204 1205 vmbus_devq_enqueue(sc, VMBUS_DEV_TYPE_ATTACH, ch); 1206 1207 #ifdef HYPERV_DEBUG 1208 printf("%s: channel %u: \"%s\"", device_xname(sc->sc_dev), ch->ch_id, 1209 ch->ch_ident); 1210 if (ch->ch_flags & CHF_MONITOR) 1211 printf(", monitor %u\n", co->chm_montrig); 1212 else 1213 printf("\n"); 1214 #endif 1215 } 1216 1217 static void 1218 vmbus_process_rescind(struct vmbus_softc *sc, 1219 struct vmbus_chanmsg_chrescind *cr) 1220 { 1221 struct vmbus_channel *ch; 1222 1223 if (cr->chm_chanid > VMBUS_CHAN_MAX) { 1224 device_printf(sc->sc_dev, "invalid revoked channel%u\n", 1225 cr->chm_chanid); 1226 return; 1227 } 1228 1229 mutex_enter(&sc->sc_channel_lock); 1230 ch = vmbus_channel_lookup(sc, cr->chm_chanid); 1231 if (ch == NULL) { 1232 mutex_exit(&sc->sc_channel_lock); 1233 device_printf(sc->sc_dev, "channel%u is not offered\n", 1234 cr->chm_chanid); 1235 return; 1236 } 1237 TAILQ_REMOVE(&sc->sc_channels, ch, ch_entry); 1238 mutex_exit(&sc->sc_channel_lock); 1239 1240 if (VMBUS_CHAN_ISPRIMARY(ch)) { 1241 mutex_enter(&sc->sc_prichan_lock); 1242 TAILQ_REMOVE(&sc->sc_prichans, ch, ch_prientry); 1243 mutex_exit(&sc->sc_prichan_lock); 1244 } 1245 1246 KASSERTMSG(!(ch->ch_flags & CHF_REVOKED), 1247 "channel%u has already been revoked", ch->ch_id); 1248 atomic_or_uint(&ch->ch_flags, CHF_REVOKED); 1249 1250 vmbus_channel_detach(ch); 1251 } 1252 1253 static int 1254 vmbus_channel_release(struct vmbus_channel *ch) 1255 { 1256 struct vmbus_softc *sc = ch->ch_sc; 1257 struct vmbus_chanmsg_chfree cmd; 1258 int rv; 1259 1260 memset(&cmd, 0, sizeof(cmd)); 1261 cmd.chm_hdr.chm_type = VMBUS_CHANMSG_CHFREE; 1262 cmd.chm_chanid = ch->ch_id; 1263 1264 rv = vmbus_cmd(sc, &cmd, sizeof(cmd), NULL, 0, 1265 HCF_NOREPLY | (cold ? HCF_NOSLEEP : HCF_SLEEPOK)); 1266 if (rv) { 1267 DPRINTF("%s: CHFREE failed with %d\n", device_xname(sc->sc_dev), 1268 rv); 1269 } 1270 return rv; 1271 } 1272 1273 struct vmbus_channel ** 1274 vmbus_subchannel_get(struct vmbus_channel *prich, int cnt) 1275 { 1276 struct vmbus_softc *sc = prich->ch_sc; 1277 struct vmbus_channel **ret, *ch; 1278 int i, s; 1279 1280 KASSERTMSG(cnt > 0, "invalid sub-channel count %d", cnt); 1281 1282 ret = kmem_zalloc(sizeof(struct vmbus_channel *) * cnt, KM_SLEEP); 1283 1284 mutex_enter(&prich->ch_subchannel_lock); 1285 1286 while (prich->ch_subchannel_count < cnt) { 1287 if (cold) { 1288 mutex_exit(&prich->ch_subchannel_lock); 1289 delay(1000); 1290 s = splnet(); 1291 hyperv_intr(); 1292 splx(s); 1293 mutex_enter(&sc->sc_chevq_lock); 1294 vmbus_process_chevq(sc); 1295 mutex_exit(&sc->sc_chevq_lock); 1296 mutex_enter(&prich->ch_subchannel_lock); 1297 } else { 1298 mtsleep(prich, PRIBIO, "hvsubch", 1, 1299 &prich->ch_subchannel_lock); 1300 } 1301 } 1302 1303 i = 0; 1304 TAILQ_FOREACH(ch, &prich->ch_subchannels, ch_subentry) { 1305 ret[i] = ch; /* XXX inc refs */ 1306 1307 if (++i == cnt) 1308 break; 1309 } 1310 1311 KASSERTMSG(i == cnt, "invalid subchan count %d, should be %d", 1312 prich->ch_subchannel_count, cnt); 1313 1314 mutex_exit(&prich->ch_subchannel_lock); 1315 1316 return ret; 1317 } 1318 1319 void 1320 vmbus_subchannel_put(struct vmbus_channel **subch, int cnt) 1321 { 1322 1323 kmem_free(subch, sizeof(struct vmbus_channel *) * cnt); 1324 } 1325 1326 static struct vmbus_channel * 1327 vmbus_channel_lookup(struct vmbus_softc *sc, uint32_t relid) 1328 { 1329 struct vmbus_channel *ch; 1330 1331 TAILQ_FOREACH(ch, &sc->sc_channels, ch_entry) { 1332 if (ch->ch_id == relid) 1333 return ch; 1334 } 1335 return NULL; 1336 } 1337 1338 static int 1339 vmbus_channel_ring_create(struct vmbus_channel *ch, uint32_t buflen) 1340 { 1341 struct vmbus_softc *sc = ch->ch_sc; 1342 1343 buflen = roundup(buflen, PAGE_SIZE) + sizeof(struct vmbus_bufring); 1344 ch->ch_ring_size = 2 * buflen; 1345 /* page aligned memory */ 1346 ch->ch_ring = hyperv_dma_alloc(sc->sc_dmat, &ch->ch_ring_dma, 1347 ch->ch_ring_size, PAGE_SIZE, 0, 1, HYPERV_DMA_SLEEPOK); 1348 if (ch->ch_ring == NULL) { 1349 device_printf(sc->sc_dev, 1350 "failed to allocate channel ring\n"); 1351 return ENOMEM; 1352 } 1353 1354 memset(&ch->ch_wrd, 0, sizeof(ch->ch_wrd)); 1355 ch->ch_wrd.rd_ring = (struct vmbus_bufring *)ch->ch_ring; 1356 ch->ch_wrd.rd_size = buflen; 1357 ch->ch_wrd.rd_dsize = buflen - sizeof(struct vmbus_bufring); 1358 mutex_init(&ch->ch_wrd.rd_lock, MUTEX_DEFAULT, IPL_NET); 1359 1360 memset(&ch->ch_rrd, 0, sizeof(ch->ch_rrd)); 1361 ch->ch_rrd.rd_ring = (struct vmbus_bufring *)((uint8_t *)ch->ch_ring + 1362 buflen); 1363 ch->ch_rrd.rd_size = buflen; 1364 ch->ch_rrd.rd_dsize = buflen - sizeof(struct vmbus_bufring); 1365 mutex_init(&ch->ch_rrd.rd_lock, MUTEX_DEFAULT, IPL_NET); 1366 1367 if (vmbus_handle_alloc(ch, &ch->ch_ring_dma, ch->ch_ring_size, 1368 &ch->ch_ring_gpadl)) { 1369 device_printf(sc->sc_dev, 1370 "failed to obtain a PA handle for the ring\n"); 1371 vmbus_channel_ring_destroy(ch); 1372 return ENOMEM; 1373 } 1374 1375 return 0; 1376 } 1377 1378 static void 1379 vmbus_channel_ring_destroy(struct vmbus_channel *ch) 1380 { 1381 struct vmbus_softc *sc = ch->ch_sc; 1382 1383 hyperv_dma_free(sc->sc_dmat, &ch->ch_ring_dma); 1384 ch->ch_ring = NULL; 1385 vmbus_handle_free(ch, ch->ch_ring_gpadl); 1386 1387 mutex_destroy(&ch->ch_wrd.rd_lock); 1388 memset(&ch->ch_wrd, 0, sizeof(ch->ch_wrd)); 1389 mutex_destroy(&ch->ch_rrd.rd_lock); 1390 memset(&ch->ch_rrd, 0, sizeof(ch->ch_rrd)); 1391 } 1392 1393 int 1394 vmbus_channel_open(struct vmbus_channel *ch, size_t buflen, void *udata, 1395 size_t udatalen, void (*handler)(void *), void *arg) 1396 { 1397 struct vmbus_softc *sc = ch->ch_sc; 1398 struct vmbus_chanmsg_chopen cmd; 1399 struct vmbus_chanmsg_chopen_resp rsp; 1400 int rv = EINVAL; 1401 1402 if (ch->ch_ring == NULL && 1403 (rv = vmbus_channel_ring_create(ch, buflen))) { 1404 DPRINTF("%s: failed to create channel ring\n", 1405 device_xname(sc->sc_dev)); 1406 return rv; 1407 } 1408 1409 memset(&cmd, 0, sizeof(cmd)); 1410 cmd.chm_hdr.chm_type = VMBUS_CHANMSG_CHOPEN; 1411 cmd.chm_openid = ch->ch_id; 1412 cmd.chm_chanid = ch->ch_id; 1413 cmd.chm_gpadl = ch->ch_ring_gpadl; 1414 cmd.chm_txbr_pgcnt = atop(ch->ch_wrd.rd_size); 1415 cmd.chm_vcpuid = ch->ch_vcpu; 1416 if (udata && udatalen > 0) 1417 memcpy(cmd.chm_udata, udata, udatalen); 1418 1419 memset(&rsp, 0, sizeof(rsp)); 1420 1421 ch->ch_handler = handler; 1422 ch->ch_ctx = arg; 1423 ch->ch_state = VMBUS_CHANSTATE_OPENED; 1424 1425 rv = vmbus_cmd(sc, &cmd, sizeof(cmd), &rsp, sizeof(rsp), 1426 cold ? HCF_NOSLEEP : HCF_SLEEPOK); 1427 if (rv) { 1428 vmbus_channel_ring_destroy(ch); 1429 DPRINTF("%s: CHOPEN failed with %d\n", device_xname(sc->sc_dev), 1430 rv); 1431 ch->ch_handler = NULL; 1432 ch->ch_ctx = NULL; 1433 ch->ch_state = VMBUS_CHANSTATE_OFFERED; 1434 return rv; 1435 } 1436 return 0; 1437 } 1438 1439 static void 1440 vmbus_channel_detach(struct vmbus_channel *ch) 1441 { 1442 u_int refs; 1443 1444 KASSERTMSG(ch->ch_refs > 0, "channel%u: invalid refcnt %d", 1445 ch->ch_id, ch->ch_refs); 1446 1447 refs = atomic_dec_uint_nv(&ch->ch_refs); 1448 if (refs == 0) { 1449 /* Detach the target channel. */ 1450 vmbus_devq_enqueue(ch->ch_sc, VMBUS_DEV_TYPE_DETACH, ch); 1451 } 1452 } 1453 1454 static int 1455 vmbus_channel_close_internal(struct vmbus_channel *ch) 1456 { 1457 struct vmbus_softc *sc = ch->ch_sc; 1458 struct vmbus_chanmsg_chclose cmd; 1459 int rv; 1460 1461 memset(&cmd, 0, sizeof(cmd)); 1462 cmd.chm_hdr.chm_type = VMBUS_CHANMSG_CHCLOSE; 1463 cmd.chm_chanid = ch->ch_id; 1464 1465 ch->ch_state = VMBUS_CHANSTATE_CLOSING; 1466 rv = vmbus_cmd(sc, &cmd, sizeof(cmd), NULL, 0, 1467 HCF_NOREPLY | (cold ? HCF_NOSLEEP : HCF_SLEEPOK)); 1468 if (rv) { 1469 DPRINTF("%s: CHCLOSE failed with %d\n", 1470 device_xname(sc->sc_dev), rv); 1471 return rv; 1472 } 1473 ch->ch_state = VMBUS_CHANSTATE_CLOSED; 1474 vmbus_channel_ring_destroy(ch); 1475 return 0; 1476 } 1477 1478 int 1479 vmbus_channel_close_direct(struct vmbus_channel *ch) 1480 { 1481 int rv; 1482 1483 rv = vmbus_channel_close_internal(ch); 1484 if (!VMBUS_CHAN_ISPRIMARY(ch)) 1485 vmbus_channel_detach(ch); 1486 return rv; 1487 } 1488 1489 int 1490 vmbus_channel_close(struct vmbus_channel *ch) 1491 { 1492 struct vmbus_channel **subch; 1493 int i, cnt, rv; 1494 1495 if (!VMBUS_CHAN_ISPRIMARY(ch)) 1496 return 0; 1497 1498 cnt = ch->ch_subchannel_count; 1499 if (cnt > 0) { 1500 subch = vmbus_subchannel_get(ch, cnt); 1501 for (i = 0; i < ch->ch_subchannel_count; i++) { 1502 rv = vmbus_channel_close_internal(subch[i]); 1503 (void) rv; /* XXX */ 1504 vmbus_channel_detach(ch); 1505 } 1506 vmbus_subchannel_put(subch, cnt); 1507 } 1508 1509 return vmbus_channel_close_internal(ch); 1510 } 1511 1512 static inline void 1513 vmbus_channel_setevent(struct vmbus_softc *sc, struct vmbus_channel *ch) 1514 { 1515 struct vmbus_mon_trig *mtg; 1516 1517 /* Each uint32_t represents 32 channels */ 1518 set_bit(ch->ch_id, sc->sc_wevents); 1519 if (ch->ch_flags & CHF_MONITOR) { 1520 mtg = &sc->sc_monitor[1]->mnf_trigs[ch->ch_mgroup]; 1521 set_bit(ch->ch_mindex, &mtg->mt_pending); 1522 } else 1523 vmbus_intr_signal(sc, hyperv_dma_get_paddr(&ch->ch_monprm_dma)); 1524 } 1525 1526 static void 1527 vmbus_channel_intr(void *arg) 1528 { 1529 struct vmbus_channel *ch = arg; 1530 1531 if (vmbus_channel_ready(ch)) 1532 ch->ch_handler(ch->ch_ctx); 1533 1534 if (vmbus_channel_unpause(ch) == 0) 1535 return; 1536 1537 vmbus_channel_pause(ch); 1538 vmbus_channel_schedule(ch); 1539 } 1540 1541 int 1542 vmbus_channel_setdeferred(struct vmbus_channel *ch, const char *name) 1543 { 1544 1545 ch->ch_taskq = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1546 vmbus_channel_intr, ch); 1547 if (ch->ch_taskq == NULL) 1548 return -1; 1549 return 0; 1550 } 1551 1552 void 1553 vmbus_channel_schedule(struct vmbus_channel *ch) 1554 { 1555 1556 if (ch->ch_handler) { 1557 if (!cold && (ch->ch_flags & CHF_BATCHED)) { 1558 vmbus_channel_pause(ch); 1559 softint_schedule(ch->ch_taskq); 1560 } else 1561 ch->ch_handler(ch->ch_ctx); 1562 } 1563 } 1564 1565 static __inline void 1566 vmbus_ring_put(struct vmbus_ring_data *wrd, uint8_t *data, uint32_t datalen) 1567 { 1568 int left = MIN(datalen, wrd->rd_dsize - wrd->rd_prod); 1569 1570 memcpy(&wrd->rd_ring->br_data[wrd->rd_prod], data, left); 1571 memcpy(&wrd->rd_ring->br_data[0], data + left, datalen - left); 1572 wrd->rd_prod += datalen; 1573 if (wrd->rd_prod >= wrd->rd_dsize) 1574 wrd->rd_prod -= wrd->rd_dsize; 1575 } 1576 1577 static inline void 1578 vmbus_ring_get(struct vmbus_ring_data *rrd, uint8_t *data, uint32_t datalen, 1579 int peek) 1580 { 1581 int left = MIN(datalen, rrd->rd_dsize - rrd->rd_cons); 1582 1583 memcpy(data, &rrd->rd_ring->br_data[rrd->rd_cons], left); 1584 memcpy(data + left, &rrd->rd_ring->br_data[0], datalen - left); 1585 if (!peek) { 1586 rrd->rd_cons += datalen; 1587 if (rrd->rd_cons >= rrd->rd_dsize) 1588 rrd->rd_cons -= rrd->rd_dsize; 1589 } 1590 } 1591 1592 static __inline void 1593 vmbus_ring_avail(struct vmbus_ring_data *rd, uint32_t *towrite, 1594 uint32_t *toread) 1595 { 1596 uint32_t ridx = rd->rd_ring->br_rindex; 1597 uint32_t widx = rd->rd_ring->br_windex; 1598 uint32_t r, w; 1599 1600 if (widx >= ridx) 1601 w = rd->rd_dsize - (widx - ridx); 1602 else 1603 w = ridx - widx; 1604 r = rd->rd_dsize - w; 1605 if (towrite) 1606 *towrite = w; 1607 if (toread) 1608 *toread = r; 1609 } 1610 1611 static int 1612 vmbus_ring_write(struct vmbus_ring_data *wrd, struct iovec *iov, int iov_cnt, 1613 int *needsig) 1614 { 1615 uint64_t indices = 0; 1616 uint32_t avail, oprod, datalen = sizeof(indices); 1617 int i; 1618 1619 for (i = 0; i < iov_cnt; i++) 1620 datalen += iov[i].iov_len; 1621 1622 KASSERT(datalen <= wrd->rd_dsize); 1623 1624 vmbus_ring_avail(wrd, &avail, NULL); 1625 if (avail <= datalen) { 1626 DPRINTF("%s: avail %u datalen %u\n", __func__, avail, datalen); 1627 return EAGAIN; 1628 } 1629 1630 oprod = wrd->rd_prod; 1631 1632 for (i = 0; i < iov_cnt; i++) 1633 vmbus_ring_put(wrd, iov[i].iov_base, iov[i].iov_len); 1634 1635 indices = (uint64_t)oprod << 32; 1636 vmbus_ring_put(wrd, (uint8_t *)&indices, sizeof(indices)); 1637 1638 membar_sync(); 1639 wrd->rd_ring->br_windex = wrd->rd_prod; 1640 membar_sync(); 1641 1642 /* Signal when the ring transitions from being empty to non-empty */ 1643 if (wrd->rd_ring->br_imask == 0 && 1644 wrd->rd_ring->br_rindex == oprod) 1645 *needsig = 1; 1646 else 1647 *needsig = 0; 1648 1649 return 0; 1650 } 1651 1652 int 1653 vmbus_channel_send(struct vmbus_channel *ch, void *data, uint32_t datalen, 1654 uint64_t rid, int type, uint32_t flags) 1655 { 1656 struct vmbus_softc *sc = ch->ch_sc; 1657 struct vmbus_chanpkt cp; 1658 struct iovec iov[3]; 1659 uint32_t pktlen, pktlen_aligned; 1660 uint64_t zeropad = 0; 1661 int rv, needsig = 0; 1662 1663 pktlen = sizeof(cp) + datalen; 1664 pktlen_aligned = roundup(pktlen, sizeof(uint64_t)); 1665 1666 cp.cp_hdr.cph_type = type; 1667 cp.cp_hdr.cph_flags = flags; 1668 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_hlen, sizeof(cp)); 1669 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_tlen, pktlen_aligned); 1670 cp.cp_hdr.cph_tid = rid; 1671 1672 iov[0].iov_base = &cp; 1673 iov[0].iov_len = sizeof(cp); 1674 1675 iov[1].iov_base = data; 1676 iov[1].iov_len = datalen; 1677 1678 iov[2].iov_base = &zeropad; 1679 iov[2].iov_len = pktlen_aligned - pktlen; 1680 1681 mutex_enter(&ch->ch_wrd.rd_lock); 1682 rv = vmbus_ring_write(&ch->ch_wrd, iov, 3, &needsig); 1683 mutex_exit(&ch->ch_wrd.rd_lock); 1684 if (rv == 0 && needsig) 1685 vmbus_channel_setevent(sc, ch); 1686 1687 return rv; 1688 } 1689 1690 int 1691 vmbus_channel_send_sgl(struct vmbus_channel *ch, struct vmbus_gpa *sgl, 1692 uint32_t nsge, void *data, uint32_t datalen, uint64_t rid) 1693 { 1694 struct vmbus_softc *sc = ch->ch_sc; 1695 struct vmbus_chanpkt_sglist cp; 1696 struct iovec iov[4]; 1697 uint32_t buflen, pktlen, pktlen_aligned; 1698 uint64_t zeropad = 0; 1699 int rv, needsig = 0; 1700 1701 buflen = sizeof(struct vmbus_gpa) * nsge; 1702 pktlen = sizeof(cp) + datalen + buflen; 1703 pktlen_aligned = roundup(pktlen, sizeof(uint64_t)); 1704 1705 cp.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1706 cp.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1707 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_hlen, sizeof(cp) + buflen); 1708 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_tlen, pktlen_aligned); 1709 cp.cp_hdr.cph_tid = rid; 1710 cp.cp_gpa_cnt = nsge; 1711 cp.cp_rsvd = 0; 1712 1713 iov[0].iov_base = &cp; 1714 iov[0].iov_len = sizeof(cp); 1715 1716 iov[1].iov_base = sgl; 1717 iov[1].iov_len = buflen; 1718 1719 iov[2].iov_base = data; 1720 iov[2].iov_len = datalen; 1721 1722 iov[3].iov_base = &zeropad; 1723 iov[3].iov_len = pktlen_aligned - pktlen; 1724 1725 mutex_enter(&ch->ch_wrd.rd_lock); 1726 rv = vmbus_ring_write(&ch->ch_wrd, iov, 4, &needsig); 1727 mutex_exit(&ch->ch_wrd.rd_lock); 1728 if (rv == 0 && needsig) 1729 vmbus_channel_setevent(sc, ch); 1730 1731 return rv; 1732 } 1733 1734 int 1735 vmbus_channel_send_prpl(struct vmbus_channel *ch, struct vmbus_gpa_range *prpl, 1736 uint32_t nprp, void *data, uint32_t datalen, uint64_t rid) 1737 { 1738 struct vmbus_softc *sc = ch->ch_sc; 1739 struct vmbus_chanpkt_prplist cp; 1740 struct iovec iov[4]; 1741 uint32_t buflen, pktlen, pktlen_aligned; 1742 uint64_t zeropad = 0; 1743 int rv, needsig = 0; 1744 1745 buflen = sizeof(struct vmbus_gpa_range) * (nprp + 1); 1746 pktlen = sizeof(cp) + datalen + buflen; 1747 pktlen_aligned = roundup(pktlen, sizeof(uint64_t)); 1748 1749 cp.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1750 cp.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1751 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_hlen, sizeof(cp) + buflen); 1752 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_tlen, pktlen_aligned); 1753 cp.cp_hdr.cph_tid = rid; 1754 cp.cp_range_cnt = 1; 1755 cp.cp_rsvd = 0; 1756 1757 iov[0].iov_base = &cp; 1758 iov[0].iov_len = sizeof(cp); 1759 1760 iov[1].iov_base = prpl; 1761 iov[1].iov_len = buflen; 1762 1763 iov[2].iov_base = data; 1764 iov[2].iov_len = datalen; 1765 1766 iov[3].iov_base = &zeropad; 1767 iov[3].iov_len = pktlen_aligned - pktlen; 1768 1769 mutex_enter(&ch->ch_wrd.rd_lock); 1770 rv = vmbus_ring_write(&ch->ch_wrd, iov, 4, &needsig); 1771 mutex_exit(&ch->ch_wrd.rd_lock); 1772 if (rv == 0 && needsig) 1773 vmbus_channel_setevent(sc, ch); 1774 1775 return rv; 1776 } 1777 1778 static int 1779 vmbus_ring_peek(struct vmbus_ring_data *rrd, void *data, uint32_t datalen) 1780 { 1781 uint32_t avail; 1782 1783 KASSERT(datalen <= rrd->rd_dsize); 1784 1785 vmbus_ring_avail(rrd, NULL, &avail); 1786 if (avail < datalen) 1787 return EAGAIN; 1788 1789 vmbus_ring_get(rrd, (uint8_t *)data, datalen, 1); 1790 return 0; 1791 } 1792 1793 static int 1794 vmbus_ring_read(struct vmbus_ring_data *rrd, void *data, uint32_t datalen, 1795 uint32_t offset) 1796 { 1797 uint64_t indices; 1798 uint32_t avail; 1799 1800 KASSERT(datalen <= rrd->rd_dsize); 1801 1802 vmbus_ring_avail(rrd, NULL, &avail); 1803 if (avail < datalen) { 1804 DPRINTF("%s: avail %u datalen %u\n", __func__, avail, datalen); 1805 return EAGAIN; 1806 } 1807 1808 if (offset) { 1809 rrd->rd_cons += offset; 1810 if (rrd->rd_cons >= rrd->rd_dsize) 1811 rrd->rd_cons -= rrd->rd_dsize; 1812 } 1813 1814 vmbus_ring_get(rrd, (uint8_t *)data, datalen, 0); 1815 vmbus_ring_get(rrd, (uint8_t *)&indices, sizeof(indices), 0); 1816 1817 membar_sync(); 1818 rrd->rd_ring->br_rindex = rrd->rd_cons; 1819 1820 return 0; 1821 } 1822 1823 int 1824 vmbus_channel_recv(struct vmbus_channel *ch, void *data, uint32_t datalen, 1825 uint32_t *rlen, uint64_t *rid, int raw) 1826 { 1827 struct vmbus_softc *sc = ch->ch_sc; 1828 struct vmbus_chanpkt_hdr cph; 1829 uint32_t offset, pktlen; 1830 int rv; 1831 1832 *rlen = 0; 1833 1834 mutex_enter(&ch->ch_rrd.rd_lock); 1835 1836 if ((rv = vmbus_ring_peek(&ch->ch_rrd, &cph, sizeof(cph))) != 0) { 1837 mutex_exit(&ch->ch_rrd.rd_lock); 1838 return rv; 1839 } 1840 1841 offset = raw ? 0 : VMBUS_CHANPKT_GETLEN(cph.cph_hlen); 1842 pktlen = VMBUS_CHANPKT_GETLEN(cph.cph_tlen) - offset; 1843 if (pktlen > datalen) { 1844 mutex_exit(&ch->ch_rrd.rd_lock); 1845 device_printf(sc->sc_dev, "%s: pktlen %u datalen %u\n", 1846 __func__, pktlen, datalen); 1847 return EINVAL; 1848 } 1849 1850 rv = vmbus_ring_read(&ch->ch_rrd, data, pktlen, offset); 1851 if (rv == 0) { 1852 *rlen = pktlen; 1853 *rid = cph.cph_tid; 1854 } 1855 1856 mutex_exit(&ch->ch_rrd.rd_lock); 1857 1858 return rv; 1859 } 1860 1861 static inline void 1862 vmbus_ring_mask(struct vmbus_ring_data *rd) 1863 { 1864 1865 membar_sync(); 1866 rd->rd_ring->br_imask = 1; 1867 membar_sync(); 1868 } 1869 1870 static inline void 1871 vmbus_ring_unmask(struct vmbus_ring_data *rd) 1872 { 1873 1874 membar_sync(); 1875 rd->rd_ring->br_imask = 0; 1876 membar_sync(); 1877 } 1878 1879 static void 1880 vmbus_channel_pause(struct vmbus_channel *ch) 1881 { 1882 1883 vmbus_ring_mask(&ch->ch_rrd); 1884 } 1885 1886 static uint32_t 1887 vmbus_channel_unpause(struct vmbus_channel *ch) 1888 { 1889 uint32_t avail; 1890 1891 vmbus_ring_unmask(&ch->ch_rrd); 1892 vmbus_ring_avail(&ch->ch_rrd, NULL, &avail); 1893 1894 return avail; 1895 } 1896 1897 static uint32_t 1898 vmbus_channel_ready(struct vmbus_channel *ch) 1899 { 1900 uint32_t avail; 1901 1902 vmbus_ring_avail(&ch->ch_rrd, NULL, &avail); 1903 1904 return avail; 1905 } 1906 1907 /* How many PFNs can be referenced by the header */ 1908 #define VMBUS_NPFNHDR ((VMBUS_MSG_DSIZE_MAX - \ 1909 sizeof(struct vmbus_chanmsg_gpadl_conn)) / sizeof(uint64_t)) 1910 1911 /* How many PFNs can be referenced by the body */ 1912 #define VMBUS_NPFNBODY ((VMBUS_MSG_DSIZE_MAX - \ 1913 sizeof(struct vmbus_chanmsg_gpadl_subconn)) / sizeof(uint64_t)) 1914 1915 int 1916 vmbus_handle_alloc(struct vmbus_channel *ch, const struct hyperv_dma *dma, 1917 uint32_t buflen, uint32_t *handle) 1918 { 1919 const int prflags = cold ? PR_NOWAIT : PR_WAITOK; 1920 const int kmemflags = cold ? KM_NOSLEEP : KM_SLEEP; 1921 const int msgflags = cold ? MSGF_NOSLEEP : 0; 1922 const int hcflags = cold ? HCF_NOSLEEP : HCF_SLEEPOK; 1923 struct vmbus_softc *sc = ch->ch_sc; 1924 struct vmbus_chanmsg_gpadl_conn *hdr; 1925 struct vmbus_chanmsg_gpadl_subconn *cmd; 1926 struct vmbus_chanmsg_gpadl_connresp rsp; 1927 struct vmbus_msg *msg; 1928 int i, j, last, left, rv; 1929 int bodylen = 0, ncmds = 0, pfn = 0; 1930 uint64_t *frames; 1931 paddr_t pa; 1932 uint8_t *body; 1933 /* Total number of pages to reference */ 1934 int total = atop(buflen); 1935 /* Number of pages that will fit the header */ 1936 int inhdr = MIN(total, VMBUS_NPFNHDR); 1937 1938 KASSERT((buflen & PAGE_MASK) == 0); 1939 KASSERT(buflen == (uint32_t)dma->map->dm_mapsize); 1940 1941 msg = pool_cache_get_paddr(sc->sc_msgpool, prflags, &pa); 1942 if (msg == NULL) 1943 return ENOMEM; 1944 1945 /* Prepare array of frame addresses */ 1946 frames = kmem_zalloc(total * sizeof(*frames), kmemflags); 1947 if (frames == NULL) { 1948 pool_cache_put_paddr(sc->sc_msgpool, msg, pa); 1949 return ENOMEM; 1950 } 1951 for (i = 0, j = 0; i < dma->map->dm_nsegs && j < total; i++) { 1952 bus_dma_segment_t *seg = &dma->map->dm_segs[i]; 1953 bus_addr_t addr = seg->ds_addr; 1954 1955 KASSERT((addr & PAGE_MASK) == 0); 1956 KASSERT((seg->ds_len & PAGE_MASK) == 0); 1957 1958 while (addr < seg->ds_addr + seg->ds_len && j < total) { 1959 frames[j++] = atop(addr); 1960 addr += PAGE_SIZE; 1961 } 1962 } 1963 1964 memset(msg, 0, sizeof(*msg)); 1965 msg->msg_req.hc_dsize = sizeof(struct vmbus_chanmsg_gpadl_conn) + 1966 inhdr * sizeof(uint64_t); 1967 hdr = (struct vmbus_chanmsg_gpadl_conn *)msg->msg_req.hc_data; 1968 msg->msg_rsp = &rsp; 1969 msg->msg_rsplen = sizeof(rsp); 1970 msg->msg_flags = msgflags; 1971 1972 left = total - inhdr; 1973 1974 /* Allocate additional gpadl_body structures if required */ 1975 if (left > 0) { 1976 ncmds = howmany(left, VMBUS_NPFNBODY); 1977 bodylen = ncmds * VMBUS_MSG_DSIZE_MAX; 1978 body = kmem_zalloc(bodylen, kmemflags); 1979 if (body == NULL) { 1980 kmem_free(frames, total * sizeof(*frames)); 1981 pool_cache_put_paddr(sc->sc_msgpool, msg, pa); 1982 return ENOMEM; 1983 } 1984 } 1985 1986 *handle = atomic_inc_32_nv(&sc->sc_handle); 1987 1988 hdr->chm_hdr.chm_type = VMBUS_CHANMSG_GPADL_CONN; 1989 hdr->chm_chanid = ch->ch_id; 1990 hdr->chm_gpadl = *handle; 1991 1992 /* Single range for a contiguous buffer */ 1993 hdr->chm_range_cnt = 1; 1994 hdr->chm_range_len = sizeof(struct vmbus_gpa_range) + total * 1995 sizeof(uint64_t); 1996 hdr->chm_range.gpa_ofs = 0; 1997 hdr->chm_range.gpa_len = buflen; 1998 1999 /* Fit as many pages as possible into the header */ 2000 for (i = 0; i < inhdr; i++) 2001 hdr->chm_range.gpa_page[i] = frames[pfn++]; 2002 2003 for (i = 0; i < ncmds; i++) { 2004 cmd = (struct vmbus_chanmsg_gpadl_subconn *)(body + 2005 VMBUS_MSG_DSIZE_MAX * i); 2006 cmd->chm_hdr.chm_type = VMBUS_CHANMSG_GPADL_SUBCONN; 2007 cmd->chm_gpadl = *handle; 2008 last = MIN(left, VMBUS_NPFNBODY); 2009 for (j = 0; j < last; j++) 2010 cmd->chm_gpa_page[j] = frames[pfn++]; 2011 left -= last; 2012 } 2013 2014 rv = vmbus_start(sc, msg, pa); 2015 if (rv != 0) { 2016 DPRINTF("%s: GPADL_CONN failed\n", device_xname(sc->sc_dev)); 2017 goto out; 2018 } 2019 for (i = 0; i < ncmds; i++) { 2020 int cmdlen = sizeof(*cmd); 2021 cmd = (struct vmbus_chanmsg_gpadl_subconn *)(body + 2022 VMBUS_MSG_DSIZE_MAX * i); 2023 /* Last element can be short */ 2024 if (i == ncmds - 1) 2025 cmdlen += last * sizeof(uint64_t); 2026 else 2027 cmdlen += VMBUS_NPFNBODY * sizeof(uint64_t); 2028 rv = vmbus_cmd(sc, cmd, cmdlen, NULL, 0, HCF_NOREPLY | hcflags); 2029 if (rv != 0) { 2030 DPRINTF("%s: GPADL_SUBCONN (iteration %d/%d) failed " 2031 "with %d\n", device_xname(sc->sc_dev), i, ncmds, 2032 rv); 2033 goto out; 2034 } 2035 } 2036 rv = vmbus_reply(sc, msg); 2037 if (rv != 0) { 2038 DPRINTF("%s: GPADL allocation failed with %d\n", 2039 device_xname(sc->sc_dev), rv); 2040 } 2041 2042 out: 2043 if (bodylen > 0) 2044 kmem_free(body, bodylen); 2045 kmem_free(frames, total * sizeof(*frames)); 2046 pool_cache_put_paddr(sc->sc_msgpool, msg, pa); 2047 if (rv) 2048 return rv; 2049 2050 KASSERT(*handle == rsp.chm_gpadl); 2051 2052 return 0; 2053 } 2054 2055 void 2056 vmbus_handle_free(struct vmbus_channel *ch, uint32_t handle) 2057 { 2058 struct vmbus_softc *sc = ch->ch_sc; 2059 struct vmbus_chanmsg_gpadl_disconn cmd; 2060 struct vmbus_chanmsg_gpadl_disconn rsp; 2061 int rv; 2062 2063 memset(&cmd, 0, sizeof(cmd)); 2064 cmd.chm_hdr.chm_type = VMBUS_CHANMSG_GPADL_DISCONN; 2065 cmd.chm_chanid = ch->ch_id; 2066 cmd.chm_gpadl = handle; 2067 2068 rv = vmbus_cmd(sc, &cmd, sizeof(cmd), &rsp, sizeof(rsp), 2069 cold ? HCF_NOSLEEP : HCF_SLEEPOK); 2070 if (rv) { 2071 DPRINTF("%s: GPADL_DISCONN failed with %d\n", 2072 device_xname(sc->sc_dev), rv); 2073 } 2074 } 2075 2076 static void 2077 vmbus_chevq_enqueue(struct vmbus_softc *sc, int type, void *arg) 2078 { 2079 struct vmbus_chev *vce; 2080 2081 vce = kmem_intr_alloc(sizeof(*vce), KM_NOSLEEP); 2082 if (vce == NULL) { 2083 device_printf(sc->sc_dev, "failed to allocate chev\n"); 2084 return; 2085 } 2086 2087 vce->vce_type = type; 2088 vce->vce_arg = arg; 2089 2090 mutex_enter(&sc->sc_chevq_lock); 2091 SIMPLEQ_INSERT_TAIL(&sc->sc_chevq, vce, vce_entry); 2092 cv_broadcast(&sc->sc_chevq_cv); 2093 mutex_exit(&sc->sc_chevq_lock); 2094 } 2095 2096 static void 2097 vmbus_process_chevq(void *arg) 2098 { 2099 struct vmbus_softc *sc = arg; 2100 struct vmbus_chev *vce; 2101 struct vmbus_chanmsg_choffer *co; 2102 struct vmbus_chanmsg_chrescind *cr; 2103 2104 KASSERT(mutex_owned(&sc->sc_chevq_lock)); 2105 2106 while (!SIMPLEQ_EMPTY(&sc->sc_chevq)) { 2107 vce = SIMPLEQ_FIRST(&sc->sc_chevq); 2108 SIMPLEQ_REMOVE_HEAD(&sc->sc_chevq, vce_entry); 2109 mutex_exit(&sc->sc_chevq_lock); 2110 2111 switch (vce->vce_type) { 2112 case VMBUS_CHEV_TYPE_OFFER: 2113 co = vce->vce_arg; 2114 vmbus_process_offer(sc, co); 2115 kmem_free(co, sizeof(*co)); 2116 break; 2117 2118 case VMBUS_CHEV_TYPE_RESCIND: 2119 cr = vce->vce_arg; 2120 vmbus_process_rescind(sc, cr); 2121 kmem_free(cr, sizeof(*cr)); 2122 break; 2123 2124 default: 2125 DPRINTF("%s: unknown chevq type %d\n", 2126 device_xname(sc->sc_dev), vce->vce_type); 2127 break; 2128 } 2129 kmem_free(vce, sizeof(*vce)); 2130 2131 mutex_enter(&sc->sc_chevq_lock); 2132 } 2133 } 2134 2135 static void 2136 vmbus_chevq_thread(void *arg) 2137 { 2138 struct vmbus_softc *sc = arg; 2139 2140 mutex_enter(&sc->sc_chevq_lock); 2141 for (;;) { 2142 if (SIMPLEQ_EMPTY(&sc->sc_chevq)) { 2143 cv_wait(&sc->sc_chevq_cv, &sc->sc_chevq_lock); 2144 continue; 2145 } 2146 2147 vmbus_process_chevq(sc); 2148 } 2149 mutex_exit(&sc->sc_chevq_lock); 2150 2151 kthread_exit(0); 2152 } 2153 2154 static void 2155 vmbus_devq_enqueue(struct vmbus_softc *sc, int type, struct vmbus_channel *ch) 2156 { 2157 struct vmbus_dev *vd; 2158 2159 vd = kmem_zalloc(sizeof(*vd), KM_SLEEP); 2160 if (vd == NULL) { 2161 device_printf(sc->sc_dev, "failed to allocate devq\n"); 2162 return; 2163 } 2164 2165 vd->vd_type = type; 2166 vd->vd_chan = ch; 2167 2168 if (VMBUS_CHAN_ISPRIMARY(ch)) { 2169 mutex_enter(&sc->sc_devq_lock); 2170 SIMPLEQ_INSERT_TAIL(&sc->sc_devq, vd, vd_entry); 2171 cv_broadcast(&sc->sc_devq_cv); 2172 mutex_exit(&sc->sc_devq_lock); 2173 } else { 2174 mutex_enter(&sc->sc_subch_devq_lock); 2175 SIMPLEQ_INSERT_TAIL(&sc->sc_subch_devq, vd, vd_entry); 2176 cv_broadcast(&sc->sc_subch_devq_cv); 2177 mutex_exit(&sc->sc_subch_devq_lock); 2178 } 2179 } 2180 2181 static void 2182 vmbus_process_devq(void *arg) 2183 { 2184 struct vmbus_softc *sc = arg; 2185 struct vmbus_dev *vd; 2186 struct vmbus_channel *ch; 2187 struct vmbus_attach_args vaa; 2188 2189 KASSERT(mutex_owned(&sc->sc_devq_lock)); 2190 2191 while (!SIMPLEQ_EMPTY(&sc->sc_devq)) { 2192 vd = SIMPLEQ_FIRST(&sc->sc_devq); 2193 SIMPLEQ_REMOVE_HEAD(&sc->sc_devq, vd_entry); 2194 mutex_exit(&sc->sc_devq_lock); 2195 2196 switch (vd->vd_type) { 2197 case VMBUS_DEV_TYPE_ATTACH: 2198 ch = vd->vd_chan; 2199 vaa.aa_type = &ch->ch_type; 2200 vaa.aa_inst = &ch->ch_inst; 2201 vaa.aa_ident = ch->ch_ident; 2202 vaa.aa_chan = ch; 2203 vaa.aa_iot = sc->sc_iot; 2204 vaa.aa_memt = sc->sc_memt; 2205 ch->ch_dev = config_found(sc->sc_dev, 2206 &vaa, vmbus_attach_print, CFARG_EOL); 2207 break; 2208 2209 case VMBUS_DEV_TYPE_DETACH: 2210 ch = vd->vd_chan; 2211 if (ch->ch_dev != NULL) { 2212 config_detach(ch->ch_dev, DETACH_FORCE); 2213 ch->ch_dev = NULL; 2214 } 2215 vmbus_channel_release(ch); 2216 vmbus_channel_free(ch); 2217 break; 2218 2219 default: 2220 DPRINTF("%s: unknown devq type %d\n", 2221 device_xname(sc->sc_dev), vd->vd_type); 2222 break; 2223 } 2224 kmem_free(vd, sizeof(*vd)); 2225 2226 mutex_enter(&sc->sc_devq_lock); 2227 } 2228 } 2229 2230 static void 2231 vmbus_devq_thread(void *arg) 2232 { 2233 struct vmbus_softc *sc = arg; 2234 2235 mutex_enter(&sc->sc_devq_lock); 2236 for (;;) { 2237 if (SIMPLEQ_EMPTY(&sc->sc_devq)) { 2238 cv_wait(&sc->sc_devq_cv, &sc->sc_devq_lock); 2239 continue; 2240 } 2241 2242 vmbus_process_devq(sc); 2243 } 2244 mutex_exit(&sc->sc_devq_lock); 2245 2246 kthread_exit(0); 2247 } 2248 2249 static void 2250 vmbus_subchannel_devq_thread(void *arg) 2251 { 2252 struct vmbus_softc *sc = arg; 2253 struct vmbus_dev *vd; 2254 struct vmbus_channel *ch, *prich; 2255 2256 mutex_enter(&sc->sc_subch_devq_lock); 2257 for (;;) { 2258 if (SIMPLEQ_EMPTY(&sc->sc_subch_devq)) { 2259 cv_wait(&sc->sc_subch_devq_cv, &sc->sc_subch_devq_lock); 2260 continue; 2261 } 2262 2263 while (!SIMPLEQ_EMPTY(&sc->sc_subch_devq)) { 2264 vd = SIMPLEQ_FIRST(&sc->sc_subch_devq); 2265 SIMPLEQ_REMOVE_HEAD(&sc->sc_subch_devq, vd_entry); 2266 mutex_exit(&sc->sc_subch_devq_lock); 2267 2268 switch (vd->vd_type) { 2269 case VMBUS_DEV_TYPE_ATTACH: 2270 /* Nothing to do */ 2271 break; 2272 2273 case VMBUS_DEV_TYPE_DETACH: 2274 ch = vd->vd_chan; 2275 2276 vmbus_channel_release(ch); 2277 2278 prich = ch->ch_primary_channel; 2279 mutex_enter(&prich->ch_subchannel_lock); 2280 TAILQ_REMOVE(&prich->ch_subchannels, ch, 2281 ch_subentry); 2282 prich->ch_subchannel_count--; 2283 mutex_exit(&prich->ch_subchannel_lock); 2284 wakeup(prich); 2285 2286 vmbus_channel_free(ch); 2287 break; 2288 2289 default: 2290 DPRINTF("%s: unknown devq type %d\n", 2291 device_xname(sc->sc_dev), vd->vd_type); 2292 break; 2293 } 2294 2295 kmem_free(vd, sizeof(*vd)); 2296 2297 mutex_enter(&sc->sc_subch_devq_lock); 2298 } 2299 } 2300 mutex_exit(&sc->sc_subch_devq_lock); 2301 2302 kthread_exit(0); 2303 } 2304 2305 2306 static int 2307 vmbus_attach_print(void *aux, const char *name) 2308 { 2309 struct vmbus_attach_args *aa = aux; 2310 2311 if (name) 2312 printf("\"%s\" at %s", aa->aa_ident, name); 2313 2314 return UNCONF; 2315 } 2316 2317 MODULE(MODULE_CLASS_DRIVER, vmbus, "hyperv"); 2318 2319 #ifdef _MODULE 2320 #include "ioconf.c" 2321 #endif 2322 2323 static int 2324 vmbus_modcmd(modcmd_t cmd, void *aux) 2325 { 2326 int rv = 0; 2327 2328 switch (cmd) { 2329 case MODULE_CMD_INIT: 2330 #ifdef _MODULE 2331 rv = config_init_component(cfdriver_ioconf_vmbus, 2332 cfattach_ioconf_vmbus, cfdata_ioconf_vmbus); 2333 #endif 2334 break; 2335 2336 case MODULE_CMD_FINI: 2337 #ifdef _MODULE 2338 rv = config_fini_component(cfdriver_ioconf_vmbus, 2339 cfattach_ioconf_vmbus, cfdata_ioconf_vmbus); 2340 #endif 2341 break; 2342 2343 default: 2344 rv = ENOTTY; 2345 break; 2346 } 2347 2348 return rv; 2349 } 2350