1 /* $NetBSD: vmbus.c,v 1.11 2020/05/26 16:08:55 nonaka Exp $ */ 2 /* $OpenBSD: hyperv.c,v 1.43 2017/06/27 13:56:15 mikeb Exp $ */ 3 4 /*- 5 * Copyright (c) 2009-2012 Microsoft Corp. 6 * Copyright (c) 2012 NetApp Inc. 7 * Copyright (c) 2012 Citrix Inc. 8 * Copyright (c) 2016 Mike Belopuhov <mike@esdenera.com> 9 * All rights reserved. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice unmodified, this list of conditions, and the following 16 * disclaimer. 17 * 2. Redistributions in binary form must reproduce the above copyright 18 * notice, this list of conditions and the following disclaimer in the 19 * documentation and/or other materials provided with the distribution. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 22 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 23 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 24 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 26 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 27 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 28 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 29 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 30 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 /* 34 * The OpenBSD port was done under funding by Esdenera Networks GmbH. 35 */ 36 37 #include <sys/cdefs.h> 38 __KERNEL_RCSID(0, "$NetBSD: vmbus.c,v 1.11 2020/05/26 16:08:55 nonaka Exp $"); 39 40 #include <sys/param.h> 41 #include <sys/systm.h> 42 #include <sys/device.h> 43 #include <sys/atomic.h> 44 #include <sys/bitops.h> 45 #include <sys/bus.h> 46 #include <sys/cpu.h> 47 #include <sys/intr.h> 48 #include <sys/kmem.h> 49 #include <sys/kthread.h> 50 #include <sys/module.h> 51 #include <sys/mutex.h> 52 #include <sys/xcall.h> 53 54 #include <uvm/uvm_extern.h> 55 56 #include <dev/hyperv/vmbusvar.h> 57 58 #define VMBUS_GPADL_START 0xffff /* 0x10000 effectively */ 59 60 /* Command submission flags */ 61 #define HCF_SLEEPOK 0x0000 62 #define HCF_NOSLEEP 0x0002 /* M_NOWAIT */ 63 #define HCF_NOREPLY 0x0004 64 65 static void vmbus_attach_deferred(device_t); 66 static int vmbus_attach_print(void *, const char *); 67 static int vmbus_alloc_dma(struct vmbus_softc *); 68 static void vmbus_free_dma(struct vmbus_softc *); 69 static int vmbus_init_interrupts(struct vmbus_softc *); 70 static void vmbus_deinit_interrupts(struct vmbus_softc *); 71 static void vmbus_init_synic(void *, void *); 72 static void vmbus_deinit_synic(void *, void *); 73 74 static int vmbus_connect(struct vmbus_softc *); 75 static int vmbus_cmd(struct vmbus_softc *, void *, size_t, void *, size_t, 76 int); 77 static int vmbus_start(struct vmbus_softc *, struct vmbus_msg *, paddr_t); 78 static int vmbus_reply(struct vmbus_softc *, struct vmbus_msg *); 79 static uint16_t vmbus_intr_signal(struct vmbus_softc *, paddr_t); 80 static void vmbus_event_proc(void *, struct cpu_info *); 81 static void vmbus_event_proc_compat(void *, struct cpu_info *); 82 static void vmbus_message_proc(void *, struct cpu_info *); 83 static void vmbus_message_softintr(void *); 84 static void vmbus_channel_response(struct vmbus_softc *, 85 struct vmbus_chanmsg_hdr *); 86 static void vmbus_channel_offer(struct vmbus_softc *, 87 struct vmbus_chanmsg_hdr *); 88 static void vmbus_channel_rescind(struct vmbus_softc *, 89 struct vmbus_chanmsg_hdr *); 90 static void vmbus_channel_delivered(struct vmbus_softc *, 91 struct vmbus_chanmsg_hdr *); 92 static int vmbus_channel_scan(struct vmbus_softc *); 93 static void vmbus_channel_cpu_default(struct vmbus_channel *); 94 static void vmbus_process_offer(struct vmbus_softc *, 95 struct vmbus_chanmsg_choffer *); 96 static void vmbus_process_rescind(struct vmbus_softc *, 97 struct vmbus_chanmsg_chrescind *); 98 static struct vmbus_channel * 99 vmbus_channel_lookup(struct vmbus_softc *, uint32_t); 100 static int vmbus_channel_ring_create(struct vmbus_channel *, uint32_t); 101 static void vmbus_channel_ring_destroy(struct vmbus_channel *); 102 static void vmbus_channel_detach(struct vmbus_channel *); 103 static void vmbus_channel_pause(struct vmbus_channel *); 104 static uint32_t vmbus_channel_unpause(struct vmbus_channel *); 105 static uint32_t vmbus_channel_ready(struct vmbus_channel *); 106 static void vmbus_chevq_enqueue(struct vmbus_softc *, int, void *); 107 static void vmbus_process_chevq(void *); 108 static void vmbus_chevq_thread(void *); 109 static void vmbus_devq_enqueue(struct vmbus_softc *, int, 110 struct vmbus_channel *); 111 static void vmbus_process_devq(void *); 112 static void vmbus_devq_thread(void *); 113 static void vmbus_subchannel_devq_thread(void *); 114 115 static struct vmbus_softc *vmbus_sc; 116 117 static const struct { 118 int hmd_response; 119 int hmd_request; 120 void (*hmd_handler)(struct vmbus_softc *, 121 struct vmbus_chanmsg_hdr *); 122 } vmbus_msg_dispatch[] = { 123 { 0, 0, NULL }, 124 { VMBUS_CHANMSG_CHOFFER, 0, vmbus_channel_offer }, 125 { VMBUS_CHANMSG_CHRESCIND, 0, vmbus_channel_rescind }, 126 { VMBUS_CHANMSG_CHREQUEST, VMBUS_CHANMSG_CHOFFER, NULL }, 127 { VMBUS_CHANMSG_CHOFFER_DONE, 0, vmbus_channel_delivered }, 128 { VMBUS_CHANMSG_CHOPEN, 0, NULL }, 129 { VMBUS_CHANMSG_CHOPEN_RESP, VMBUS_CHANMSG_CHOPEN, 130 vmbus_channel_response }, 131 { VMBUS_CHANMSG_CHCLOSE, 0, NULL }, 132 { VMBUS_CHANMSG_GPADL_CONN, 0, NULL }, 133 { VMBUS_CHANMSG_GPADL_SUBCONN, 0, NULL }, 134 { VMBUS_CHANMSG_GPADL_CONNRESP, VMBUS_CHANMSG_GPADL_CONN, 135 vmbus_channel_response }, 136 { VMBUS_CHANMSG_GPADL_DISCONN, 0, NULL }, 137 { VMBUS_CHANMSG_GPADL_DISCONNRESP, VMBUS_CHANMSG_GPADL_DISCONN, 138 vmbus_channel_response }, 139 { VMBUS_CHANMSG_CHFREE, 0, NULL }, 140 { VMBUS_CHANMSG_CONNECT, 0, NULL }, 141 { VMBUS_CHANMSG_CONNECT_RESP, VMBUS_CHANMSG_CONNECT, 142 vmbus_channel_response }, 143 { VMBUS_CHANMSG_DISCONNECT, 0, NULL }, 144 }; 145 146 const struct hyperv_guid hyperv_guid_network = { 147 { 0x63, 0x51, 0x61, 0xf8, 0x3e, 0xdf, 0xc5, 0x46, 148 0x91, 0x3f, 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0x0e } 149 }; 150 151 const struct hyperv_guid hyperv_guid_ide = { 152 { 0x32, 0x26, 0x41, 0x32, 0xcb, 0x86, 0xa2, 0x44, 153 0x9b, 0x5c, 0x50, 0xd1, 0x41, 0x73, 0x54, 0xf5 } 154 }; 155 156 const struct hyperv_guid hyperv_guid_scsi = { 157 { 0xd9, 0x63, 0x61, 0xba, 0xa1, 0x04, 0x29, 0x4d, 158 0xb6, 0x05, 0x72, 0xe2, 0xff, 0xb1, 0xdc, 0x7f } 159 }; 160 161 const struct hyperv_guid hyperv_guid_shutdown = { 162 { 0x31, 0x60, 0x0b, 0x0e, 0x13, 0x52, 0x34, 0x49, 163 0x81, 0x8b, 0x38, 0xd9, 0x0c, 0xed, 0x39, 0xdb } 164 }; 165 166 const struct hyperv_guid hyperv_guid_timesync = { 167 { 0x30, 0xe6, 0x27, 0x95, 0xae, 0xd0, 0x7b, 0x49, 168 0xad, 0xce, 0xe8, 0x0a, 0xb0, 0x17, 0x5c, 0xaf } 169 }; 170 171 const struct hyperv_guid hyperv_guid_heartbeat = { 172 { 0x39, 0x4f, 0x16, 0x57, 0x15, 0x91, 0x78, 0x4e, 173 0xab, 0x55, 0x38, 0x2f, 0x3b, 0xd5, 0x42, 0x2d } 174 }; 175 176 const struct hyperv_guid hyperv_guid_kvp = { 177 { 0xe7, 0xf4, 0xa0, 0xa9, 0x45, 0x5a, 0x96, 0x4d, 178 0xb8, 0x27, 0x8a, 0x84, 0x1e, 0x8c, 0x03, 0xe6 } 179 }; 180 181 const struct hyperv_guid hyperv_guid_vss = { 182 { 0x29, 0x2e, 0xfa, 0x35, 0x23, 0xea, 0x36, 0x42, 183 0x96, 0xae, 0x3a, 0x6e, 0xba, 0xcb, 0xa4, 0x40 } 184 }; 185 186 const struct hyperv_guid hyperv_guid_dynmem = { 187 { 0xdc, 0x74, 0x50, 0x52, 0x85, 0x89, 0xe2, 0x46, 188 0x80, 0x57, 0xa3, 0x07, 0xdc, 0x18, 0xa5, 0x02 } 189 }; 190 191 const struct hyperv_guid hyperv_guid_mouse = { 192 { 0x9e, 0xb6, 0xa8, 0xcf, 0x4a, 0x5b, 0xc0, 0x4c, 193 0xb9, 0x8b, 0x8b, 0xa1, 0xa1, 0xf3, 0xf9, 0x5a } 194 }; 195 196 const struct hyperv_guid hyperv_guid_kbd = { 197 { 0x6d, 0xad, 0x12, 0xf9, 0x17, 0x2b, 0xea, 0x48, 198 0xbd, 0x65, 0xf9, 0x27, 0xa6, 0x1c, 0x76, 0x84 } 199 }; 200 201 const struct hyperv_guid hyperv_guid_video = { 202 { 0x02, 0x78, 0x0a, 0xda, 0x77, 0xe3, 0xac, 0x4a, 203 0x8e, 0x77, 0x05, 0x58, 0xeb, 0x10, 0x73, 0xf8 } 204 }; 205 206 const struct hyperv_guid hyperv_guid_fc = { 207 { 0x4a, 0xcc, 0x9b, 0x2f, 0x69, 0x00, 0xf3, 0x4a, 208 0xb7, 0x6b, 0x6f, 0xd0, 0xbe, 0x52, 0x8c, 0xda } 209 }; 210 211 const struct hyperv_guid hyperv_guid_fcopy = { 212 { 0xe3, 0x4b, 0xd1, 0x34, 0xe4, 0xde, 0xc8, 0x41, 213 0x9a, 0xe7, 0x6b, 0x17, 0x49, 0x77, 0xc1, 0x92 } 214 }; 215 216 const struct hyperv_guid hyperv_guid_pcie = { 217 { 0x1d, 0xf6, 0xc4, 0x44, 0x44, 0x44, 0x00, 0x44, 218 0x9d, 0x52, 0x80, 0x2e, 0x27, 0xed, 0xe1, 0x9f } 219 }; 220 221 const struct hyperv_guid hyperv_guid_netdir = { 222 { 0x3d, 0xaf, 0x2e, 0x8c, 0xa7, 0x32, 0x09, 0x4b, 223 0xab, 0x99, 0xbd, 0x1f, 0x1c, 0x86, 0xb5, 0x01 } 224 }; 225 226 const struct hyperv_guid hyperv_guid_rdesktop = { 227 { 0xf4, 0xac, 0x6a, 0x27, 0x15, 0xac, 0x6c, 0x42, 228 0x98, 0xdd, 0x75, 0x21, 0xad, 0x3f, 0x01, 0xfe } 229 }; 230 231 /* Automatic Virtual Machine Activation (AVMA) Services */ 232 const struct hyperv_guid hyperv_guid_avma1 = { 233 { 0x55, 0xb2, 0x87, 0x44, 0x8c, 0xb8, 0x3f, 0x40, 234 0xbb, 0x51, 0xd1, 0xf6, 0x9c, 0xf1, 0x7f, 0x87 } 235 }; 236 237 const struct hyperv_guid hyperv_guid_avma2 = { 238 { 0xf4, 0xba, 0x75, 0x33, 0x15, 0x9e, 0x30, 0x4b, 239 0xb7, 0x65, 0x67, 0xac, 0xb1, 0x0d, 0x60, 0x7b } 240 }; 241 242 const struct hyperv_guid hyperv_guid_avma3 = { 243 { 0xa0, 0x1f, 0x22, 0x99, 0xad, 0x24, 0xe2, 0x11, 244 0xbe, 0x98, 0x00, 0x1a, 0xa0, 0x1b, 0xbf, 0x6e } 245 }; 246 247 const struct hyperv_guid hyperv_guid_avma4 = { 248 { 0x16, 0x57, 0xe6, 0xf8, 0xb3, 0x3c, 0x06, 0x4a, 249 0x9a, 0x60, 0x18, 0x89, 0xc5, 0xcc, 0xca, 0xb5 } 250 }; 251 252 int 253 vmbus_match(device_t parent, cfdata_t cf, void *aux) 254 { 255 256 if (cf->cf_unit != 0 || 257 !hyperv_hypercall_enabled() || 258 !hyperv_synic_supported()) 259 return 0; 260 261 return 1; 262 } 263 264 int 265 vmbus_attach(struct vmbus_softc *sc) 266 { 267 268 aprint_naive("\n"); 269 aprint_normal(": Hyper-V VMBus\n"); 270 271 vmbus_sc = sc; 272 273 sc->sc_msgpool = pool_cache_init(sizeof(struct vmbus_msg), 8, 0, 0, 274 "hvmsg", NULL, IPL_NET, NULL, NULL, NULL); 275 hyperv_set_message_proc(vmbus_message_proc, sc); 276 277 if (vmbus_alloc_dma(sc)) 278 goto cleanup; 279 280 if (vmbus_init_interrupts(sc)) 281 goto cleanup; 282 283 if (vmbus_connect(sc)) 284 goto cleanup; 285 286 aprint_normal_dev(sc->sc_dev, "protocol %d.%d\n", 287 VMBUS_VERSION_MAJOR(sc->sc_proto), 288 VMBUS_VERSION_MINOR(sc->sc_proto)); 289 290 if (sc->sc_proto == VMBUS_VERSION_WS2008 || 291 sc->sc_proto == VMBUS_VERSION_WIN7) { 292 hyperv_set_event_proc(vmbus_event_proc_compat, sc); 293 sc->sc_channel_max = VMBUS_CHAN_MAX_COMPAT; 294 } else { 295 hyperv_set_event_proc(vmbus_event_proc, sc); 296 sc->sc_channel_max = VMBUS_CHAN_MAX; 297 } 298 299 if (vmbus_channel_scan(sc)) 300 goto cleanup; 301 302 config_interrupts(sc->sc_dev, vmbus_attach_deferred); 303 304 return 0; 305 306 cleanup: 307 vmbus_deinit_interrupts(sc); 308 vmbus_free_dma(sc); 309 return -1; 310 } 311 312 static void 313 vmbus_attach_deferred(device_t self) 314 { 315 struct vmbus_softc *sc = device_private(self); 316 317 xc_wait(xc_broadcast(0, vmbus_init_synic, sc, NULL)); 318 } 319 320 int 321 vmbus_detach(struct vmbus_softc *sc, int flags) 322 { 323 324 vmbus_deinit_interrupts(sc); 325 vmbus_free_dma(sc); 326 327 return 0; 328 } 329 330 static int 331 vmbus_alloc_dma(struct vmbus_softc *sc) 332 { 333 CPU_INFO_ITERATOR cii; 334 struct cpu_info *ci; 335 struct vmbus_percpu_data *pd; 336 int i; 337 338 /* 339 * Per-CPU messages and event flags. 340 */ 341 for (CPU_INFO_FOREACH(cii, ci)) { 342 pd = &sc->sc_percpu[cpu_index(ci)]; 343 344 pd->simp = hyperv_dma_alloc(sc->sc_dmat, &pd->simp_dma, 345 PAGE_SIZE, PAGE_SIZE, 0, 1, HYPERV_DMA_SLEEPOK); 346 if (pd->simp == NULL) 347 return ENOMEM; 348 349 pd->siep = hyperv_dma_alloc(sc->sc_dmat, &pd->siep_dma, 350 PAGE_SIZE, PAGE_SIZE, 0, 1, HYPERV_DMA_SLEEPOK); 351 if (pd->siep == NULL) 352 return ENOMEM; 353 } 354 355 sc->sc_events = hyperv_dma_alloc(sc->sc_dmat, &sc->sc_events_dma, 356 PAGE_SIZE, PAGE_SIZE, 0, 1, HYPERV_DMA_SLEEPOK); 357 if (sc->sc_events == NULL) 358 return ENOMEM; 359 sc->sc_wevents = (u_long *)sc->sc_events; 360 sc->sc_revents = (u_long *)((uint8_t *)sc->sc_events + (PAGE_SIZE / 2)); 361 362 for (i = 0; i < __arraycount(sc->sc_monitor); i++) { 363 sc->sc_monitor[i] = hyperv_dma_alloc(sc->sc_dmat, 364 &sc->sc_monitor_dma[i], PAGE_SIZE, PAGE_SIZE, 0, 1, 365 HYPERV_DMA_SLEEPOK); 366 if (sc->sc_monitor[i] == NULL) 367 return ENOMEM; 368 } 369 370 return 0; 371 } 372 373 static void 374 vmbus_free_dma(struct vmbus_softc *sc) 375 { 376 CPU_INFO_ITERATOR cii; 377 struct cpu_info *ci; 378 int i; 379 380 if (sc->sc_events != NULL) { 381 sc->sc_events = sc->sc_wevents = sc->sc_revents = NULL; 382 hyperv_dma_free(sc->sc_dmat, &sc->sc_events_dma); 383 } 384 385 for (i = 0; i < __arraycount(sc->sc_monitor); i++) { 386 sc->sc_monitor[i] = NULL; 387 hyperv_dma_free(sc->sc_dmat, &sc->sc_monitor_dma[i]); 388 } 389 390 for (CPU_INFO_FOREACH(cii, ci)) { 391 struct vmbus_percpu_data *pd = &sc->sc_percpu[cpu_index(ci)]; 392 393 if (pd->simp != NULL) { 394 pd->simp = NULL; 395 hyperv_dma_free(sc->sc_dmat, &pd->simp_dma); 396 } 397 if (pd->siep != NULL) { 398 pd->siep = NULL; 399 hyperv_dma_free(sc->sc_dmat, &pd->siep_dma); 400 } 401 } 402 } 403 404 static int 405 vmbus_init_interrupts(struct vmbus_softc *sc) 406 { 407 408 TAILQ_INIT(&sc->sc_reqs); 409 mutex_init(&sc->sc_req_lock, MUTEX_DEFAULT, IPL_NET); 410 411 TAILQ_INIT(&sc->sc_rsps); 412 mutex_init(&sc->sc_rsp_lock, MUTEX_DEFAULT, IPL_NET); 413 414 sc->sc_proto = VMBUS_VERSION_WS2008; 415 416 /* XXX event_tq */ 417 418 sc->sc_msg_sih = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 419 vmbus_message_softintr, sc); 420 if (sc->sc_msg_sih == NULL) 421 return -1; 422 423 vmbus_init_interrupts_md(sc); 424 425 kcpuset_create(&sc->sc_intr_cpuset, true); 426 if (cold) { 427 /* Initialize other CPUs later. */ 428 vmbus_init_synic(sc, NULL); 429 } else 430 xc_wait(xc_broadcast(0, vmbus_init_synic, sc, NULL)); 431 atomic_or_32(&sc->sc_flags, VMBUS_SCFLAG_SYNIC); 432 433 return 0; 434 } 435 436 static void 437 vmbus_deinit_interrupts(struct vmbus_softc *sc) 438 { 439 440 if (ISSET(sc->sc_flags, VMBUS_SCFLAG_SYNIC)) { 441 if (cold) 442 vmbus_deinit_synic(sc, NULL); 443 else 444 xc_wait(xc_broadcast(0, vmbus_deinit_synic, sc, NULL)); 445 atomic_and_32(&sc->sc_flags, (uint32_t)~VMBUS_SCFLAG_SYNIC); 446 } 447 448 /* XXX event_tq */ 449 450 if (sc->sc_msg_sih != NULL) { 451 softint_disestablish(sc->sc_msg_sih); 452 sc->sc_msg_sih = NULL; 453 } 454 455 vmbus_deinit_interrupts_md(sc); 456 } 457 458 static void 459 vmbus_init_synic(void *arg1, void *arg2) 460 { 461 struct vmbus_softc *sc = arg1; 462 cpuid_t cpu; 463 int s; 464 465 s = splhigh(); 466 467 cpu = cpu_index(curcpu()); 468 if (!kcpuset_isset(sc->sc_intr_cpuset, cpu)) { 469 kcpuset_atomic_set(sc->sc_intr_cpuset, cpu); 470 vmbus_init_synic_md(sc, cpu); 471 } 472 473 splx(s); 474 } 475 476 static void 477 vmbus_deinit_synic(void *arg1, void *arg2) 478 { 479 struct vmbus_softc *sc = arg1; 480 cpuid_t cpu; 481 int s; 482 483 s = splhigh(); 484 485 cpu = cpu_index(curcpu()); 486 if (kcpuset_isset(sc->sc_intr_cpuset, cpu)) { 487 vmbus_deinit_synic_md(sc, cpu); 488 kcpuset_atomic_clear(sc->sc_intr_cpuset, cpu); 489 } 490 491 splx(s); 492 } 493 494 static int 495 vmbus_connect(struct vmbus_softc *sc) 496 { 497 static const uint32_t versions[] = { 498 VMBUS_VERSION_WIN8_1, 499 VMBUS_VERSION_WIN8, 500 VMBUS_VERSION_WIN7, 501 VMBUS_VERSION_WS2008 502 }; 503 struct vmbus_chanmsg_connect cmd; 504 struct vmbus_chanmsg_connect_resp rsp; 505 int i, rv; 506 507 memset(&cmd, 0, sizeof(cmd)); 508 cmd.chm_hdr.chm_type = VMBUS_CHANMSG_CONNECT; 509 cmd.chm_evtflags = hyperv_dma_get_paddr(&sc->sc_events_dma); 510 cmd.chm_mnf1 = hyperv_dma_get_paddr(&sc->sc_monitor_dma[0]); 511 cmd.chm_mnf2 = hyperv_dma_get_paddr(&sc->sc_monitor_dma[1]); 512 513 memset(&rsp, 0, sizeof(rsp)); 514 515 for (i = 0; i < __arraycount(versions); i++) { 516 cmd.chm_ver = versions[i]; 517 rv = vmbus_cmd(sc, &cmd, sizeof(cmd), &rsp, sizeof(rsp), 518 cold ? HCF_NOSLEEP : HCF_SLEEPOK); 519 if (rv) { 520 DPRINTF("%s: CONNECT failed\n", 521 device_xname(sc->sc_dev)); 522 return rv; 523 } 524 if (rsp.chm_done) { 525 atomic_or_32(&sc->sc_flags, VMBUS_SCFLAG_CONNECTED); 526 sc->sc_proto = versions[i]; 527 sc->sc_handle = VMBUS_GPADL_START; 528 break; 529 } 530 } 531 if (i == __arraycount(versions)) { 532 device_printf(sc->sc_dev, 533 "failed to negotiate protocol version\n"); 534 return ENXIO; 535 } 536 537 return 0; 538 } 539 540 static int 541 vmbus_cmd(struct vmbus_softc *sc, void *cmd, size_t cmdlen, void *rsp, 542 size_t rsplen, int flags) 543 { 544 const int prflags = cold ? PR_NOWAIT : PR_WAITOK; 545 struct vmbus_msg *msg; 546 paddr_t pa; 547 int rv; 548 549 if (cmdlen > VMBUS_MSG_DSIZE_MAX) { 550 device_printf(sc->sc_dev, "payload too large (%zu)\n", 551 cmdlen); 552 return EMSGSIZE; 553 } 554 555 msg = pool_cache_get_paddr(sc->sc_msgpool, prflags, &pa); 556 if (msg == NULL) { 557 device_printf(sc->sc_dev, "couldn't get msgpool\n"); 558 return ENOMEM; 559 } 560 memset(msg, 0, sizeof(*msg)); 561 msg->msg_req.hc_dsize = cmdlen; 562 memcpy(msg->msg_req.hc_data, cmd, cmdlen); 563 564 if (!(flags & HCF_NOREPLY)) { 565 msg->msg_rsp = rsp; 566 msg->msg_rsplen = rsplen; 567 } else 568 msg->msg_flags |= MSGF_NOQUEUE; 569 570 if (flags & HCF_NOSLEEP) 571 msg->msg_flags |= MSGF_NOSLEEP; 572 573 rv = vmbus_start(sc, msg, pa); 574 if (rv == 0) 575 rv = vmbus_reply(sc, msg); 576 pool_cache_put_paddr(sc->sc_msgpool, msg, pa); 577 return rv; 578 } 579 580 static int 581 vmbus_start(struct vmbus_softc *sc, struct vmbus_msg *msg, paddr_t msg_pa) 582 { 583 static const int delays[] = { 584 100, 100, 100, 500, 500, 5000, 5000, 5000 585 }; 586 const char *wchan = "hvstart"; 587 uint16_t status; 588 int i, s; 589 590 msg->msg_req.hc_connid = VMBUS_CONNID_MESSAGE; 591 msg->msg_req.hc_msgtype = 1; 592 593 if (!(msg->msg_flags & MSGF_NOQUEUE)) { 594 mutex_enter(&sc->sc_req_lock); 595 TAILQ_INSERT_TAIL(&sc->sc_reqs, msg, msg_entry); 596 mutex_exit(&sc->sc_req_lock); 597 } 598 599 for (i = 0; i < __arraycount(delays); i++) { 600 status = hyperv_hypercall_post_message( 601 msg_pa + offsetof(struct vmbus_msg, msg_req)); 602 if (status == HYPERCALL_STATUS_SUCCESS) 603 break; 604 605 if (msg->msg_flags & MSGF_NOSLEEP) { 606 delay(delays[i]); 607 s = splnet(); 608 hyperv_intr(); 609 splx(s); 610 } else 611 tsleep(wchan, PRIBIO, wchan, mstohz(delays[i])); 612 } 613 if (status != HYPERCALL_STATUS_SUCCESS) { 614 device_printf(sc->sc_dev, 615 "posting vmbus message failed with %d\n", status); 616 if (!(msg->msg_flags & MSGF_NOQUEUE)) { 617 mutex_enter(&sc->sc_req_lock); 618 TAILQ_REMOVE(&sc->sc_reqs, msg, msg_entry); 619 mutex_exit(&sc->sc_req_lock); 620 } 621 return EIO; 622 } 623 624 return 0; 625 } 626 627 static int 628 vmbus_reply_done(struct vmbus_softc *sc, struct vmbus_msg *msg) 629 { 630 struct vmbus_msg *m; 631 632 mutex_enter(&sc->sc_rsp_lock); 633 TAILQ_FOREACH(m, &sc->sc_rsps, msg_entry) { 634 if (m == msg) { 635 mutex_exit(&sc->sc_rsp_lock); 636 return 1; 637 } 638 } 639 mutex_exit(&sc->sc_rsp_lock); 640 return 0; 641 } 642 643 static int 644 vmbus_reply(struct vmbus_softc *sc, struct vmbus_msg *msg) 645 { 646 int s; 647 648 if (msg->msg_flags & MSGF_NOQUEUE) 649 return 0; 650 651 while (!vmbus_reply_done(sc, msg)) { 652 if (msg->msg_flags & MSGF_NOSLEEP) { 653 delay(1000); 654 s = splnet(); 655 hyperv_intr(); 656 splx(s); 657 } else 658 tsleep(msg, PRIBIO, "hvreply", 1); 659 } 660 661 mutex_enter(&sc->sc_rsp_lock); 662 TAILQ_REMOVE(&sc->sc_rsps, msg, msg_entry); 663 mutex_exit(&sc->sc_rsp_lock); 664 665 return 0; 666 } 667 668 static uint16_t 669 vmbus_intr_signal(struct vmbus_softc *sc, paddr_t con_pa) 670 { 671 uint64_t status; 672 673 status = hyperv_hypercall_signal_event(con_pa); 674 return (uint16_t)status; 675 } 676 677 #if LONG_BIT == 64 678 #define ffsl(v) ffs64(v) 679 #elif LONG_BIT == 32 680 #define ffsl(v) ffs32(v) 681 #else 682 #error unsupport LONG_BIT 683 #endif /* LONG_BIT */ 684 685 static void 686 vmbus_event_flags_proc(struct vmbus_softc *sc, volatile u_long *revents, 687 int maxrow) 688 { 689 struct vmbus_channel *ch; 690 u_long pending; 691 uint32_t chanid, chanid_base; 692 int row, chanid_ofs; 693 694 for (row = 0; row < maxrow; row++) { 695 if (revents[row] == 0) 696 continue; 697 698 pending = atomic_swap_ulong(&revents[row], 0); 699 chanid_base = row * LONG_BIT; 700 701 while ((chanid_ofs = ffsl(pending)) != 0) { 702 chanid_ofs--; /* NOTE: ffs is 1-based */ 703 pending &= ~(1UL << chanid_ofs); 704 705 chanid = chanid_base + chanid_ofs; 706 /* vmbus channel protocol message */ 707 if (chanid == 0) 708 continue; 709 710 ch = vmbus_channel_lookup(sc, chanid); 711 if (ch == NULL) { 712 device_printf(sc->sc_dev, 713 "unhandled event on %d\n", chanid); 714 continue; 715 } 716 if (ch->ch_state != VMBUS_CHANSTATE_OPENED) { 717 device_printf(sc->sc_dev, 718 "channel %d is not active\n", chanid); 719 continue; 720 } 721 ch->ch_evcnt.ev_count++; 722 vmbus_channel_schedule(ch); 723 } 724 } 725 } 726 727 static void 728 vmbus_event_proc(void *arg, struct cpu_info *ci) 729 { 730 struct vmbus_softc *sc = arg; 731 struct vmbus_evtflags *evt; 732 733 /* 734 * On Host with Win8 or above, the event page can be 735 * checked directly to get the id of the channel 736 * that has the pending interrupt. 737 */ 738 evt = (struct vmbus_evtflags *)sc->sc_percpu[cpu_index(ci)].siep + 739 VMBUS_SINT_MESSAGE; 740 741 vmbus_event_flags_proc(sc, evt->evt_flags, 742 __arraycount(evt->evt_flags)); 743 } 744 745 static void 746 vmbus_event_proc_compat(void *arg, struct cpu_info *ci) 747 { 748 struct vmbus_softc *sc = arg; 749 struct vmbus_evtflags *evt; 750 751 evt = (struct vmbus_evtflags *)sc->sc_percpu[cpu_index(ci)].siep + 752 VMBUS_SINT_MESSAGE; 753 754 if (test_bit(0, &evt->evt_flags[0])) { 755 clear_bit(0, &evt->evt_flags[0]); 756 /* 757 * receive size is 1/2 page and divide that by 4 bytes 758 */ 759 vmbus_event_flags_proc(sc, sc->sc_revents, 760 VMBUS_CHAN_MAX_COMPAT / VMBUS_EVTFLAG_LEN); 761 } 762 } 763 764 static void 765 vmbus_message_proc(void *arg, struct cpu_info *ci) 766 { 767 struct vmbus_softc *sc = arg; 768 struct vmbus_message *msg; 769 770 msg = (struct vmbus_message *)sc->sc_percpu[cpu_index(ci)].simp + 771 VMBUS_SINT_MESSAGE; 772 if (__predict_false(msg->msg_type != HYPERV_MSGTYPE_NONE)) { 773 if (__predict_true(!cold)) 774 softint_schedule_cpu(sc->sc_msg_sih, ci); 775 else 776 vmbus_message_softintr(sc); 777 } 778 } 779 780 static void 781 vmbus_message_softintr(void *arg) 782 { 783 struct vmbus_softc *sc = arg; 784 struct vmbus_message *msg; 785 struct vmbus_chanmsg_hdr *hdr; 786 uint32_t type; 787 cpuid_t cpu; 788 789 cpu = cpu_index(curcpu()); 790 791 for (;;) { 792 msg = (struct vmbus_message *)sc->sc_percpu[cpu].simp + 793 VMBUS_SINT_MESSAGE; 794 if (msg->msg_type == HYPERV_MSGTYPE_NONE) 795 break; 796 797 hdr = (struct vmbus_chanmsg_hdr *)msg->msg_data; 798 type = hdr->chm_type; 799 if (type >= VMBUS_CHANMSG_COUNT) { 800 device_printf(sc->sc_dev, 801 "unhandled message type %u flags %#x\n", type, 802 msg->msg_flags); 803 } else { 804 if (vmbus_msg_dispatch[type].hmd_handler) { 805 vmbus_msg_dispatch[type].hmd_handler(sc, hdr); 806 } else { 807 device_printf(sc->sc_dev, 808 "unhandled message type %u\n", type); 809 } 810 } 811 812 msg->msg_type = HYPERV_MSGTYPE_NONE; 813 membar_sync(); 814 if (msg->msg_flags & VMBUS_MSGFLAG_PENDING) 815 hyperv_send_eom(); 816 } 817 } 818 819 static void 820 vmbus_channel_response(struct vmbus_softc *sc, struct vmbus_chanmsg_hdr *rsphdr) 821 { 822 struct vmbus_msg *msg; 823 struct vmbus_chanmsg_hdr *reqhdr; 824 int req; 825 826 req = vmbus_msg_dispatch[rsphdr->chm_type].hmd_request; 827 mutex_enter(&sc->sc_req_lock); 828 TAILQ_FOREACH(msg, &sc->sc_reqs, msg_entry) { 829 reqhdr = (struct vmbus_chanmsg_hdr *)&msg->msg_req.hc_data; 830 if (reqhdr->chm_type == req) { 831 TAILQ_REMOVE(&sc->sc_reqs, msg, msg_entry); 832 break; 833 } 834 } 835 mutex_exit(&sc->sc_req_lock); 836 if (msg != NULL) { 837 memcpy(msg->msg_rsp, rsphdr, msg->msg_rsplen); 838 mutex_enter(&sc->sc_rsp_lock); 839 TAILQ_INSERT_TAIL(&sc->sc_rsps, msg, msg_entry); 840 mutex_exit(&sc->sc_rsp_lock); 841 wakeup(msg); 842 } 843 } 844 845 static void 846 vmbus_channel_offer(struct vmbus_softc *sc, struct vmbus_chanmsg_hdr *hdr) 847 { 848 struct vmbus_chanmsg_choffer *co; 849 850 co = kmem_intr_alloc(sizeof(*co), KM_NOSLEEP); 851 if (co == NULL) { 852 device_printf(sc->sc_dev, 853 "failed to allocate an offer object\n"); 854 return; 855 } 856 857 memcpy(co, hdr, sizeof(*co)); 858 vmbus_chevq_enqueue(sc, VMBUS_CHEV_TYPE_OFFER, co); 859 } 860 861 static void 862 vmbus_channel_rescind(struct vmbus_softc *sc, struct vmbus_chanmsg_hdr *hdr) 863 { 864 struct vmbus_chanmsg_chrescind *cr; 865 866 cr = kmem_intr_alloc(sizeof(*cr), KM_NOSLEEP); 867 if (cr == NULL) { 868 device_printf(sc->sc_dev, 869 "failed to allocate an rescind object\n"); 870 return; 871 } 872 873 memcpy(cr, hdr, sizeof(*cr)); 874 vmbus_chevq_enqueue(sc, VMBUS_CHEV_TYPE_RESCIND, cr); 875 } 876 877 static void 878 vmbus_channel_delivered(struct vmbus_softc *sc, struct vmbus_chanmsg_hdr *hdr) 879 { 880 881 atomic_or_32(&sc->sc_flags, VMBUS_SCFLAG_OFFERS_DELIVERED); 882 wakeup(&sc->sc_devq); 883 } 884 885 static void 886 hyperv_guid_sprint(struct hyperv_guid *guid, char *str, size_t size) 887 { 888 static const struct { 889 const struct hyperv_guid *guid; 890 const char *ident; 891 } map[] = { 892 { &hyperv_guid_network, "network" }, 893 { &hyperv_guid_ide, "ide" }, 894 { &hyperv_guid_scsi, "scsi" }, 895 { &hyperv_guid_shutdown, "shutdown" }, 896 { &hyperv_guid_timesync, "timesync" }, 897 { &hyperv_guid_heartbeat, "heartbeat" }, 898 { &hyperv_guid_kvp, "kvp" }, 899 { &hyperv_guid_vss, "vss" }, 900 { &hyperv_guid_dynmem, "dynamic-memory" }, 901 { &hyperv_guid_mouse, "mouse" }, 902 { &hyperv_guid_kbd, "keyboard" }, 903 { &hyperv_guid_video, "video" }, 904 { &hyperv_guid_fc, "fiber-channel" }, 905 { &hyperv_guid_fcopy, "file-copy" }, 906 { &hyperv_guid_pcie, "pcie-passthrough" }, 907 { &hyperv_guid_netdir, "network-direct" }, 908 { &hyperv_guid_rdesktop, "remote-desktop" }, 909 { &hyperv_guid_avma1, "avma-1" }, 910 { &hyperv_guid_avma2, "avma-2" }, 911 { &hyperv_guid_avma3, "avma-3" }, 912 { &hyperv_guid_avma4, "avma-4" }, 913 }; 914 int i; 915 916 for (i = 0; i < __arraycount(map); i++) { 917 if (memcmp(guid, map[i].guid, sizeof(*guid)) == 0) { 918 strlcpy(str, map[i].ident, size); 919 return; 920 } 921 } 922 hyperv_guid2str(guid, str, size); 923 } 924 925 static int 926 vmbus_channel_scan(struct vmbus_softc *sc) 927 { 928 struct vmbus_chanmsg_hdr hdr; 929 struct vmbus_chanmsg_choffer rsp; 930 931 TAILQ_INIT(&sc->sc_prichans); 932 mutex_init(&sc->sc_prichan_lock, MUTEX_DEFAULT, IPL_NET); 933 TAILQ_INIT(&sc->sc_channels); 934 mutex_init(&sc->sc_channel_lock, MUTEX_DEFAULT, IPL_NET); 935 936 /* 937 * This queue serializes vmbus channel offer and rescind messages. 938 */ 939 SIMPLEQ_INIT(&sc->sc_chevq); 940 mutex_init(&sc->sc_chevq_lock, MUTEX_DEFAULT, IPL_NET); 941 cv_init(&sc->sc_chevq_cv, "hvchevcv"); 942 if (kthread_create(PRI_NONE, KTHREAD_MPSAFE, NULL, 943 vmbus_chevq_thread, sc, NULL, "hvchevq") != 0) { 944 DPRINTF("%s: failed to create prich chevq thread\n", 945 device_xname(sc->sc_dev)); 946 return -1; 947 } 948 949 /* 950 * This queue serializes vmbus devices' attach and detach 951 * for channel offer and rescind messages. 952 */ 953 SIMPLEQ_INIT(&sc->sc_devq); 954 mutex_init(&sc->sc_devq_lock, MUTEX_DEFAULT, IPL_NET); 955 cv_init(&sc->sc_devq_cv, "hvdevqcv"); 956 if (kthread_create(PRI_NONE, KTHREAD_MPSAFE, NULL, 957 vmbus_devq_thread, sc, NULL, "hvdevq") != 0) { 958 DPRINTF("%s: failed to create prich devq thread\n", 959 device_xname(sc->sc_dev)); 960 return -1; 961 } 962 963 /* 964 * This queue handles sub-channel detach, so that vmbus 965 * device's detach running in sc_devq can drain its sub-channels. 966 */ 967 SIMPLEQ_INIT(&sc->sc_subch_devq); 968 mutex_init(&sc->sc_subch_devq_lock, MUTEX_DEFAULT, IPL_NET); 969 cv_init(&sc->sc_subch_devq_cv, "hvsdvqcv"); 970 if (kthread_create(PRI_NONE, KTHREAD_MPSAFE, NULL, 971 vmbus_subchannel_devq_thread, sc, NULL, "hvsdevq") != 0) { 972 DPRINTF("%s: failed to create subch devq thread\n", 973 device_xname(sc->sc_dev)); 974 return -1; 975 } 976 977 memset(&hdr, 0, sizeof(hdr)); 978 hdr.chm_type = VMBUS_CHANMSG_CHREQUEST; 979 980 if (vmbus_cmd(sc, &hdr, sizeof(hdr), &rsp, sizeof(rsp), 981 HCF_NOREPLY | (cold ? HCF_NOSLEEP : HCF_SLEEPOK))) { 982 DPRINTF("%s: CHREQUEST failed\n", device_xname(sc->sc_dev)); 983 return -1; 984 } 985 986 while (!ISSET(sc->sc_flags, VMBUS_SCFLAG_OFFERS_DELIVERED)) 987 tsleep(&sc->sc_devq, PRIBIO, "hvscan", 1); 988 989 mutex_enter(&sc->sc_chevq_lock); 990 vmbus_process_chevq(sc); 991 mutex_exit(&sc->sc_chevq_lock); 992 mutex_enter(&sc->sc_devq_lock); 993 vmbus_process_devq(sc); 994 mutex_exit(&sc->sc_devq_lock); 995 996 return 0; 997 } 998 999 static struct vmbus_channel * 1000 vmbus_channel_alloc(struct vmbus_softc *sc) 1001 { 1002 struct vmbus_channel *ch; 1003 1004 ch = kmem_zalloc(sizeof(*ch), KM_SLEEP); 1005 1006 ch->ch_monprm = hyperv_dma_alloc(sc->sc_dmat, &ch->ch_monprm_dma, 1007 sizeof(*ch->ch_monprm), 8, 0, 1, HYPERV_DMA_SLEEPOK); 1008 if (ch->ch_monprm == NULL) { 1009 device_printf(sc->sc_dev, "monprm alloc failed\n"); 1010 kmem_free(ch, sizeof(*ch)); 1011 return NULL; 1012 } 1013 1014 ch->ch_refs = 1; 1015 ch->ch_sc = sc; 1016 mutex_init(&ch->ch_subchannel_lock, MUTEX_DEFAULT, IPL_NET); 1017 TAILQ_INIT(&ch->ch_subchannels); 1018 1019 ch->ch_state = VMBUS_CHANSTATE_CLOSED; 1020 1021 return ch; 1022 } 1023 1024 static void 1025 vmbus_channel_free(struct vmbus_channel *ch) 1026 { 1027 struct vmbus_softc *sc = ch->ch_sc; 1028 1029 KASSERTMSG(TAILQ_EMPTY(&ch->ch_subchannels) && 1030 ch->ch_subchannel_count == 0, "still owns sub-channels"); 1031 KASSERTMSG(ch->ch_state == 0 || ch->ch_state == VMBUS_CHANSTATE_CLOSED, 1032 "free busy channel"); 1033 KASSERTMSG(ch->ch_refs == 0, "channel %u: invalid refcnt %d", 1034 ch->ch_id, ch->ch_refs); 1035 1036 hyperv_dma_free(sc->sc_dmat, &ch->ch_monprm_dma); 1037 mutex_destroy(&ch->ch_subchannel_lock); 1038 /* XXX ch_evcnt */ 1039 if (ch->ch_taskq != NULL) 1040 softint_disestablish(ch->ch_taskq); 1041 kmem_free(ch, sizeof(*ch)); 1042 } 1043 1044 static int 1045 vmbus_channel_add(struct vmbus_channel *nch) 1046 { 1047 struct vmbus_softc *sc = nch->ch_sc; 1048 struct vmbus_channel *ch; 1049 u_int refs __diagused; 1050 1051 if (nch->ch_id == 0) { 1052 device_printf(sc->sc_dev, "got channel 0 offer, discard\n"); 1053 return EINVAL; 1054 } else if (nch->ch_id >= sc->sc_channel_max) { 1055 device_printf(sc->sc_dev, "invalid channel %u offer\n", 1056 nch->ch_id); 1057 return EINVAL; 1058 } 1059 1060 mutex_enter(&sc->sc_prichan_lock); 1061 TAILQ_FOREACH(ch, &sc->sc_prichans, ch_prientry) { 1062 if (!memcmp(&ch->ch_type, &nch->ch_type, sizeof(ch->ch_type)) && 1063 !memcmp(&ch->ch_inst, &nch->ch_inst, sizeof(ch->ch_inst))) 1064 break; 1065 } 1066 if (VMBUS_CHAN_ISPRIMARY(nch)) { 1067 if (ch == NULL) { 1068 TAILQ_INSERT_TAIL(&sc->sc_prichans, nch, ch_prientry); 1069 mutex_exit(&sc->sc_prichan_lock); 1070 goto done; 1071 } else { 1072 mutex_exit(&sc->sc_prichan_lock); 1073 device_printf(sc->sc_dev, 1074 "duplicated primary channel%u\n", nch->ch_id); 1075 return EINVAL; 1076 } 1077 } else { 1078 if (ch == NULL) { 1079 mutex_exit(&sc->sc_prichan_lock); 1080 device_printf(sc->sc_dev, "no primary channel%u\n", 1081 nch->ch_id); 1082 return EINVAL; 1083 } 1084 } 1085 mutex_exit(&sc->sc_prichan_lock); 1086 1087 KASSERT(!VMBUS_CHAN_ISPRIMARY(nch)); 1088 KASSERT(ch != NULL); 1089 1090 refs = atomic_inc_uint_nv(&nch->ch_refs); 1091 KASSERT(refs == 2); 1092 1093 nch->ch_primary_channel = ch; 1094 nch->ch_dev = ch->ch_dev; 1095 1096 mutex_enter(&ch->ch_subchannel_lock); 1097 TAILQ_INSERT_TAIL(&ch->ch_subchannels, nch, ch_subentry); 1098 ch->ch_subchannel_count++; 1099 mutex_exit(&ch->ch_subchannel_lock); 1100 wakeup(ch); 1101 1102 done: 1103 mutex_enter(&sc->sc_channel_lock); 1104 TAILQ_INSERT_TAIL(&sc->sc_channels, nch, ch_entry); 1105 mutex_exit(&sc->sc_channel_lock); 1106 1107 vmbus_channel_cpu_default(nch); 1108 1109 return 0; 1110 } 1111 1112 void 1113 vmbus_channel_cpu_set(struct vmbus_channel *ch, int cpu) 1114 { 1115 struct vmbus_softc *sc = ch->ch_sc; 1116 1117 KASSERTMSG(cpu >= 0 && cpu < ncpu, "invalid cpu %d", cpu); 1118 1119 if (sc->sc_proto == VMBUS_VERSION_WS2008 || 1120 sc->sc_proto == VMBUS_VERSION_WIN7) { 1121 /* Only cpu0 is supported */ 1122 cpu = 0; 1123 } 1124 1125 ch->ch_cpuid = cpu; 1126 ch->ch_vcpu = hyperv_get_vcpuid(cpu); 1127 } 1128 1129 void 1130 vmbus_channel_cpu_rr(struct vmbus_channel *ch) 1131 { 1132 static uint32_t vmbus_channel_nextcpu; 1133 int cpu; 1134 1135 cpu = atomic_inc_32_nv(&vmbus_channel_nextcpu) % ncpu; 1136 vmbus_channel_cpu_set(ch, cpu); 1137 } 1138 1139 static void 1140 vmbus_channel_cpu_default(struct vmbus_channel *ch) 1141 { 1142 1143 /* 1144 * By default, pin the channel to cpu0. Devices having 1145 * special channel-cpu mapping requirement should call 1146 * vmbus_channel_cpu_{set,rr}(). 1147 */ 1148 vmbus_channel_cpu_set(ch, 0); 1149 } 1150 1151 bool 1152 vmbus_channel_is_revoked(struct vmbus_channel *ch) 1153 { 1154 1155 return (ch->ch_flags & CHF_REVOKED) ? true : false; 1156 } 1157 1158 static void 1159 vmbus_process_offer(struct vmbus_softc *sc, struct vmbus_chanmsg_choffer *co) 1160 { 1161 struct vmbus_channel *ch; 1162 1163 ch = vmbus_channel_alloc(sc); 1164 if (ch == NULL) { 1165 device_printf(sc->sc_dev, "allocate channel %u failed\n", 1166 co->chm_chanid); 1167 return; 1168 } 1169 1170 /* 1171 * By default we setup state to enable batched reading. 1172 * A specific service can choose to disable this prior 1173 * to opening the channel. 1174 */ 1175 ch->ch_flags |= CHF_BATCHED; 1176 1177 hyperv_guid_sprint(&co->chm_chtype, ch->ch_ident, 1178 sizeof(ch->ch_ident)); 1179 1180 ch->ch_monprm->mp_connid = VMBUS_CONNID_EVENT; 1181 if (sc->sc_proto > VMBUS_VERSION_WS2008) 1182 ch->ch_monprm->mp_connid = co->chm_connid; 1183 1184 if (co->chm_flags1 & VMBUS_CHOFFER_FLAG1_HASMNF) { 1185 ch->ch_mgroup = co->chm_montrig / VMBUS_MONTRIG_LEN; 1186 ch->ch_mindex = co->chm_montrig % VMBUS_MONTRIG_LEN; 1187 ch->ch_flags |= CHF_MONITOR; 1188 } 1189 1190 ch->ch_id = co->chm_chanid; 1191 ch->ch_subidx = co->chm_subidx; 1192 1193 memcpy(&ch->ch_type, &co->chm_chtype, sizeof(ch->ch_type)); 1194 memcpy(&ch->ch_inst, &co->chm_chinst, sizeof(ch->ch_inst)); 1195 1196 if (vmbus_channel_add(ch) != 0) { 1197 atomic_dec_uint(&ch->ch_refs); 1198 vmbus_channel_free(ch); 1199 return; 1200 } 1201 1202 ch->ch_state = VMBUS_CHANSTATE_OFFERED; 1203 1204 vmbus_devq_enqueue(sc, VMBUS_DEV_TYPE_ATTACH, ch); 1205 1206 #ifdef HYPERV_DEBUG 1207 printf("%s: channel %u: \"%s\"", device_xname(sc->sc_dev), ch->ch_id, 1208 ch->ch_ident); 1209 if (ch->ch_flags & CHF_MONITOR) 1210 printf(", monitor %u\n", co->chm_montrig); 1211 else 1212 printf("\n"); 1213 #endif 1214 } 1215 1216 static void 1217 vmbus_process_rescind(struct vmbus_softc *sc, 1218 struct vmbus_chanmsg_chrescind *cr) 1219 { 1220 struct vmbus_channel *ch; 1221 1222 if (cr->chm_chanid > VMBUS_CHAN_MAX) { 1223 device_printf(sc->sc_dev, "invalid revoked channel%u\n", 1224 cr->chm_chanid); 1225 return; 1226 } 1227 1228 mutex_enter(&sc->sc_channel_lock); 1229 ch = vmbus_channel_lookup(sc, cr->chm_chanid); 1230 if (ch == NULL) { 1231 mutex_exit(&sc->sc_channel_lock); 1232 device_printf(sc->sc_dev, "channel%u is not offered\n", 1233 cr->chm_chanid); 1234 return; 1235 } 1236 TAILQ_REMOVE(&sc->sc_channels, ch, ch_entry); 1237 mutex_exit(&sc->sc_channel_lock); 1238 1239 if (VMBUS_CHAN_ISPRIMARY(ch)) { 1240 mutex_enter(&sc->sc_prichan_lock); 1241 TAILQ_REMOVE(&sc->sc_prichans, ch, ch_prientry); 1242 mutex_exit(&sc->sc_prichan_lock); 1243 } 1244 1245 KASSERTMSG(!(ch->ch_flags & CHF_REVOKED), 1246 "channel%u has already been revoked", ch->ch_id); 1247 atomic_or_uint(&ch->ch_flags, CHF_REVOKED); 1248 1249 vmbus_channel_detach(ch); 1250 } 1251 1252 static int 1253 vmbus_channel_release(struct vmbus_channel *ch) 1254 { 1255 struct vmbus_softc *sc = ch->ch_sc; 1256 struct vmbus_chanmsg_chfree cmd; 1257 int rv; 1258 1259 memset(&cmd, 0, sizeof(cmd)); 1260 cmd.chm_hdr.chm_type = VMBUS_CHANMSG_CHFREE; 1261 cmd.chm_chanid = ch->ch_id; 1262 1263 rv = vmbus_cmd(sc, &cmd, sizeof(cmd), NULL, 0, 1264 HCF_NOREPLY | (cold ? HCF_NOSLEEP : HCF_SLEEPOK)); 1265 if (rv) { 1266 DPRINTF("%s: CHFREE failed with %d\n", device_xname(sc->sc_dev), 1267 rv); 1268 } 1269 return rv; 1270 } 1271 1272 struct vmbus_channel ** 1273 vmbus_subchannel_get(struct vmbus_channel *prich, int cnt) 1274 { 1275 struct vmbus_softc *sc = prich->ch_sc; 1276 struct vmbus_channel **ret, *ch; 1277 int i, s; 1278 1279 KASSERTMSG(cnt > 0, "invalid sub-channel count %d", cnt); 1280 1281 ret = kmem_zalloc(sizeof(struct vmbus_channel *) * cnt, KM_SLEEP); 1282 1283 mutex_enter(&prich->ch_subchannel_lock); 1284 1285 while (prich->ch_subchannel_count < cnt) { 1286 if (cold) { 1287 mutex_exit(&prich->ch_subchannel_lock); 1288 delay(1000); 1289 s = splnet(); 1290 hyperv_intr(); 1291 splx(s); 1292 mutex_enter(&sc->sc_chevq_lock); 1293 vmbus_process_chevq(sc); 1294 mutex_exit(&sc->sc_chevq_lock); 1295 mutex_enter(&prich->ch_subchannel_lock); 1296 } else { 1297 mtsleep(prich, PRIBIO, "hvsubch", 1, 1298 &prich->ch_subchannel_lock); 1299 } 1300 } 1301 1302 i = 0; 1303 TAILQ_FOREACH(ch, &prich->ch_subchannels, ch_subentry) { 1304 ret[i] = ch; /* XXX inc refs */ 1305 1306 if (++i == cnt) 1307 break; 1308 } 1309 1310 KASSERTMSG(i == cnt, "invalid subchan count %d, should be %d", 1311 prich->ch_subchannel_count, cnt); 1312 1313 mutex_exit(&prich->ch_subchannel_lock); 1314 1315 return ret; 1316 } 1317 1318 void 1319 vmbus_subchannel_put(struct vmbus_channel **subch, int cnt) 1320 { 1321 1322 kmem_free(subch, sizeof(struct vmbus_channel *) * cnt); 1323 } 1324 1325 static struct vmbus_channel * 1326 vmbus_channel_lookup(struct vmbus_softc *sc, uint32_t relid) 1327 { 1328 struct vmbus_channel *ch; 1329 1330 TAILQ_FOREACH(ch, &sc->sc_channels, ch_entry) { 1331 if (ch->ch_id == relid) 1332 return ch; 1333 } 1334 return NULL; 1335 } 1336 1337 static int 1338 vmbus_channel_ring_create(struct vmbus_channel *ch, uint32_t buflen) 1339 { 1340 struct vmbus_softc *sc = ch->ch_sc; 1341 1342 buflen = roundup(buflen, PAGE_SIZE) + sizeof(struct vmbus_bufring); 1343 ch->ch_ring_size = 2 * buflen; 1344 /* page aligned memory */ 1345 ch->ch_ring = hyperv_dma_alloc(sc->sc_dmat, &ch->ch_ring_dma, 1346 ch->ch_ring_size, PAGE_SIZE, 0, 1, HYPERV_DMA_SLEEPOK); 1347 if (ch->ch_ring == NULL) { 1348 device_printf(sc->sc_dev, 1349 "failed to allocate channel ring\n"); 1350 return ENOMEM; 1351 } 1352 1353 memset(&ch->ch_wrd, 0, sizeof(ch->ch_wrd)); 1354 ch->ch_wrd.rd_ring = (struct vmbus_bufring *)ch->ch_ring; 1355 ch->ch_wrd.rd_size = buflen; 1356 ch->ch_wrd.rd_dsize = buflen - sizeof(struct vmbus_bufring); 1357 mutex_init(&ch->ch_wrd.rd_lock, MUTEX_DEFAULT, IPL_NET); 1358 1359 memset(&ch->ch_rrd, 0, sizeof(ch->ch_rrd)); 1360 ch->ch_rrd.rd_ring = (struct vmbus_bufring *)((uint8_t *)ch->ch_ring + 1361 buflen); 1362 ch->ch_rrd.rd_size = buflen; 1363 ch->ch_rrd.rd_dsize = buflen - sizeof(struct vmbus_bufring); 1364 mutex_init(&ch->ch_rrd.rd_lock, MUTEX_DEFAULT, IPL_NET); 1365 1366 if (vmbus_handle_alloc(ch, &ch->ch_ring_dma, ch->ch_ring_size, 1367 &ch->ch_ring_gpadl)) { 1368 device_printf(sc->sc_dev, 1369 "failed to obtain a PA handle for the ring\n"); 1370 vmbus_channel_ring_destroy(ch); 1371 return ENOMEM; 1372 } 1373 1374 return 0; 1375 } 1376 1377 static void 1378 vmbus_channel_ring_destroy(struct vmbus_channel *ch) 1379 { 1380 struct vmbus_softc *sc = ch->ch_sc; 1381 1382 hyperv_dma_free(sc->sc_dmat, &ch->ch_ring_dma); 1383 ch->ch_ring = NULL; 1384 vmbus_handle_free(ch, ch->ch_ring_gpadl); 1385 1386 mutex_destroy(&ch->ch_wrd.rd_lock); 1387 memset(&ch->ch_wrd, 0, sizeof(ch->ch_wrd)); 1388 mutex_destroy(&ch->ch_rrd.rd_lock); 1389 memset(&ch->ch_rrd, 0, sizeof(ch->ch_rrd)); 1390 } 1391 1392 int 1393 vmbus_channel_open(struct vmbus_channel *ch, size_t buflen, void *udata, 1394 size_t udatalen, void (*handler)(void *), void *arg) 1395 { 1396 struct vmbus_softc *sc = ch->ch_sc; 1397 struct vmbus_chanmsg_chopen cmd; 1398 struct vmbus_chanmsg_chopen_resp rsp; 1399 int rv = EINVAL; 1400 1401 if (ch->ch_ring == NULL && 1402 (rv = vmbus_channel_ring_create(ch, buflen))) { 1403 DPRINTF("%s: failed to create channel ring\n", 1404 device_xname(sc->sc_dev)); 1405 return rv; 1406 } 1407 1408 memset(&cmd, 0, sizeof(cmd)); 1409 cmd.chm_hdr.chm_type = VMBUS_CHANMSG_CHOPEN; 1410 cmd.chm_openid = ch->ch_id; 1411 cmd.chm_chanid = ch->ch_id; 1412 cmd.chm_gpadl = ch->ch_ring_gpadl; 1413 cmd.chm_txbr_pgcnt = atop(ch->ch_wrd.rd_size); 1414 cmd.chm_vcpuid = ch->ch_vcpu; 1415 if (udata && udatalen > 0) 1416 memcpy(cmd.chm_udata, udata, udatalen); 1417 1418 memset(&rsp, 0, sizeof(rsp)); 1419 1420 ch->ch_handler = handler; 1421 ch->ch_ctx = arg; 1422 ch->ch_state = VMBUS_CHANSTATE_OPENED; 1423 1424 rv = vmbus_cmd(sc, &cmd, sizeof(cmd), &rsp, sizeof(rsp), 1425 cold ? HCF_NOSLEEP : HCF_SLEEPOK); 1426 if (rv) { 1427 vmbus_channel_ring_destroy(ch); 1428 DPRINTF("%s: CHOPEN failed with %d\n", device_xname(sc->sc_dev), 1429 rv); 1430 ch->ch_handler = NULL; 1431 ch->ch_ctx = NULL; 1432 ch->ch_state = VMBUS_CHANSTATE_OFFERED; 1433 return rv; 1434 } 1435 return 0; 1436 } 1437 1438 static void 1439 vmbus_channel_detach(struct vmbus_channel *ch) 1440 { 1441 u_int refs; 1442 1443 KASSERTMSG(ch->ch_refs > 0, "channel%u: invalid refcnt %d", 1444 ch->ch_id, ch->ch_refs); 1445 1446 refs = atomic_dec_uint_nv(&ch->ch_refs); 1447 if (refs == 0) { 1448 /* Detach the target channel. */ 1449 vmbus_devq_enqueue(ch->ch_sc, VMBUS_DEV_TYPE_DETACH, ch); 1450 } 1451 } 1452 1453 static int 1454 vmbus_channel_close_internal(struct vmbus_channel *ch) 1455 { 1456 struct vmbus_softc *sc = ch->ch_sc; 1457 struct vmbus_chanmsg_chclose cmd; 1458 int rv; 1459 1460 memset(&cmd, 0, sizeof(cmd)); 1461 cmd.chm_hdr.chm_type = VMBUS_CHANMSG_CHCLOSE; 1462 cmd.chm_chanid = ch->ch_id; 1463 1464 ch->ch_state = VMBUS_CHANSTATE_CLOSING; 1465 rv = vmbus_cmd(sc, &cmd, sizeof(cmd), NULL, 0, 1466 HCF_NOREPLY | (cold ? HCF_NOSLEEP : HCF_SLEEPOK)); 1467 if (rv) { 1468 DPRINTF("%s: CHCLOSE failed with %d\n", 1469 device_xname(sc->sc_dev), rv); 1470 return rv; 1471 } 1472 ch->ch_state = VMBUS_CHANSTATE_CLOSED; 1473 vmbus_channel_ring_destroy(ch); 1474 return 0; 1475 } 1476 1477 int 1478 vmbus_channel_close_direct(struct vmbus_channel *ch) 1479 { 1480 int rv; 1481 1482 rv = vmbus_channel_close_internal(ch); 1483 if (!VMBUS_CHAN_ISPRIMARY(ch)) 1484 vmbus_channel_detach(ch); 1485 return rv; 1486 } 1487 1488 int 1489 vmbus_channel_close(struct vmbus_channel *ch) 1490 { 1491 struct vmbus_channel **subch; 1492 int i, cnt, rv; 1493 1494 if (!VMBUS_CHAN_ISPRIMARY(ch)) 1495 return 0; 1496 1497 cnt = ch->ch_subchannel_count; 1498 if (cnt > 0) { 1499 subch = vmbus_subchannel_get(ch, cnt); 1500 for (i = 0; i < ch->ch_subchannel_count; i++) { 1501 rv = vmbus_channel_close_internal(subch[i]); 1502 (void) rv; /* XXX */ 1503 vmbus_channel_detach(ch); 1504 } 1505 vmbus_subchannel_put(subch, cnt); 1506 } 1507 1508 return vmbus_channel_close_internal(ch); 1509 } 1510 1511 static inline void 1512 vmbus_channel_setevent(struct vmbus_softc *sc, struct vmbus_channel *ch) 1513 { 1514 struct vmbus_mon_trig *mtg; 1515 1516 /* Each uint32_t represents 32 channels */ 1517 set_bit(ch->ch_id, sc->sc_wevents); 1518 if (ch->ch_flags & CHF_MONITOR) { 1519 mtg = &sc->sc_monitor[1]->mnf_trigs[ch->ch_mgroup]; 1520 set_bit(ch->ch_mindex, &mtg->mt_pending); 1521 } else 1522 vmbus_intr_signal(sc, hyperv_dma_get_paddr(&ch->ch_monprm_dma)); 1523 } 1524 1525 static void 1526 vmbus_channel_intr(void *arg) 1527 { 1528 struct vmbus_channel *ch = arg; 1529 1530 if (vmbus_channel_ready(ch)) 1531 ch->ch_handler(ch->ch_ctx); 1532 1533 if (vmbus_channel_unpause(ch) == 0) 1534 return; 1535 1536 vmbus_channel_pause(ch); 1537 vmbus_channel_schedule(ch); 1538 } 1539 1540 int 1541 vmbus_channel_setdeferred(struct vmbus_channel *ch, const char *name) 1542 { 1543 1544 ch->ch_taskq = softint_establish(SOFTINT_NET | SOFTINT_MPSAFE, 1545 vmbus_channel_intr, ch); 1546 if (ch->ch_taskq == NULL) 1547 return -1; 1548 return 0; 1549 } 1550 1551 void 1552 vmbus_channel_schedule(struct vmbus_channel *ch) 1553 { 1554 1555 if (ch->ch_handler) { 1556 if (!cold && (ch->ch_flags & CHF_BATCHED)) { 1557 vmbus_channel_pause(ch); 1558 softint_schedule(ch->ch_taskq); 1559 } else 1560 ch->ch_handler(ch->ch_ctx); 1561 } 1562 } 1563 1564 static __inline void 1565 vmbus_ring_put(struct vmbus_ring_data *wrd, uint8_t *data, uint32_t datalen) 1566 { 1567 int left = MIN(datalen, wrd->rd_dsize - wrd->rd_prod); 1568 1569 memcpy(&wrd->rd_ring->br_data[wrd->rd_prod], data, left); 1570 memcpy(&wrd->rd_ring->br_data[0], data + left, datalen - left); 1571 wrd->rd_prod += datalen; 1572 if (wrd->rd_prod >= wrd->rd_dsize) 1573 wrd->rd_prod -= wrd->rd_dsize; 1574 } 1575 1576 static inline void 1577 vmbus_ring_get(struct vmbus_ring_data *rrd, uint8_t *data, uint32_t datalen, 1578 int peek) 1579 { 1580 int left = MIN(datalen, rrd->rd_dsize - rrd->rd_cons); 1581 1582 memcpy(data, &rrd->rd_ring->br_data[rrd->rd_cons], left); 1583 memcpy(data + left, &rrd->rd_ring->br_data[0], datalen - left); 1584 if (!peek) { 1585 rrd->rd_cons += datalen; 1586 if (rrd->rd_cons >= rrd->rd_dsize) 1587 rrd->rd_cons -= rrd->rd_dsize; 1588 } 1589 } 1590 1591 static __inline void 1592 vmbus_ring_avail(struct vmbus_ring_data *rd, uint32_t *towrite, 1593 uint32_t *toread) 1594 { 1595 uint32_t ridx = rd->rd_ring->br_rindex; 1596 uint32_t widx = rd->rd_ring->br_windex; 1597 uint32_t r, w; 1598 1599 if (widx >= ridx) 1600 w = rd->rd_dsize - (widx - ridx); 1601 else 1602 w = ridx - widx; 1603 r = rd->rd_dsize - w; 1604 if (towrite) 1605 *towrite = w; 1606 if (toread) 1607 *toread = r; 1608 } 1609 1610 static int 1611 vmbus_ring_write(struct vmbus_ring_data *wrd, struct iovec *iov, int iov_cnt, 1612 int *needsig) 1613 { 1614 uint64_t indices = 0; 1615 uint32_t avail, oprod, datalen = sizeof(indices); 1616 int i; 1617 1618 for (i = 0; i < iov_cnt; i++) 1619 datalen += iov[i].iov_len; 1620 1621 KASSERT(datalen <= wrd->rd_dsize); 1622 1623 vmbus_ring_avail(wrd, &avail, NULL); 1624 if (avail <= datalen) { 1625 DPRINTF("%s: avail %u datalen %u\n", __func__, avail, datalen); 1626 return EAGAIN; 1627 } 1628 1629 oprod = wrd->rd_prod; 1630 1631 for (i = 0; i < iov_cnt; i++) 1632 vmbus_ring_put(wrd, iov[i].iov_base, iov[i].iov_len); 1633 1634 indices = (uint64_t)oprod << 32; 1635 vmbus_ring_put(wrd, (uint8_t *)&indices, sizeof(indices)); 1636 1637 membar_sync(); 1638 wrd->rd_ring->br_windex = wrd->rd_prod; 1639 membar_sync(); 1640 1641 /* Signal when the ring transitions from being empty to non-empty */ 1642 if (wrd->rd_ring->br_imask == 0 && 1643 wrd->rd_ring->br_rindex == oprod) 1644 *needsig = 1; 1645 else 1646 *needsig = 0; 1647 1648 return 0; 1649 } 1650 1651 int 1652 vmbus_channel_send(struct vmbus_channel *ch, void *data, uint32_t datalen, 1653 uint64_t rid, int type, uint32_t flags) 1654 { 1655 struct vmbus_softc *sc = ch->ch_sc; 1656 struct vmbus_chanpkt cp; 1657 struct iovec iov[3]; 1658 uint32_t pktlen, pktlen_aligned; 1659 uint64_t zeropad = 0; 1660 int rv, needsig = 0; 1661 1662 pktlen = sizeof(cp) + datalen; 1663 pktlen_aligned = roundup(pktlen, sizeof(uint64_t)); 1664 1665 cp.cp_hdr.cph_type = type; 1666 cp.cp_hdr.cph_flags = flags; 1667 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_hlen, sizeof(cp)); 1668 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_tlen, pktlen_aligned); 1669 cp.cp_hdr.cph_tid = rid; 1670 1671 iov[0].iov_base = &cp; 1672 iov[0].iov_len = sizeof(cp); 1673 1674 iov[1].iov_base = data; 1675 iov[1].iov_len = datalen; 1676 1677 iov[2].iov_base = &zeropad; 1678 iov[2].iov_len = pktlen_aligned - pktlen; 1679 1680 mutex_enter(&ch->ch_wrd.rd_lock); 1681 rv = vmbus_ring_write(&ch->ch_wrd, iov, 3, &needsig); 1682 mutex_exit(&ch->ch_wrd.rd_lock); 1683 if (rv == 0 && needsig) 1684 vmbus_channel_setevent(sc, ch); 1685 1686 return rv; 1687 } 1688 1689 int 1690 vmbus_channel_send_sgl(struct vmbus_channel *ch, struct vmbus_gpa *sgl, 1691 uint32_t nsge, void *data, uint32_t datalen, uint64_t rid) 1692 { 1693 struct vmbus_softc *sc = ch->ch_sc; 1694 struct vmbus_chanpkt_sglist cp; 1695 struct iovec iov[4]; 1696 uint32_t buflen, pktlen, pktlen_aligned; 1697 uint64_t zeropad = 0; 1698 int rv, needsig = 0; 1699 1700 buflen = sizeof(struct vmbus_gpa) * nsge; 1701 pktlen = sizeof(cp) + datalen + buflen; 1702 pktlen_aligned = roundup(pktlen, sizeof(uint64_t)); 1703 1704 cp.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1705 cp.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1706 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_hlen, sizeof(cp) + buflen); 1707 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_tlen, pktlen_aligned); 1708 cp.cp_hdr.cph_tid = rid; 1709 cp.cp_gpa_cnt = nsge; 1710 cp.cp_rsvd = 0; 1711 1712 iov[0].iov_base = &cp; 1713 iov[0].iov_len = sizeof(cp); 1714 1715 iov[1].iov_base = sgl; 1716 iov[1].iov_len = buflen; 1717 1718 iov[2].iov_base = data; 1719 iov[2].iov_len = datalen; 1720 1721 iov[3].iov_base = &zeropad; 1722 iov[3].iov_len = pktlen_aligned - pktlen; 1723 1724 mutex_enter(&ch->ch_wrd.rd_lock); 1725 rv = vmbus_ring_write(&ch->ch_wrd, iov, 4, &needsig); 1726 mutex_exit(&ch->ch_wrd.rd_lock); 1727 if (rv == 0 && needsig) 1728 vmbus_channel_setevent(sc, ch); 1729 1730 return rv; 1731 } 1732 1733 int 1734 vmbus_channel_send_prpl(struct vmbus_channel *ch, struct vmbus_gpa_range *prpl, 1735 uint32_t nprp, void *data, uint32_t datalen, uint64_t rid) 1736 { 1737 struct vmbus_softc *sc = ch->ch_sc; 1738 struct vmbus_chanpkt_prplist cp; 1739 struct iovec iov[4]; 1740 uint32_t buflen, pktlen, pktlen_aligned; 1741 uint64_t zeropad = 0; 1742 int rv, needsig = 0; 1743 1744 buflen = sizeof(struct vmbus_gpa_range) * (nprp + 1); 1745 pktlen = sizeof(cp) + datalen + buflen; 1746 pktlen_aligned = roundup(pktlen, sizeof(uint64_t)); 1747 1748 cp.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1749 cp.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1750 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_hlen, sizeof(cp) + buflen); 1751 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_tlen, pktlen_aligned); 1752 cp.cp_hdr.cph_tid = rid; 1753 cp.cp_range_cnt = 1; 1754 cp.cp_rsvd = 0; 1755 1756 iov[0].iov_base = &cp; 1757 iov[0].iov_len = sizeof(cp); 1758 1759 iov[1].iov_base = prpl; 1760 iov[1].iov_len = buflen; 1761 1762 iov[2].iov_base = data; 1763 iov[2].iov_len = datalen; 1764 1765 iov[3].iov_base = &zeropad; 1766 iov[3].iov_len = pktlen_aligned - pktlen; 1767 1768 mutex_enter(&ch->ch_wrd.rd_lock); 1769 rv = vmbus_ring_write(&ch->ch_wrd, iov, 4, &needsig); 1770 mutex_exit(&ch->ch_wrd.rd_lock); 1771 if (rv == 0 && needsig) 1772 vmbus_channel_setevent(sc, ch); 1773 1774 return rv; 1775 } 1776 1777 static int 1778 vmbus_ring_peek(struct vmbus_ring_data *rrd, void *data, uint32_t datalen) 1779 { 1780 uint32_t avail; 1781 1782 KASSERT(datalen <= rrd->rd_dsize); 1783 1784 vmbus_ring_avail(rrd, NULL, &avail); 1785 if (avail < datalen) 1786 return EAGAIN; 1787 1788 vmbus_ring_get(rrd, (uint8_t *)data, datalen, 1); 1789 return 0; 1790 } 1791 1792 static int 1793 vmbus_ring_read(struct vmbus_ring_data *rrd, void *data, uint32_t datalen, 1794 uint32_t offset) 1795 { 1796 uint64_t indices; 1797 uint32_t avail; 1798 1799 KASSERT(datalen <= rrd->rd_dsize); 1800 1801 vmbus_ring_avail(rrd, NULL, &avail); 1802 if (avail < datalen) { 1803 DPRINTF("%s: avail %u datalen %u\n", __func__, avail, datalen); 1804 return EAGAIN; 1805 } 1806 1807 if (offset) { 1808 rrd->rd_cons += offset; 1809 if (rrd->rd_cons >= rrd->rd_dsize) 1810 rrd->rd_cons -= rrd->rd_dsize; 1811 } 1812 1813 vmbus_ring_get(rrd, (uint8_t *)data, datalen, 0); 1814 vmbus_ring_get(rrd, (uint8_t *)&indices, sizeof(indices), 0); 1815 1816 membar_sync(); 1817 rrd->rd_ring->br_rindex = rrd->rd_cons; 1818 1819 return 0; 1820 } 1821 1822 int 1823 vmbus_channel_recv(struct vmbus_channel *ch, void *data, uint32_t datalen, 1824 uint32_t *rlen, uint64_t *rid, int raw) 1825 { 1826 struct vmbus_softc *sc = ch->ch_sc; 1827 struct vmbus_chanpkt_hdr cph; 1828 uint32_t offset, pktlen; 1829 int rv; 1830 1831 *rlen = 0; 1832 1833 mutex_enter(&ch->ch_rrd.rd_lock); 1834 1835 if ((rv = vmbus_ring_peek(&ch->ch_rrd, &cph, sizeof(cph))) != 0) { 1836 mutex_exit(&ch->ch_rrd.rd_lock); 1837 return rv; 1838 } 1839 1840 offset = raw ? 0 : VMBUS_CHANPKT_GETLEN(cph.cph_hlen); 1841 pktlen = VMBUS_CHANPKT_GETLEN(cph.cph_tlen) - offset; 1842 if (pktlen > datalen) { 1843 mutex_exit(&ch->ch_rrd.rd_lock); 1844 device_printf(sc->sc_dev, "%s: pktlen %u datalen %u\n", 1845 __func__, pktlen, datalen); 1846 return EINVAL; 1847 } 1848 1849 rv = vmbus_ring_read(&ch->ch_rrd, data, pktlen, offset); 1850 if (rv == 0) { 1851 *rlen = pktlen; 1852 *rid = cph.cph_tid; 1853 } 1854 1855 mutex_exit(&ch->ch_rrd.rd_lock); 1856 1857 return rv; 1858 } 1859 1860 static inline void 1861 vmbus_ring_mask(struct vmbus_ring_data *rd) 1862 { 1863 1864 membar_sync(); 1865 rd->rd_ring->br_imask = 1; 1866 membar_sync(); 1867 } 1868 1869 static inline void 1870 vmbus_ring_unmask(struct vmbus_ring_data *rd) 1871 { 1872 1873 membar_sync(); 1874 rd->rd_ring->br_imask = 0; 1875 membar_sync(); 1876 } 1877 1878 static void 1879 vmbus_channel_pause(struct vmbus_channel *ch) 1880 { 1881 1882 vmbus_ring_mask(&ch->ch_rrd); 1883 } 1884 1885 static uint32_t 1886 vmbus_channel_unpause(struct vmbus_channel *ch) 1887 { 1888 uint32_t avail; 1889 1890 vmbus_ring_unmask(&ch->ch_rrd); 1891 vmbus_ring_avail(&ch->ch_rrd, NULL, &avail); 1892 1893 return avail; 1894 } 1895 1896 static uint32_t 1897 vmbus_channel_ready(struct vmbus_channel *ch) 1898 { 1899 uint32_t avail; 1900 1901 vmbus_ring_avail(&ch->ch_rrd, NULL, &avail); 1902 1903 return avail; 1904 } 1905 1906 /* How many PFNs can be referenced by the header */ 1907 #define VMBUS_NPFNHDR ((VMBUS_MSG_DSIZE_MAX - \ 1908 sizeof(struct vmbus_chanmsg_gpadl_conn)) / sizeof(uint64_t)) 1909 1910 /* How many PFNs can be referenced by the body */ 1911 #define VMBUS_NPFNBODY ((VMBUS_MSG_DSIZE_MAX - \ 1912 sizeof(struct vmbus_chanmsg_gpadl_subconn)) / sizeof(uint64_t)) 1913 1914 int 1915 vmbus_handle_alloc(struct vmbus_channel *ch, const struct hyperv_dma *dma, 1916 uint32_t buflen, uint32_t *handle) 1917 { 1918 const int prflags = cold ? PR_NOWAIT : PR_WAITOK; 1919 const int kmemflags = cold ? KM_NOSLEEP : KM_SLEEP; 1920 const int msgflags = cold ? MSGF_NOSLEEP : 0; 1921 const int hcflags = cold ? HCF_NOSLEEP : HCF_SLEEPOK; 1922 struct vmbus_softc *sc = ch->ch_sc; 1923 struct vmbus_chanmsg_gpadl_conn *hdr; 1924 struct vmbus_chanmsg_gpadl_subconn *cmd; 1925 struct vmbus_chanmsg_gpadl_connresp rsp; 1926 struct vmbus_msg *msg; 1927 int i, j, last, left, rv; 1928 int bodylen = 0, ncmds = 0, pfn = 0; 1929 uint64_t *frames; 1930 paddr_t pa; 1931 uint8_t *body; 1932 /* Total number of pages to reference */ 1933 int total = atop(buflen); 1934 /* Number of pages that will fit the header */ 1935 int inhdr = MIN(total, VMBUS_NPFNHDR); 1936 1937 KASSERT((buflen & PAGE_MASK) == 0); 1938 KASSERT(buflen == (uint32_t)dma->map->dm_mapsize); 1939 1940 msg = pool_cache_get_paddr(sc->sc_msgpool, prflags, &pa); 1941 if (msg == NULL) 1942 return ENOMEM; 1943 1944 /* Prepare array of frame addresses */ 1945 frames = kmem_zalloc(total * sizeof(*frames), kmemflags); 1946 if (frames == NULL) { 1947 pool_cache_put_paddr(sc->sc_msgpool, msg, pa); 1948 return ENOMEM; 1949 } 1950 for (i = 0, j = 0; i < dma->map->dm_nsegs && j < total; i++) { 1951 bus_dma_segment_t *seg = &dma->map->dm_segs[i]; 1952 bus_addr_t addr = seg->ds_addr; 1953 1954 KASSERT((addr & PAGE_MASK) == 0); 1955 KASSERT((seg->ds_len & PAGE_MASK) == 0); 1956 1957 while (addr < seg->ds_addr + seg->ds_len && j < total) { 1958 frames[j++] = atop(addr); 1959 addr += PAGE_SIZE; 1960 } 1961 } 1962 1963 memset(msg, 0, sizeof(*msg)); 1964 msg->msg_req.hc_dsize = sizeof(struct vmbus_chanmsg_gpadl_conn) + 1965 inhdr * sizeof(uint64_t); 1966 hdr = (struct vmbus_chanmsg_gpadl_conn *)msg->msg_req.hc_data; 1967 msg->msg_rsp = &rsp; 1968 msg->msg_rsplen = sizeof(rsp); 1969 msg->msg_flags = msgflags; 1970 1971 left = total - inhdr; 1972 1973 /* Allocate additional gpadl_body structures if required */ 1974 if (left > 0) { 1975 ncmds = howmany(left, VMBUS_NPFNBODY); 1976 bodylen = ncmds * VMBUS_MSG_DSIZE_MAX; 1977 body = kmem_zalloc(bodylen, kmemflags); 1978 if (body == NULL) { 1979 kmem_free(frames, total * sizeof(*frames)); 1980 pool_cache_put_paddr(sc->sc_msgpool, msg, pa); 1981 return ENOMEM; 1982 } 1983 } 1984 1985 *handle = atomic_inc_32_nv(&sc->sc_handle); 1986 1987 hdr->chm_hdr.chm_type = VMBUS_CHANMSG_GPADL_CONN; 1988 hdr->chm_chanid = ch->ch_id; 1989 hdr->chm_gpadl = *handle; 1990 1991 /* Single range for a contiguous buffer */ 1992 hdr->chm_range_cnt = 1; 1993 hdr->chm_range_len = sizeof(struct vmbus_gpa_range) + total * 1994 sizeof(uint64_t); 1995 hdr->chm_range.gpa_ofs = 0; 1996 hdr->chm_range.gpa_len = buflen; 1997 1998 /* Fit as many pages as possible into the header */ 1999 for (i = 0; i < inhdr; i++) 2000 hdr->chm_range.gpa_page[i] = frames[pfn++]; 2001 2002 for (i = 0; i < ncmds; i++) { 2003 cmd = (struct vmbus_chanmsg_gpadl_subconn *)(body + 2004 VMBUS_MSG_DSIZE_MAX * i); 2005 cmd->chm_hdr.chm_type = VMBUS_CHANMSG_GPADL_SUBCONN; 2006 cmd->chm_gpadl = *handle; 2007 last = MIN(left, VMBUS_NPFNBODY); 2008 for (j = 0; j < last; j++) 2009 cmd->chm_gpa_page[j] = frames[pfn++]; 2010 left -= last; 2011 } 2012 2013 rv = vmbus_start(sc, msg, pa); 2014 if (rv != 0) { 2015 DPRINTF("%s: GPADL_CONN failed\n", device_xname(sc->sc_dev)); 2016 goto out; 2017 } 2018 for (i = 0; i < ncmds; i++) { 2019 int cmdlen = sizeof(*cmd); 2020 cmd = (struct vmbus_chanmsg_gpadl_subconn *)(body + 2021 VMBUS_MSG_DSIZE_MAX * i); 2022 /* Last element can be short */ 2023 if (i == ncmds - 1) 2024 cmdlen += last * sizeof(uint64_t); 2025 else 2026 cmdlen += VMBUS_NPFNBODY * sizeof(uint64_t); 2027 rv = vmbus_cmd(sc, cmd, cmdlen, NULL, 0, HCF_NOREPLY | hcflags); 2028 if (rv != 0) { 2029 DPRINTF("%s: GPADL_SUBCONN (iteration %d/%d) failed " 2030 "with %d\n", device_xname(sc->sc_dev), i, ncmds, 2031 rv); 2032 goto out; 2033 } 2034 } 2035 rv = vmbus_reply(sc, msg); 2036 if (rv != 0) { 2037 DPRINTF("%s: GPADL allocation failed with %d\n", 2038 device_xname(sc->sc_dev), rv); 2039 } 2040 2041 out: 2042 if (bodylen > 0) 2043 kmem_free(body, bodylen); 2044 kmem_free(frames, total * sizeof(*frames)); 2045 pool_cache_put_paddr(sc->sc_msgpool, msg, pa); 2046 if (rv) 2047 return rv; 2048 2049 KASSERT(*handle == rsp.chm_gpadl); 2050 2051 return 0; 2052 } 2053 2054 void 2055 vmbus_handle_free(struct vmbus_channel *ch, uint32_t handle) 2056 { 2057 struct vmbus_softc *sc = ch->ch_sc; 2058 struct vmbus_chanmsg_gpadl_disconn cmd; 2059 struct vmbus_chanmsg_gpadl_disconn rsp; 2060 int rv; 2061 2062 memset(&cmd, 0, sizeof(cmd)); 2063 cmd.chm_hdr.chm_type = VMBUS_CHANMSG_GPADL_DISCONN; 2064 cmd.chm_chanid = ch->ch_id; 2065 cmd.chm_gpadl = handle; 2066 2067 rv = vmbus_cmd(sc, &cmd, sizeof(cmd), &rsp, sizeof(rsp), 2068 cold ? HCF_NOSLEEP : HCF_SLEEPOK); 2069 if (rv) { 2070 DPRINTF("%s: GPADL_DISCONN failed with %d\n", 2071 device_xname(sc->sc_dev), rv); 2072 } 2073 } 2074 2075 static void 2076 vmbus_chevq_enqueue(struct vmbus_softc *sc, int type, void *arg) 2077 { 2078 struct vmbus_chev *vce; 2079 2080 vce = kmem_intr_alloc(sizeof(*vce), KM_NOSLEEP); 2081 if (vce == NULL) { 2082 device_printf(sc->sc_dev, "failed to allocate chev\n"); 2083 return; 2084 } 2085 2086 vce->vce_type = type; 2087 vce->vce_arg = arg; 2088 2089 mutex_enter(&sc->sc_chevq_lock); 2090 SIMPLEQ_INSERT_TAIL(&sc->sc_chevq, vce, vce_entry); 2091 cv_broadcast(&sc->sc_chevq_cv); 2092 mutex_exit(&sc->sc_chevq_lock); 2093 } 2094 2095 static void 2096 vmbus_process_chevq(void *arg) 2097 { 2098 struct vmbus_softc *sc = arg; 2099 struct vmbus_chev *vce; 2100 struct vmbus_chanmsg_choffer *co; 2101 struct vmbus_chanmsg_chrescind *cr; 2102 2103 KASSERT(mutex_owned(&sc->sc_chevq_lock)); 2104 2105 while (!SIMPLEQ_EMPTY(&sc->sc_chevq)) { 2106 vce = SIMPLEQ_FIRST(&sc->sc_chevq); 2107 SIMPLEQ_REMOVE_HEAD(&sc->sc_chevq, vce_entry); 2108 mutex_exit(&sc->sc_chevq_lock); 2109 2110 switch (vce->vce_type) { 2111 case VMBUS_CHEV_TYPE_OFFER: 2112 co = vce->vce_arg; 2113 vmbus_process_offer(sc, co); 2114 kmem_free(co, sizeof(*co)); 2115 break; 2116 2117 case VMBUS_CHEV_TYPE_RESCIND: 2118 cr = vce->vce_arg; 2119 vmbus_process_rescind(sc, cr); 2120 kmem_free(cr, sizeof(*cr)); 2121 break; 2122 2123 default: 2124 DPRINTF("%s: unknown chevq type %d\n", 2125 device_xname(sc->sc_dev), vce->vce_type); 2126 break; 2127 } 2128 kmem_free(vce, sizeof(*vce)); 2129 2130 mutex_enter(&sc->sc_chevq_lock); 2131 } 2132 } 2133 2134 static void 2135 vmbus_chevq_thread(void *arg) 2136 { 2137 struct vmbus_softc *sc = arg; 2138 2139 mutex_enter(&sc->sc_chevq_lock); 2140 for (;;) { 2141 if (SIMPLEQ_EMPTY(&sc->sc_chevq)) { 2142 cv_wait(&sc->sc_chevq_cv, &sc->sc_chevq_lock); 2143 continue; 2144 } 2145 2146 vmbus_process_chevq(sc); 2147 } 2148 mutex_exit(&sc->sc_chevq_lock); 2149 2150 kthread_exit(0); 2151 } 2152 2153 static void 2154 vmbus_devq_enqueue(struct vmbus_softc *sc, int type, struct vmbus_channel *ch) 2155 { 2156 struct vmbus_dev *vd; 2157 2158 vd = kmem_zalloc(sizeof(*vd), KM_SLEEP); 2159 if (vd == NULL) { 2160 device_printf(sc->sc_dev, "failed to allocate devq\n"); 2161 return; 2162 } 2163 2164 vd->vd_type = type; 2165 vd->vd_chan = ch; 2166 2167 if (VMBUS_CHAN_ISPRIMARY(ch)) { 2168 mutex_enter(&sc->sc_devq_lock); 2169 SIMPLEQ_INSERT_TAIL(&sc->sc_devq, vd, vd_entry); 2170 cv_broadcast(&sc->sc_devq_cv); 2171 mutex_exit(&sc->sc_devq_lock); 2172 } else { 2173 mutex_enter(&sc->sc_subch_devq_lock); 2174 SIMPLEQ_INSERT_TAIL(&sc->sc_subch_devq, vd, vd_entry); 2175 cv_broadcast(&sc->sc_subch_devq_cv); 2176 mutex_exit(&sc->sc_subch_devq_lock); 2177 } 2178 } 2179 2180 static void 2181 vmbus_process_devq(void *arg) 2182 { 2183 struct vmbus_softc *sc = arg; 2184 struct vmbus_dev *vd; 2185 struct vmbus_channel *ch; 2186 struct vmbus_attach_args vaa; 2187 2188 KASSERT(mutex_owned(&sc->sc_devq_lock)); 2189 2190 while (!SIMPLEQ_EMPTY(&sc->sc_devq)) { 2191 vd = SIMPLEQ_FIRST(&sc->sc_devq); 2192 SIMPLEQ_REMOVE_HEAD(&sc->sc_devq, vd_entry); 2193 mutex_exit(&sc->sc_devq_lock); 2194 2195 switch (vd->vd_type) { 2196 case VMBUS_DEV_TYPE_ATTACH: 2197 ch = vd->vd_chan; 2198 vaa.aa_type = &ch->ch_type; 2199 vaa.aa_inst = &ch->ch_inst; 2200 vaa.aa_ident = ch->ch_ident; 2201 vaa.aa_chan = ch; 2202 vaa.aa_iot = sc->sc_iot; 2203 vaa.aa_memt = sc->sc_memt; 2204 ch->ch_dev = config_found_ia(sc->sc_dev, 2205 "hypervvmbus", &vaa, vmbus_attach_print); 2206 break; 2207 2208 case VMBUS_DEV_TYPE_DETACH: 2209 ch = vd->vd_chan; 2210 if (ch->ch_dev != NULL) { 2211 config_detach(ch->ch_dev, DETACH_FORCE); 2212 ch->ch_dev = NULL; 2213 } 2214 vmbus_channel_release(ch); 2215 vmbus_channel_free(ch); 2216 break; 2217 2218 default: 2219 DPRINTF("%s: unknown devq type %d\n", 2220 device_xname(sc->sc_dev), vd->vd_type); 2221 break; 2222 } 2223 kmem_free(vd, sizeof(*vd)); 2224 2225 mutex_enter(&sc->sc_devq_lock); 2226 } 2227 } 2228 2229 static void 2230 vmbus_devq_thread(void *arg) 2231 { 2232 struct vmbus_softc *sc = arg; 2233 2234 mutex_enter(&sc->sc_devq_lock); 2235 for (;;) { 2236 if (SIMPLEQ_EMPTY(&sc->sc_devq)) { 2237 cv_wait(&sc->sc_devq_cv, &sc->sc_devq_lock); 2238 continue; 2239 } 2240 2241 vmbus_process_devq(sc); 2242 } 2243 mutex_exit(&sc->sc_devq_lock); 2244 2245 kthread_exit(0); 2246 } 2247 2248 static void 2249 vmbus_subchannel_devq_thread(void *arg) 2250 { 2251 struct vmbus_softc *sc = arg; 2252 struct vmbus_dev *vd; 2253 struct vmbus_channel *ch, *prich; 2254 2255 mutex_enter(&sc->sc_subch_devq_lock); 2256 for (;;) { 2257 if (SIMPLEQ_EMPTY(&sc->sc_subch_devq)) { 2258 cv_wait(&sc->sc_subch_devq_cv, &sc->sc_subch_devq_lock); 2259 continue; 2260 } 2261 2262 while (!SIMPLEQ_EMPTY(&sc->sc_subch_devq)) { 2263 vd = SIMPLEQ_FIRST(&sc->sc_subch_devq); 2264 SIMPLEQ_REMOVE_HEAD(&sc->sc_subch_devq, vd_entry); 2265 mutex_exit(&sc->sc_subch_devq_lock); 2266 2267 switch (vd->vd_type) { 2268 case VMBUS_DEV_TYPE_ATTACH: 2269 /* Nothing to do */ 2270 break; 2271 2272 case VMBUS_DEV_TYPE_DETACH: 2273 ch = vd->vd_chan; 2274 2275 vmbus_channel_release(ch); 2276 2277 prich = ch->ch_primary_channel; 2278 mutex_enter(&prich->ch_subchannel_lock); 2279 TAILQ_REMOVE(&prich->ch_subchannels, ch, 2280 ch_subentry); 2281 prich->ch_subchannel_count--; 2282 mutex_exit(&prich->ch_subchannel_lock); 2283 wakeup(prich); 2284 2285 vmbus_channel_free(ch); 2286 break; 2287 2288 default: 2289 DPRINTF("%s: unknown devq type %d\n", 2290 device_xname(sc->sc_dev), vd->vd_type); 2291 break; 2292 } 2293 2294 kmem_free(vd, sizeof(*vd)); 2295 2296 mutex_enter(&sc->sc_subch_devq_lock); 2297 } 2298 } 2299 mutex_exit(&sc->sc_subch_devq_lock); 2300 2301 kthread_exit(0); 2302 } 2303 2304 2305 static int 2306 vmbus_attach_print(void *aux, const char *name) 2307 { 2308 struct vmbus_attach_args *aa = aux; 2309 2310 if (name) 2311 printf("\"%s\" at %s", aa->aa_ident, name); 2312 2313 return UNCONF; 2314 } 2315 2316 MODULE(MODULE_CLASS_DRIVER, vmbus, "hyperv"); 2317 2318 #ifdef _MODULE 2319 #include "ioconf.c" 2320 #endif 2321 2322 static int 2323 vmbus_modcmd(modcmd_t cmd, void *aux) 2324 { 2325 int rv = 0; 2326 2327 switch (cmd) { 2328 case MODULE_CMD_INIT: 2329 #ifdef _MODULE 2330 rv = config_init_component(cfdriver_ioconf_vmbus, 2331 cfattach_ioconf_vmbus, cfdata_ioconf_vmbus); 2332 #endif 2333 break; 2334 2335 case MODULE_CMD_FINI: 2336 #ifdef _MODULE 2337 rv = config_fini_component(cfdriver_ioconf_vmbus, 2338 cfattach_ioconf_vmbus, cfdata_ioconf_vmbus); 2339 #endif 2340 break; 2341 2342 default: 2343 rv = ENOTTY; 2344 break; 2345 } 2346 2347 return rv; 2348 } 2349