1 /*- 2 * Copyright (c) 2009-2012 Microsoft Corp. 3 * Copyright (c) 2012 NetApp Inc. 4 * Copyright (c) 2012 Citrix Inc. 5 * Copyright (c) 2016 Mike Belopuhov <mike@esdenera.com> 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice unmodified, this list of conditions, and the following 13 * disclaimer. 14 * 2. Redistributions in binary form must reproduce the above copyright 15 * notice, this list of conditions and the following disclaimer in the 16 * documentation and/or other materials provided with the distribution. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, BUT 23 * NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, 24 * DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED AND ON ANY 25 * THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, OR TORT 26 * (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY OUT OF THE USE OF 27 * THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF SUCH DAMAGE. 28 */ 29 30 /* 31 * The OpenBSD port was done under funding by Esdenera Networks GmbH. 32 */ 33 34 #include <sys/param.h> 35 36 /* Hyperv requires locked atomic operations */ 37 #ifndef MULTIPROCESSOR 38 #define _HYPERVMPATOMICS 39 #define MULTIPROCESSOR 40 #endif 41 #include <sys/atomic.h> 42 #ifdef _HYPERVMPATOMICS 43 #undef MULTIPROCESSOR 44 #undef _HYPERVMPATOMICS 45 #endif 46 47 #include <sys/systm.h> 48 #include <sys/proc.h> 49 #include <sys/signal.h> 50 #include <sys/signalvar.h> 51 #include <sys/malloc.h> 52 #include <sys/kernel.h> 53 #include <sys/device.h> 54 #include <sys/timetc.h> 55 #include <sys/task.h> 56 #include <sys/syslog.h> 57 58 #include <machine/bus.h> 59 #include <machine/cpu.h> 60 #include <machine/cpufunc.h> 61 62 #include <uvm/uvm_extern.h> 63 64 #include <machine/i82489var.h> 65 66 #include <dev/rndvar.h> 67 68 #include <dev/pv/pvvar.h> 69 #include <dev/pv/pvreg.h> 70 #include <dev/pv/hypervreg.h> 71 #include <dev/pv/hypervvar.h> 72 73 /* Command submission flags */ 74 #define HCF_SLEEPOK 0x0001 /* M_WAITOK */ 75 #define HCF_NOSLEEP 0x0002 /* M_NOWAIT */ 76 #define HCF_NOREPLY 0x0004 77 78 struct hv_softc *hv_sc; 79 80 int hv_match(struct device *, void *, void *); 81 void hv_attach(struct device *, struct device *, void *); 82 void hv_deferred(void *); 83 void hv_set_version(struct hv_softc *); 84 u_int hv_gettime(struct timecounter *); 85 int hv_init_hypercall(struct hv_softc *); 86 uint64_t hv_hypercall(struct hv_softc *, uint64_t, void *, void *); 87 int hv_init_interrupts(struct hv_softc *); 88 int hv_init_synic(struct hv_softc *); 89 int hv_cmd(struct hv_softc *, void *, size_t, void *, size_t, int); 90 int hv_start(struct hv_softc *, struct hv_msg *); 91 int hv_reply(struct hv_softc *, struct hv_msg *); 92 void hv_wait(struct hv_softc *, int (*done)(struct hv_softc *, 93 struct hv_msg *), struct hv_msg *, void *, const char *); 94 uint16_t hv_intr_signal(struct hv_softc *, void *); 95 void hv_intr(void); 96 void hv_event_intr(struct hv_softc *); 97 void hv_message_intr(struct hv_softc *); 98 int hv_vmbus_connect(struct hv_softc *); 99 void hv_channel_response(struct hv_softc *, struct vmbus_chanmsg_hdr *); 100 void hv_channel_offer(struct hv_softc *, struct vmbus_chanmsg_hdr *); 101 void hv_channel_delivered(struct hv_softc *, struct vmbus_chanmsg_hdr *); 102 int hv_channel_scan(struct hv_softc *); 103 void hv_process_offer(struct hv_softc *, struct hv_offer *); 104 struct hv_channel * 105 hv_channel_lookup(struct hv_softc *, uint32_t); 106 int hv_channel_ring_create(struct hv_channel *, uint32_t); 107 void hv_channel_ring_destroy(struct hv_channel *); 108 extern void hv_attach_icdevs(struct hv_softc *); 109 int hv_attach_devices(struct hv_softc *); 110 111 struct { 112 int hmd_response; 113 int hmd_request; 114 void (*hmd_handler)(struct hv_softc *, 115 struct vmbus_chanmsg_hdr *); 116 } hv_msg_dispatch[] = { 117 { 0, 0, NULL }, 118 { VMBUS_CHANMSG_CHOFFER, 0, hv_channel_offer }, 119 { VMBUS_CHANMSG_CHRESCIND, 0, NULL }, 120 { VMBUS_CHANMSG_CHREQUEST, VMBUS_CHANMSG_CHOFFER, 121 NULL }, 122 { VMBUS_CHANMSG_CHOFFER_DONE, 0, 123 hv_channel_delivered }, 124 { VMBUS_CHANMSG_CHOPEN, 0, NULL }, 125 { VMBUS_CHANMSG_CHOPEN_RESP, VMBUS_CHANMSG_CHOPEN, 126 hv_channel_response }, 127 { VMBUS_CHANMSG_CHCLOSE, 0, NULL }, 128 { VMBUS_CHANMSG_GPADL_CONN, 0, NULL }, 129 { VMBUS_CHANMSG_GPADL_SUBCONN, 0, NULL }, 130 { VMBUS_CHANMSG_GPADL_CONNRESP, VMBUS_CHANMSG_GPADL_CONN, 131 hv_channel_response }, 132 { VMBUS_CHANMSG_GPADL_DISCONN, 0, NULL }, 133 { VMBUS_CHANMSG_GPADL_DISCONNRESP, VMBUS_CHANMSG_GPADL_DISCONN, 134 hv_channel_response }, 135 { VMBUS_CHANMSG_CHFREE, 0, NULL }, 136 { VMBUS_CHANMSG_CONNECT, 0, NULL }, 137 { VMBUS_CHANMSG_CONNECT_RESP, VMBUS_CHANMSG_CONNECT, 138 hv_channel_response }, 139 { VMBUS_CHANMSG_DISCONNECT, 0, NULL }, 140 }; 141 142 struct timecounter hv_timecounter = { 143 hv_gettime, 0, 0xffffffff, 10000000, "hyperv", 9001 144 }; 145 146 struct cfdriver hyperv_cd = { 147 NULL, "hyperv", DV_DULL 148 }; 149 150 const struct cfattach hyperv_ca = { 151 sizeof(struct hv_softc), hv_match, hv_attach 152 }; 153 154 const struct hv_guid hv_guid_network = { 155 { 0x63, 0x51, 0x61, 0xf8, 0x3e, 0xdf, 0xc5, 0x46, 156 0x91, 0x3f, 0xf2, 0xd2, 0xf9, 0x65, 0xed, 0x0e } 157 }; 158 159 const struct hv_guid hv_guid_ide = { 160 { 0x32, 0x26, 0x41, 0x32, 0xcb, 0x86, 0xa2, 0x44, 161 0x9b, 0x5c, 0x50, 0xd1, 0x41, 0x73, 0x54, 0xf5 } 162 }; 163 164 const struct hv_guid hv_guid_scsi = { 165 { 0xd9, 0x63, 0x61, 0xba, 0xa1, 0x04, 0x29, 0x4d, 166 0xb6, 0x05, 0x72, 0xe2, 0xff, 0xb1, 0xdc, 0x7f } 167 }; 168 169 const struct hv_guid hv_guid_shutdown = { 170 { 0x31, 0x60, 0x0b, 0x0e, 0x13, 0x52, 0x34, 0x49, 171 0x81, 0x8b, 0x38, 0xd9, 0x0c, 0xed, 0x39, 0xdb } 172 }; 173 174 const struct hv_guid hv_guid_timesync = { 175 { 0x30, 0xe6, 0x27, 0x95, 0xae, 0xd0, 0x7b, 0x49, 176 0xad, 0xce, 0xe8, 0x0a, 0xb0, 0x17, 0x5c, 0xaf } 177 }; 178 179 const struct hv_guid hv_guid_heartbeat = { 180 { 0x39, 0x4f, 0x16, 0x57, 0x15, 0x91, 0x78, 0x4e, 181 0xab, 0x55, 0x38, 0x2f, 0x3b, 0xd5, 0x42, 0x2d } 182 }; 183 184 const struct hv_guid hv_guid_kvp = { 185 { 0xe7, 0xf4, 0xa0, 0xa9, 0x45, 0x5a, 0x96, 0x4d, 186 0xb8, 0x27, 0x8a, 0x84, 0x1e, 0x8c, 0x03, 0xe6 } 187 }; 188 189 #ifdef HYPERV_DEBUG 190 const struct hv_guid hv_guid_vss = { 191 { 0x29, 0x2e, 0xfa, 0x35, 0x23, 0xea, 0x36, 0x42, 192 0x96, 0xae, 0x3a, 0x6e, 0xba, 0xcb, 0xa4, 0x40 } 193 }; 194 195 const struct hv_guid hv_guid_dynmem = { 196 { 0xdc, 0x74, 0x50, 0x52, 0x85, 0x89, 0xe2, 0x46, 197 0x80, 0x57, 0xa3, 0x07, 0xdc, 0x18, 0xa5, 0x02 } 198 }; 199 200 const struct hv_guid hv_guid_mouse = { 201 { 0x9e, 0xb6, 0xa8, 0xcf, 0x4a, 0x5b, 0xc0, 0x4c, 202 0xb9, 0x8b, 0x8b, 0xa1, 0xa1, 0xf3, 0xf9, 0x5a } 203 }; 204 205 const struct hv_guid hv_guid_kbd = { 206 { 0x6d, 0xad, 0x12, 0xf9, 0x17, 0x2b, 0xea, 0x48, 207 0xbd, 0x65, 0xf9, 0x27, 0xa6, 0x1c, 0x76, 0x84 } 208 }; 209 210 const struct hv_guid hv_guid_video = { 211 { 0x02, 0x78, 0x0a, 0xda, 0x77, 0xe3, 0xac, 0x4a, 212 0x8e, 0x77, 0x05, 0x58, 0xeb, 0x10, 0x73, 0xf8 } 213 }; 214 215 const struct hv_guid hv_guid_fc = { 216 { 0x4a, 0xcc, 0x9b, 0x2f, 0x69, 0x00, 0xf3, 0x4a, 217 0xb7, 0x6b, 0x6f, 0xd0, 0xbe, 0x52, 0x8c, 0xda } 218 }; 219 220 const struct hv_guid hv_guid_fcopy = { 221 { 0xe3, 0x4b, 0xd1, 0x34, 0xe4, 0xde, 0xc8, 0x41, 222 0x9a, 0xe7, 0x6b, 0x17, 0x49, 0x77, 0xc1, 0x92 } 223 }; 224 225 const struct hv_guid hv_guid_pcie = { 226 { 0x1d, 0xf6, 0xc4, 0x44, 0x44, 0x44, 0x00, 0x44, 227 0x9d, 0x52, 0x80, 0x2e, 0x27, 0xed, 0xe1, 0x9f } 228 }; 229 230 const struct hv_guid hv_guid_netdir = { 231 { 0x3d, 0xaf, 0x2e, 0x8c, 0xa7, 0x32, 0x09, 0x4b, 232 0xab, 0x99, 0xbd, 0x1f, 0x1c, 0x86, 0xb5, 0x01 } 233 }; 234 235 const struct hv_guid hv_guid_rdesktop = { 236 { 0xf4, 0xac, 0x6a, 0x27, 0x15, 0xac, 0x6c, 0x42, 237 0x98, 0xdd, 0x75, 0x21, 0xad, 0x3f, 0x01, 0xfe } 238 }; 239 240 /* Automatic Virtual Machine Activation (AVMA) Services */ 241 const struct hv_guid hv_guid_avma1 = { 242 { 0x55, 0xb2, 0x87, 0x44, 0x8c, 0xb8, 0x3f, 0x40, 243 0xbb, 0x51, 0xd1, 0xf6, 0x9c, 0xf1, 0x7f, 0x87 } 244 }; 245 246 const struct hv_guid hv_guid_avma2 = { 247 { 0xf4, 0xba, 0x75, 0x33, 0x15, 0x9e, 0x30, 0x4b, 248 0xb7, 0x65, 0x67, 0xac, 0xb1, 0x0d, 0x60, 0x7b } 249 }; 250 251 const struct hv_guid hv_guid_avma3 = { 252 { 0xa0, 0x1f, 0x22, 0x99, 0xad, 0x24, 0xe2, 0x11, 253 0xbe, 0x98, 0x00, 0x1a, 0xa0, 0x1b, 0xbf, 0x6e } 254 }; 255 256 const struct hv_guid hv_guid_avma4 = { 257 { 0x16, 0x57, 0xe6, 0xf8, 0xb3, 0x3c, 0x06, 0x4a, 258 0x9a, 0x60, 0x18, 0x89, 0xc5, 0xcc, 0xca, 0xb5 } 259 }; 260 #endif /* HYPERV_DEBUG */ 261 262 int 263 hv_match(struct device *parent, void *match, void *aux) 264 { 265 struct pv_attach_args *pva = aux; 266 struct pvbus_hv *hv = &pva->pva_hv[PVBUS_HYPERV]; 267 268 if ((hv->hv_major == 0 && hv->hv_minor == 0) || hv->hv_base == 0) 269 return (0); 270 271 return (1); 272 } 273 274 void 275 hv_attach(struct device *parent, struct device *self, void *aux) 276 { 277 struct hv_softc *sc = (struct hv_softc *)self; 278 struct pv_attach_args *pva = aux; 279 struct pvbus_hv *hv = &pva->pva_hv[PVBUS_HYPERV]; 280 281 sc->sc_pvbus = hv; 282 sc->sc_dmat = pva->pva_dmat; 283 284 if (!(hv->hv_features & CPUID_HV_MSR_HYPERCALL) || 285 !(hv->hv_features & CPUID_HV_MSR_SYNIC)) { 286 printf(": not functional\n"); 287 return; 288 } 289 290 DPRINTF("\n"); 291 292 hv_set_version(sc); 293 294 if (hv->hv_features & CPUID_HV_MSR_TIME_REFCNT) 295 tc_init(&hv_timecounter); 296 297 if (hv_init_hypercall(sc)) 298 return; 299 300 /* Wire it up to the global */ 301 hv_sc = sc; 302 303 if (hv_init_interrupts(sc)) 304 return; 305 306 if (hv_vmbus_connect(sc)) 307 return; 308 309 DPRINTF("%s", sc->sc_dev.dv_xname); 310 printf(": protocol %d.%d, features %#x\n", 311 VMBUS_VERSION_MAJOR(sc->sc_proto), 312 VMBUS_VERSION_MINOR(sc->sc_proto), 313 hv->hv_features); 314 315 if (hv_channel_scan(sc)) 316 return; 317 318 /* Attach heartbeat, KVP and other "internal" services */ 319 hv_attach_icdevs(sc); 320 321 startuphook_establish(hv_deferred, sc); 322 } 323 324 void 325 hv_deferred(void *arg) 326 { 327 struct hv_softc *sc = arg; 328 329 if (hv_attach_devices(sc)) 330 return; 331 } 332 333 void 334 hv_set_version(struct hv_softc *sc) 335 { 336 uint64_t ver; 337 338 /* OpenBSD build date */ 339 ver = MSR_HV_GUESTID_OSTYPE_OPENBSD; 340 ver |= (uint64_t)OpenBSD << MSR_HV_GUESTID_VERSION_SHIFT; 341 wrmsr(MSR_HV_GUEST_OS_ID, ver); 342 } 343 344 u_int 345 hv_gettime(struct timecounter *tc) 346 { 347 u_int now = rdmsr(MSR_HV_TIME_REF_COUNT); 348 349 return (now); 350 } 351 352 int 353 hv_init_hypercall(struct hv_softc *sc) 354 { 355 extern void *hv_hypercall_page; 356 uint64_t msr; 357 paddr_t pa; 358 359 sc->sc_hc = &hv_hypercall_page; 360 361 if (!pmap_extract(pmap_kernel(), (vaddr_t)sc->sc_hc, &pa)) { 362 printf(": hypercall page PA extraction failed\n"); 363 return (-1); 364 } 365 366 msr = (atop(pa) << MSR_HV_HYPERCALL_PGSHIFT) | MSR_HV_HYPERCALL_ENABLE; 367 wrmsr(MSR_HV_HYPERCALL, msr); 368 369 if (!(rdmsr(MSR_HV_HYPERCALL) & MSR_HV_HYPERCALL_ENABLE)) { 370 printf(": failed to set up a hypercall page\n"); 371 return (-1); 372 } 373 374 return (0); 375 } 376 377 uint64_t 378 hv_hypercall(struct hv_softc *sc, uint64_t control, void *input, 379 void *output) 380 { 381 paddr_t input_pa = 0, output_pa = 0; 382 uint64_t status = 0; 383 384 if (input != NULL && 385 pmap_extract(pmap_kernel(), (vaddr_t)input, &input_pa) == 0) { 386 printf("%s: hypercall input PA extraction failed\n", 387 sc->sc_dev.dv_xname); 388 return (~HYPERCALL_STATUS_SUCCESS); 389 } 390 391 if (output != NULL && 392 pmap_extract(pmap_kernel(), (vaddr_t)output, &output_pa) == 0) { 393 printf("%s: hypercall output PA extraction failed\n", 394 sc->sc_dev.dv_xname); 395 return (~HYPERCALL_STATUS_SUCCESS); 396 } 397 398 #ifdef __amd64__ 399 __asm__ __volatile__ ("mov %0, %%r8" : : "r" (output_pa) : "r8"); 400 __asm__ __volatile__ ("call *%3" : "=a" (status) : "c" (control), 401 "d" (input_pa), "m" (sc->sc_hc)); 402 #else /* __i386__ */ 403 { 404 uint32_t control_hi = control >> 32; 405 uint32_t control_lo = control & 0xfffffffff; 406 uint32_t status_hi = 1; 407 uint32_t status_lo = 1; 408 409 __asm__ __volatile__ ("call *%8" : 410 "=d" (status_hi), "=a"(status_lo) : 411 "d" (control_hi), "a" (control_lo), 412 "b" (0), "c" (input_pa), "D" (0), "S" (output_pa), 413 "m" (sc->sc_hc)); 414 415 status = status_lo | ((uint64_t)status_hi << 32); 416 } 417 #endif /* __amd64__ */ 418 419 return (status); 420 } 421 422 int 423 hv_init_interrupts(struct hv_softc *sc) 424 { 425 struct cpu_info *ci = curcpu(); 426 int cpu = CPU_INFO_UNIT(ci); 427 428 sc->sc_idtvec = LAPIC_HYPERV_VECTOR; 429 430 TAILQ_INIT(&sc->sc_reqs); 431 mtx_init(&sc->sc_reqlck, IPL_NET); 432 433 TAILQ_INIT(&sc->sc_rsps); 434 mtx_init(&sc->sc_rsplck, IPL_NET); 435 436 sc->sc_simp[cpu] = km_alloc(PAGE_SIZE, &kv_any, &kp_zero, &kd_nowait); 437 if (sc->sc_simp[cpu] == NULL) { 438 printf(": failed to allocate SIMP\n"); 439 return (-1); 440 } 441 442 sc->sc_siep[cpu] = km_alloc(PAGE_SIZE, &kv_any, &kp_zero, &kd_nowait); 443 if (sc->sc_siep[cpu] == NULL) { 444 printf(": failed to allocate SIEP\n"); 445 km_free(sc->sc_simp[cpu], PAGE_SIZE, &kv_any, &kp_zero); 446 return (-1); 447 } 448 449 sc->sc_proto = VMBUS_VERSION_WS2008; 450 451 return (hv_init_synic(sc)); 452 } 453 454 int 455 hv_init_synic(struct hv_softc *sc) 456 { 457 struct cpu_info *ci = curcpu(); 458 int cpu = CPU_INFO_UNIT(ci); 459 uint64_t simp, siefp, sctrl, sint; 460 paddr_t pa; 461 462 /* 463 * Setup the Synic's message page 464 */ 465 if (!pmap_extract(pmap_kernel(), (vaddr_t)sc->sc_simp[cpu], &pa)) { 466 printf(": SIMP PA extraction failed\n"); 467 return (-1); 468 } 469 simp = rdmsr(MSR_HV_SIMP); 470 simp &= (1 << MSR_HV_SIMP_PGSHIFT) - 1; 471 simp |= (atop(pa) << MSR_HV_SIMP_PGSHIFT); 472 simp |= MSR_HV_SIMP_ENABLE; 473 wrmsr(MSR_HV_SIMP, simp); 474 475 /* 476 * Setup the Synic's event page 477 */ 478 if (!pmap_extract(pmap_kernel(), (vaddr_t)sc->sc_siep[cpu], &pa)) { 479 printf(": SIEP PA extraction failed\n"); 480 return (-1); 481 } 482 siefp = rdmsr(MSR_HV_SIEFP); 483 siefp &= (1<<MSR_HV_SIEFP_PGSHIFT) - 1; 484 siefp |= (atop(pa) << MSR_HV_SIEFP_PGSHIFT); 485 siefp |= MSR_HV_SIEFP_ENABLE; 486 wrmsr(MSR_HV_SIEFP, siefp); 487 488 /* 489 * Configure and unmask SINT for message and event flags 490 */ 491 sint = rdmsr(MSR_HV_SINT0 + VMBUS_SINT_MESSAGE); 492 sint = sc->sc_idtvec | MSR_HV_SINT_AUTOEOI | 493 (sint & MSR_HV_SINT_RSVD_MASK); 494 wrmsr(MSR_HV_SINT0 + VMBUS_SINT_MESSAGE, sint); 495 496 /* Enable the global synic bit */ 497 sctrl = rdmsr(MSR_HV_SCONTROL); 498 sctrl |= MSR_HV_SCTRL_ENABLE; 499 wrmsr(MSR_HV_SCONTROL, sctrl); 500 501 sc->sc_vcpus[cpu] = rdmsr(MSR_HV_VP_INDEX); 502 503 DPRINTF("vcpu%u: SIMP %#llx SIEFP %#llx SCTRL %#llx\n", 504 sc->sc_vcpus[cpu], simp, siefp, sctrl); 505 506 return (0); 507 } 508 509 int 510 hv_cmd(struct hv_softc *sc, void *cmd, size_t cmdlen, void *rsp, 511 size_t rsplen, int flags) 512 { 513 struct hv_msg msg; 514 int rv; 515 516 if (cmdlen > VMBUS_MSG_DSIZE_MAX) { 517 printf("%s: payload too large (%lu)\n", sc->sc_dev.dv_xname, 518 cmdlen); 519 return (EMSGSIZE); 520 } 521 522 memset(&msg, 0, sizeof(msg)); 523 524 msg.msg_req.hc_dsize = cmdlen; 525 memcpy(msg.msg_req.hc_data, cmd, cmdlen); 526 527 if (!(flags & HCF_NOREPLY)) { 528 msg.msg_rsp = rsp; 529 msg.msg_rsplen = rsplen; 530 } else 531 msg.msg_flags |= MSGF_NOQUEUE; 532 533 if (flags & HCF_NOSLEEP) 534 msg.msg_flags |= MSGF_NOSLEEP; 535 536 if ((rv = hv_start(sc, &msg)) != 0) 537 return (rv); 538 return (hv_reply(sc, &msg)); 539 } 540 541 int 542 hv_start(struct hv_softc *sc, struct hv_msg *msg) 543 { 544 const int delays[] = { 100, 100, 100, 500, 500, 5000, 5000, 5000 }; 545 const char *wchan = "hvstart"; 546 uint16_t status; 547 int i, s; 548 549 msg->msg_req.hc_connid = VMBUS_CONNID_MESSAGE; 550 msg->msg_req.hc_msgtype = 1; 551 552 if (!(msg->msg_flags & MSGF_NOQUEUE)) { 553 mtx_enter(&sc->sc_reqlck); 554 TAILQ_INSERT_TAIL(&sc->sc_reqs, msg, msg_entry); 555 mtx_leave(&sc->sc_reqlck); 556 } 557 558 for (i = 0; i < nitems(delays); i++) { 559 status = hv_hypercall(sc, HYPERCALL_POST_MESSAGE, 560 &msg->msg_req, NULL); 561 if (status == HYPERCALL_STATUS_SUCCESS) 562 break; 563 if (msg->msg_flags & MSGF_NOSLEEP) { 564 delay(delays[i]); 565 s = splnet(); 566 hv_intr(); 567 splx(s); 568 } else 569 tsleep(wchan, PRIBIO, wchan, 1); 570 } 571 if (status != 0) { 572 printf("%s: posting vmbus message failed with %d\n", 573 sc->sc_dev.dv_xname, status); 574 if (!(msg->msg_flags & MSGF_NOQUEUE)) { 575 mtx_enter(&sc->sc_reqlck); 576 TAILQ_REMOVE(&sc->sc_reqs, msg, msg_entry); 577 mtx_leave(&sc->sc_reqlck); 578 } 579 return (EIO); 580 } 581 582 return (0); 583 } 584 585 static int 586 hv_reply_done(struct hv_softc *sc, struct hv_msg *msg) 587 { 588 struct hv_msg *m; 589 590 mtx_enter(&sc->sc_rsplck); 591 TAILQ_FOREACH(m, &sc->sc_rsps, msg_entry) { 592 if (m == msg) { 593 mtx_leave(&sc->sc_rsplck); 594 return (1); 595 } 596 } 597 mtx_leave(&sc->sc_rsplck); 598 return (0); 599 } 600 601 int 602 hv_reply(struct hv_softc *sc, struct hv_msg *msg) 603 { 604 if (msg->msg_flags & MSGF_NOQUEUE) 605 return (0); 606 607 hv_wait(sc, hv_reply_done, msg, msg, "hvreply"); 608 609 mtx_enter(&sc->sc_rsplck); 610 TAILQ_REMOVE(&sc->sc_rsps, msg, msg_entry); 611 mtx_leave(&sc->sc_rsplck); 612 613 return (0); 614 } 615 616 void 617 hv_wait(struct hv_softc *sc, int (*cond)(struct hv_softc *, struct hv_msg *), 618 struct hv_msg *msg, void *wchan, const char *wmsg) 619 { 620 int s; 621 622 KASSERT(cold ? msg->msg_flags & MSGF_NOSLEEP : 1); 623 624 while (!cond(sc, msg)) { 625 if (msg->msg_flags & MSGF_NOSLEEP) { 626 delay(1000); 627 s = splnet(); 628 hv_intr(); 629 splx(s); 630 } else 631 tsleep(wchan, PRIBIO, wmsg ? wmsg : "hvwait", 1); 632 } 633 } 634 635 uint16_t 636 hv_intr_signal(struct hv_softc *sc, void *con) 637 { 638 uint64_t status; 639 640 status = hv_hypercall(sc, HYPERCALL_SIGNAL_EVENT, con, NULL); 641 return ((uint16_t)status); 642 } 643 644 void 645 hv_intr(void) 646 { 647 struct hv_softc *sc = hv_sc; 648 649 hv_event_intr(sc); 650 hv_message_intr(sc); 651 } 652 653 void 654 hv_event_intr(struct hv_softc *sc) 655 { 656 struct vmbus_evtflags *evt; 657 struct cpu_info *ci = curcpu(); 658 int cpu = CPU_INFO_UNIT(ci); 659 int bit, row, maxrow, chanid; 660 struct hv_channel *ch; 661 u_long *revents, pending; 662 663 evt = (struct vmbus_evtflags *)sc->sc_siep[cpu] + 664 VMBUS_SINT_MESSAGE; 665 if ((sc->sc_proto == VMBUS_VERSION_WS2008) || 666 (sc->sc_proto == VMBUS_VERSION_WIN7)) { 667 if (!test_bit(0, &evt->evt_flags[0])) 668 return; 669 clear_bit(0, &evt->evt_flags[0]); 670 maxrow = VMBUS_CHAN_MAX_COMPAT / VMBUS_EVTFLAG_LEN; 671 /* 672 * receive size is 1/2 page and divide that by 4 bytes 673 */ 674 revents = sc->sc_revents; 675 } else { 676 maxrow = nitems(evt->evt_flags); 677 /* 678 * On Host with Win8 or above, the event page can be 679 * checked directly to get the id of the channel 680 * that has the pending interrupt. 681 */ 682 revents = &evt->evt_flags[0]; 683 } 684 685 for (row = 0; row < maxrow; row++) { 686 if (revents[row] == 0) 687 continue; 688 pending = atomic_swap_ulong(&revents[row], 0); 689 for (bit = 0; pending > 0; pending >>= 1, bit++) { 690 if ((pending & 1) == 0) 691 continue; 692 chanid = (row * LONG_BIT) + bit; 693 /* vmbus channel protocol message */ 694 if (chanid == 0) 695 continue; 696 ch = hv_channel_lookup(sc, chanid); 697 if (ch == NULL) { 698 printf("%s: unhandled event on %d\n", 699 sc->sc_dev.dv_xname, chanid); 700 continue; 701 } 702 if (ch->ch_state != HV_CHANSTATE_OPENED) { 703 printf("%s: channel %d is not active\n", 704 sc->sc_dev.dv_xname, chanid); 705 continue; 706 } 707 ch->ch_evcnt.ec_count++; 708 if (ch->ch_handler) 709 ch->ch_handler(ch->ch_ctx); 710 } 711 } 712 } 713 714 void 715 hv_message_intr(struct hv_softc *sc) 716 { 717 struct vmbus_message *msg; 718 struct vmbus_chanmsg_hdr *hdr; 719 struct cpu_info *ci = curcpu(); 720 int cpu = CPU_INFO_UNIT(ci); 721 722 for (;;) { 723 msg = (struct vmbus_message *)sc->sc_simp[cpu] + 724 VMBUS_SINT_MESSAGE; 725 if (msg->msg_type == VMBUS_MSGTYPE_NONE) 726 break; 727 728 hdr = (struct vmbus_chanmsg_hdr *)msg->msg_data; 729 if (hdr->chm_type >= VMBUS_CHANMSG_COUNT) { 730 printf("%s: unhandled message type %u flags %#x\n", 731 sc->sc_dev.dv_xname, hdr->chm_type, 732 msg->msg_flags); 733 goto skip; 734 } 735 if (hv_msg_dispatch[hdr->chm_type].hmd_handler) 736 hv_msg_dispatch[hdr->chm_type].hmd_handler(sc, hdr); 737 else 738 printf("%s: unhandled message type %u\n", 739 sc->sc_dev.dv_xname, hdr->chm_type); 740 skip: 741 msg->msg_type = VMBUS_MSGTYPE_NONE; 742 membar_sync(); 743 if (msg->msg_flags & VMBUS_MSGFLAG_PENDING) 744 wrmsr(MSR_HV_EOM, 0); 745 } 746 } 747 748 void 749 hv_channel_response(struct hv_softc *sc, struct vmbus_chanmsg_hdr *rsphdr) 750 { 751 struct hv_msg *msg; 752 struct vmbus_chanmsg_hdr *reqhdr; 753 int req; 754 755 req = hv_msg_dispatch[rsphdr->chm_type].hmd_request; 756 mtx_enter(&sc->sc_reqlck); 757 TAILQ_FOREACH(msg, &sc->sc_reqs, msg_entry) { 758 reqhdr = (struct vmbus_chanmsg_hdr *)&msg->msg_req.hc_data; 759 if (reqhdr->chm_type == req) { 760 TAILQ_REMOVE(&sc->sc_reqs, msg, msg_entry); 761 break; 762 } 763 } 764 mtx_leave(&sc->sc_reqlck); 765 if (msg != NULL) { 766 memcpy(msg->msg_rsp, rsphdr, msg->msg_rsplen); 767 mtx_enter(&sc->sc_rsplck); 768 TAILQ_INSERT_TAIL(&sc->sc_rsps, msg, msg_entry); 769 mtx_leave(&sc->sc_rsplck); 770 wakeup(msg); 771 } 772 } 773 774 void 775 hv_channel_offer(struct hv_softc *sc, struct vmbus_chanmsg_hdr *hdr) 776 { 777 struct hv_offer *co; 778 779 co = malloc(sizeof(*co), M_DEVBUF, M_NOWAIT | M_ZERO); 780 if (co == NULL) { 781 printf("%s: failed to allocate an offer object\n", 782 sc->sc_dev.dv_xname); 783 return; 784 } 785 786 memcpy(&co->co_chan, hdr, sizeof(co->co_chan)); 787 788 mtx_enter(&sc->sc_offerlck); 789 SIMPLEQ_INSERT_TAIL(&sc->sc_offers, co, co_entry); 790 mtx_leave(&sc->sc_offerlck); 791 } 792 793 void 794 hv_channel_delivered(struct hv_softc *sc, struct vmbus_chanmsg_hdr *hdr) 795 { 796 atomic_setbits_int(&sc->sc_flags, HSF_OFFERS_DELIVERED); 797 wakeup(&sc->sc_offers); 798 } 799 800 int 801 hv_vmbus_connect(struct hv_softc *sc) 802 { 803 const uint32_t versions[] = { 804 VMBUS_VERSION_WIN8_1, VMBUS_VERSION_WIN8, 805 VMBUS_VERSION_WIN7, VMBUS_VERSION_WS2008 806 }; 807 struct vmbus_chanmsg_connect cmd; 808 struct vmbus_chanmsg_connect_resp rsp; 809 paddr_t epa, mpa1, mpa2; 810 int i; 811 812 sc->sc_events = km_alloc(PAGE_SIZE, &kv_any, &kp_zero, &kd_nowait); 813 if (sc->sc_events == NULL) { 814 printf(": failed to allocate channel port events page\n"); 815 goto errout; 816 } 817 if (!pmap_extract(pmap_kernel(), (vaddr_t)sc->sc_events, &epa)) { 818 printf(": channel port events page PA extraction failed\n"); 819 goto errout; 820 } 821 822 sc->sc_wevents = (u_long *)sc->sc_events; 823 sc->sc_revents = (u_long *)((caddr_t)sc->sc_events + (PAGE_SIZE >> 1)); 824 825 sc->sc_monitor[0] = km_alloc(PAGE_SIZE, &kv_any, &kp_zero, &kd_nowait); 826 if (sc->sc_monitor[0] == NULL) { 827 printf(": failed to allocate monitor page 1\n"); 828 goto errout; 829 } 830 if (!pmap_extract(pmap_kernel(), (vaddr_t)sc->sc_monitor[0], &mpa1)) { 831 printf(": monitor page 1 PA extraction failed\n"); 832 goto errout; 833 } 834 835 sc->sc_monitor[1] = km_alloc(PAGE_SIZE, &kv_any, &kp_zero, &kd_nowait); 836 if (sc->sc_monitor[1] == NULL) { 837 printf(": failed to allocate monitor page 2\n"); 838 goto errout; 839 } 840 if (!pmap_extract(pmap_kernel(), (vaddr_t)sc->sc_monitor[1], &mpa2)) { 841 printf(": monitor page 2 PA extraction failed\n"); 842 goto errout; 843 } 844 845 memset(&cmd, 0, sizeof(cmd)); 846 cmd.chm_hdr.chm_type = VMBUS_CHANMSG_CONNECT; 847 cmd.chm_evtflags = (uint64_t)epa; 848 cmd.chm_mnf1 = (uint64_t)mpa1; 849 cmd.chm_mnf2 = (uint64_t)mpa2; 850 851 memset(&rsp, 0, sizeof(rsp)); 852 853 for (i = 0; i < nitems(versions); i++) { 854 cmd.chm_ver = versions[i]; 855 if (hv_cmd(sc, &cmd, sizeof(cmd), &rsp, sizeof(rsp), 856 HCF_NOSLEEP)) { 857 DPRINTF("%s: CONNECT failed\n", 858 sc->sc_dev.dv_xname); 859 goto errout; 860 } 861 if (rsp.chm_done) { 862 sc->sc_flags |= HSF_CONNECTED; 863 sc->sc_proto = versions[i]; 864 sc->sc_handle = VMBUS_GPADL_START; 865 break; 866 } 867 } 868 if (i == nitems(versions)) { 869 printf("%s: failed to negotiate protocol version\n", 870 sc->sc_dev.dv_xname); 871 goto errout; 872 } 873 874 return (0); 875 876 errout: 877 if (sc->sc_events) { 878 km_free(sc->sc_events, PAGE_SIZE, &kv_any, &kp_zero); 879 sc->sc_events = NULL; 880 sc->sc_wevents = NULL; 881 sc->sc_revents = NULL; 882 } 883 if (sc->sc_monitor[0]) { 884 km_free(sc->sc_monitor[0], PAGE_SIZE, &kv_any, &kp_zero); 885 sc->sc_monitor[0] = NULL; 886 } 887 if (sc->sc_monitor[1]) { 888 km_free(sc->sc_monitor[1], PAGE_SIZE, &kv_any, &kp_zero); 889 sc->sc_monitor[1] = NULL; 890 } 891 return (-1); 892 } 893 894 #ifdef HYPERV_DEBUG 895 static inline char * 896 guidprint(struct hv_guid *a) 897 { 898 /* 3 0 5 4 7 6 8 9 10 15 */ 899 /* 33221100-5544-7766-9988-FFEEDDCCBBAA */ 900 static char buf[16 * 2 + 4 + 1]; 901 int i, j = 0; 902 903 for (i = 3; i != -1; i -= 1, j += 2) 904 snprintf(&buf[j], 3, "%02x", (uint8_t)a->data[i]); 905 buf[j++] = '-'; 906 for (i = 5; i != 3; i -= 1, j += 2) 907 snprintf(&buf[j], 3, "%02x", (uint8_t)a->data[i]); 908 buf[j++] = '-'; 909 for (i = 7; i != 5; i -= 1, j += 2) 910 snprintf(&buf[j], 3, "%02x", (uint8_t)a->data[i]); 911 buf[j++] = '-'; 912 for (i = 8; i < 10; i += 1, j += 2) 913 snprintf(&buf[j], 3, "%02x", (uint8_t)a->data[i]); 914 buf[j++] = '-'; 915 for (i = 10; i < 16; i += 1, j += 2) 916 snprintf(&buf[j], 3, "%02x", (uint8_t)a->data[i]); 917 return (&buf[0]); 918 } 919 #endif /* HYPERV_DEBUG */ 920 921 void 922 hv_guid_sprint(struct hv_guid *guid, char *str, size_t size) 923 { 924 const struct { 925 const struct hv_guid *guid; 926 const char *ident; 927 } map[] = { 928 { &hv_guid_network, "network" }, 929 { &hv_guid_ide, "ide" }, 930 { &hv_guid_scsi, "scsi" }, 931 { &hv_guid_shutdown, "shutdown" }, 932 { &hv_guid_timesync, "timesync" }, 933 { &hv_guid_heartbeat, "heartbeat" }, 934 { &hv_guid_kvp, "kvp" }, 935 #ifdef HYPERV_DEBUG 936 { &hv_guid_vss, "vss" }, 937 { &hv_guid_dynmem, "dynamic-memory" }, 938 { &hv_guid_mouse, "mouse" }, 939 { &hv_guid_kbd, "keyboard" }, 940 { &hv_guid_video, "video" }, 941 { &hv_guid_fc, "fiber-channel" }, 942 { &hv_guid_fcopy, "file-copy" }, 943 { &hv_guid_pcie, "pcie-passthrough" }, 944 { &hv_guid_netdir, "network-direct" }, 945 { &hv_guid_rdesktop, "remote-desktop" }, 946 { &hv_guid_avma1, "avma-1" }, 947 { &hv_guid_avma2, "avma-2" }, 948 { &hv_guid_avma3, "avma-3" }, 949 { &hv_guid_avma4, "avma-4" }, 950 #endif 951 }; 952 int i; 953 954 for (i = 0; i < nitems(map); i++) { 955 if (memcmp(guid, map[i].guid, sizeof(*guid)) == 0) { 956 strlcpy(str, map[i].ident, size); 957 return; 958 } 959 } 960 #ifdef HYPERV_DEBUG 961 strlcpy(str, guidprint(guid), size); 962 #endif 963 } 964 965 static int 966 hv_channel_scan_done(struct hv_softc *sc, struct hv_msg *msg __unused) 967 { 968 return (sc->sc_flags & HSF_OFFERS_DELIVERED); 969 } 970 971 int 972 hv_channel_scan(struct hv_softc *sc) 973 { 974 struct vmbus_chanmsg_hdr hdr; 975 struct vmbus_chanmsg_choffer rsp; 976 struct hv_offer *co; 977 978 SIMPLEQ_INIT(&sc->sc_offers); 979 mtx_init(&sc->sc_offerlck, IPL_NET); 980 981 memset(&hdr, 0, sizeof(hdr)); 982 hdr.chm_type = VMBUS_CHANMSG_CHREQUEST; 983 984 if (hv_cmd(sc, &hdr, sizeof(hdr), &rsp, sizeof(rsp), 985 HCF_NOSLEEP | HCF_NOREPLY)) { 986 DPRINTF("%s: CHREQUEST failed\n", sc->sc_dev.dv_xname); 987 return (-1); 988 } 989 990 hv_wait(sc, hv_channel_scan_done, (struct hv_msg *)&hdr, 991 &sc->sc_offers, "hvscan"); 992 993 TAILQ_INIT(&sc->sc_channels); 994 mtx_init(&sc->sc_channelck, IPL_NET); 995 996 mtx_enter(&sc->sc_offerlck); 997 while (!SIMPLEQ_EMPTY(&sc->sc_offers)) { 998 co = SIMPLEQ_FIRST(&sc->sc_offers); 999 SIMPLEQ_REMOVE_HEAD(&sc->sc_offers, co_entry); 1000 mtx_leave(&sc->sc_offerlck); 1001 1002 hv_process_offer(sc, co); 1003 free(co, M_DEVBUF, sizeof(*co)); 1004 1005 mtx_enter(&sc->sc_offerlck); 1006 } 1007 mtx_leave(&sc->sc_offerlck); 1008 1009 return (0); 1010 } 1011 1012 void 1013 hv_process_offer(struct hv_softc *sc, struct hv_offer *co) 1014 { 1015 struct hv_channel *ch, *nch; 1016 1017 nch = malloc(sizeof(*nch), M_DEVBUF, M_ZERO | M_NOWAIT); 1018 if (nch == NULL) { 1019 printf("%s: failed to allocate memory for the channel\n", 1020 sc->sc_dev.dv_xname); 1021 return; 1022 } 1023 nch->ch_sc = sc; 1024 hv_guid_sprint(&co->co_chan.chm_chtype, nch->ch_ident, 1025 sizeof(nch->ch_ident)); 1026 1027 /* 1028 * By default we setup state to enable batched reading. 1029 * A specific service can choose to disable this prior 1030 * to opening the channel. 1031 */ 1032 nch->ch_flags |= CHF_BATCHED; 1033 1034 KASSERT((((vaddr_t)&nch->ch_monprm) & 0x7) == 0); 1035 memset(&nch->ch_monprm, 0, sizeof(nch->ch_monprm)); 1036 nch->ch_monprm.mp_connid = VMBUS_CONNID_EVENT; 1037 1038 if (sc->sc_proto != VMBUS_VERSION_WS2008) 1039 nch->ch_monprm.mp_connid = co->co_chan.chm_connid; 1040 1041 if (co->co_chan.chm_flags1 & VMBUS_CHOFFER_FLAG1_HASMNF) { 1042 nch->ch_mgroup = co->co_chan.chm_montrig / VMBUS_MONTRIG_LEN; 1043 nch->ch_mindex = co->co_chan.chm_montrig % VMBUS_MONTRIG_LEN; 1044 nch->ch_flags |= CHF_MONITOR; 1045 } 1046 1047 nch->ch_id = co->co_chan.chm_chanid; 1048 1049 memcpy(&nch->ch_type, &co->co_chan.chm_chtype, sizeof(ch->ch_type)); 1050 memcpy(&nch->ch_inst, &co->co_chan.chm_chinst, sizeof(ch->ch_inst)); 1051 1052 mtx_enter(&sc->sc_channelck); 1053 TAILQ_FOREACH(ch, &sc->sc_channels, ch_entry) { 1054 if (!memcmp(&ch->ch_type, &nch->ch_type, sizeof(ch->ch_type)) && 1055 !memcmp(&ch->ch_inst, &nch->ch_inst, sizeof(ch->ch_inst))) 1056 break; 1057 } 1058 if (ch != NULL) { 1059 if (co->co_chan.chm_subidx == 0) { 1060 printf("%s: unknown offer \"%s\"\n", 1061 sc->sc_dev.dv_xname, nch->ch_ident); 1062 mtx_leave(&sc->sc_channelck); 1063 free(nch, M_DEVBUF, sizeof(*nch)); 1064 return; 1065 } 1066 #ifdef HYPERV_DEBUG 1067 printf("%s: subchannel %u for \"%s\"\n", sc->sc_dev.dv_xname, 1068 co->co_chan.chm_subidx, ch->ch_ident); 1069 #endif 1070 mtx_leave(&sc->sc_channelck); 1071 free(nch, M_DEVBUF, sizeof(*nch)); 1072 return; 1073 } 1074 1075 nch->ch_state = HV_CHANSTATE_OFFERED; 1076 1077 TAILQ_INSERT_TAIL(&sc->sc_channels, nch, ch_entry); 1078 mtx_leave(&sc->sc_channelck); 1079 1080 #ifdef HYPERV_DEBUG 1081 printf("%s: channel %u: \"%s\"", sc->sc_dev.dv_xname, nch->ch_id, 1082 nch->ch_ident); 1083 if (nch->ch_flags & CHF_MONITOR) 1084 printf(", monitor %u\n", co->co_chan.chm_montrig); 1085 else 1086 printf("\n"); 1087 #endif 1088 } 1089 1090 struct hv_channel * 1091 hv_channel_lookup(struct hv_softc *sc, uint32_t relid) 1092 { 1093 struct hv_channel *ch; 1094 1095 TAILQ_FOREACH(ch, &sc->sc_channels, ch_entry) { 1096 if (ch->ch_id == relid) 1097 return (ch); 1098 } 1099 return (NULL); 1100 } 1101 1102 int 1103 hv_channel_ring_create(struct hv_channel *ch, uint32_t buflen) 1104 { 1105 struct hv_softc *sc = ch->ch_sc; 1106 1107 buflen = roundup(buflen, PAGE_SIZE) + sizeof(struct vmbus_bufring); 1108 ch->ch_ring = km_alloc(2 * buflen, &kv_any, &kp_zero, cold ? 1109 &kd_nowait : &kd_waitok); 1110 if (ch->ch_ring == NULL) { 1111 printf("%s: failed to allocate channel ring\n", 1112 sc->sc_dev.dv_xname); 1113 return (-1); 1114 } 1115 ch->ch_ring_size = 2 * buflen; 1116 1117 memset(&ch->ch_wrd, 0, sizeof(ch->ch_wrd)); 1118 ch->ch_wrd.rd_ring = (struct vmbus_bufring *)ch->ch_ring; 1119 ch->ch_wrd.rd_size = buflen; 1120 ch->ch_wrd.rd_dsize = buflen - sizeof(struct vmbus_bufring); 1121 mtx_init(&ch->ch_wrd.rd_lock, IPL_NET); 1122 1123 memset(&ch->ch_rrd, 0, sizeof(ch->ch_rrd)); 1124 ch->ch_rrd.rd_ring = (struct vmbus_bufring *)((uint8_t *)ch->ch_ring + 1125 buflen); 1126 ch->ch_rrd.rd_size = buflen; 1127 ch->ch_rrd.rd_dsize = buflen - sizeof(struct vmbus_bufring); 1128 mtx_init(&ch->ch_rrd.rd_lock, IPL_NET); 1129 1130 if (hv_handle_alloc(ch, ch->ch_ring, 2 * buflen, &ch->ch_ring_gpadl)) { 1131 printf("%s: failed to obtain a PA handle for the ring\n", 1132 sc->sc_dev.dv_xname); 1133 hv_channel_ring_destroy(ch); 1134 return (-1); 1135 } 1136 1137 return (0); 1138 } 1139 1140 void 1141 hv_channel_ring_destroy(struct hv_channel *ch) 1142 { 1143 km_free(ch->ch_ring, ch->ch_ring_size, &kv_any, &kp_zero); 1144 ch->ch_ring = NULL; 1145 hv_handle_free(ch, ch->ch_ring_gpadl); 1146 1147 memset(&ch->ch_wrd, 0, sizeof(ch->ch_wrd)); 1148 memset(&ch->ch_rrd, 0, sizeof(ch->ch_rrd)); 1149 } 1150 1151 int 1152 hv_channel_open(struct hv_channel *ch, size_t buflen, void *udata, 1153 size_t udatalen, void (*handler)(void *), void *arg) 1154 { 1155 struct hv_softc *sc = ch->ch_sc; 1156 struct vmbus_chanmsg_chopen cmd; 1157 struct vmbus_chanmsg_chopen_resp rsp; 1158 int rv; 1159 1160 if (ch->ch_ring == NULL && 1161 hv_channel_ring_create(ch, buflen)) { 1162 DPRINTF("%s: failed to create channel ring\n", 1163 sc->sc_dev.dv_xname); 1164 return (-1); 1165 } 1166 1167 memset(&cmd, 0, sizeof(cmd)); 1168 cmd.chm_hdr.chm_type = VMBUS_CHANMSG_CHOPEN; 1169 cmd.chm_openid = ch->ch_id; 1170 cmd.chm_chanid = ch->ch_id; 1171 cmd.chm_gpadl = ch->ch_ring_gpadl; 1172 cmd.chm_txbr_pgcnt = ch->ch_wrd.rd_size >> PAGE_SHIFT; 1173 cmd.chm_vcpuid = ch->ch_vcpu; 1174 1175 if (udata && udatalen > 0) 1176 memcpy(cmd.chm_udata, udata, udatalen); 1177 1178 memset(&rsp, 0, sizeof(rsp)); 1179 1180 ch->ch_handler = handler; 1181 ch->ch_ctx = arg; 1182 1183 ch->ch_state = HV_CHANSTATE_OPENED; 1184 1185 rv = hv_cmd(sc, &cmd, sizeof(cmd), &rsp, sizeof(rsp), 1186 cold ? HCF_NOSLEEP : HCF_SLEEPOK); 1187 if (rv) { 1188 hv_channel_ring_destroy(ch); 1189 DPRINTF("%s: CHOPEN failed with %d\n", 1190 sc->sc_dev.dv_xname, rv); 1191 ch->ch_handler = NULL; 1192 ch->ch_ctx = NULL; 1193 ch->ch_state = HV_CHANSTATE_OFFERED; 1194 return (-1); 1195 } 1196 1197 return (0); 1198 } 1199 1200 int 1201 hv_channel_close(struct hv_channel *ch) 1202 { 1203 struct hv_softc *sc = ch->ch_sc; 1204 struct vmbus_chanmsg_chclose cmd; 1205 int rv; 1206 1207 memset(&cmd, 0, sizeof(cmd)); 1208 cmd.chm_hdr.chm_type = VMBUS_CHANMSG_CHCLOSE; 1209 cmd.chm_chanid = ch->ch_id; 1210 1211 ch->ch_state = HV_CHANSTATE_CLOSING; 1212 rv = hv_cmd(sc, &cmd, sizeof(cmd), NULL, 0, HCF_NOREPLY); 1213 if (rv) { 1214 DPRINTF("%s: CHCLOSE failed with %d\n", 1215 sc->sc_dev.dv_xname, rv); 1216 return (-1); 1217 } 1218 ch->ch_state = HV_CHANSTATE_CLOSED; 1219 hv_channel_ring_destroy(ch); 1220 return (0); 1221 } 1222 1223 static inline void 1224 hv_channel_setevent(struct hv_softc *sc, struct hv_channel *ch) 1225 { 1226 struct vmbus_mon_trig *mtg; 1227 1228 /* Each uint32_t represents 32 channels */ 1229 set_bit(ch->ch_id, sc->sc_wevents); 1230 if (ch->ch_flags & CHF_MONITOR) { 1231 mtg = &sc->sc_monitor[1]->mnf_trigs[ch->ch_mgroup]; 1232 set_bit(ch->ch_mindex, &mtg->mt_pending); 1233 } else 1234 hv_intr_signal(sc, &ch->ch_monprm); 1235 } 1236 1237 static inline void 1238 hv_ring_put(struct hv_ring_data *wrd, uint8_t *data, uint32_t datalen) 1239 { 1240 int left = MIN(datalen, wrd->rd_dsize - wrd->rd_prod); 1241 1242 memcpy(&wrd->rd_ring->br_data[wrd->rd_prod], data, left); 1243 memcpy(&wrd->rd_ring->br_data[0], data + left, datalen - left); 1244 wrd->rd_prod += datalen; 1245 if (wrd->rd_prod >= wrd->rd_dsize) 1246 wrd->rd_prod -= wrd->rd_dsize; 1247 } 1248 1249 static inline void 1250 hv_ring_get(struct hv_ring_data *rrd, uint8_t *data, uint32_t datalen, 1251 int peek) 1252 { 1253 int left = MIN(datalen, rrd->rd_dsize - rrd->rd_cons); 1254 1255 memcpy(data, &rrd->rd_ring->br_data[rrd->rd_cons], left); 1256 memcpy(data + left, &rrd->rd_ring->br_data[0], datalen - left); 1257 if (!peek) { 1258 rrd->rd_cons += datalen; 1259 if (rrd->rd_cons >= rrd->rd_dsize) 1260 rrd->rd_cons -= rrd->rd_dsize; 1261 } 1262 } 1263 1264 static inline void 1265 hv_ring_avail(struct hv_ring_data *rd, uint32_t *towrite, uint32_t *toread) 1266 { 1267 uint32_t ridx = rd->rd_ring->br_rindex; 1268 uint32_t widx = rd->rd_ring->br_windex; 1269 uint32_t r, w; 1270 1271 if (widx >= ridx) 1272 w = rd->rd_dsize - (widx - ridx); 1273 else 1274 w = ridx - widx; 1275 r = rd->rd_dsize - w; 1276 if (towrite) 1277 *towrite = w; 1278 if (toread) 1279 *toread = r; 1280 } 1281 1282 int 1283 hv_ring_write(struct hv_ring_data *wrd, struct iovec *iov, int iov_cnt, 1284 int *needsig) 1285 { 1286 uint64_t indices = 0; 1287 uint32_t avail, oprod, datalen = sizeof(indices); 1288 int i; 1289 1290 for (i = 0; i < iov_cnt; i++) 1291 datalen += iov[i].iov_len; 1292 1293 KASSERT(datalen <= wrd->rd_dsize); 1294 1295 hv_ring_avail(wrd, &avail, NULL); 1296 if (avail <= datalen) { 1297 DPRINTF("%s: avail %u datalen %u\n", __func__, avail, datalen); 1298 return (EAGAIN); 1299 } 1300 1301 oprod = wrd->rd_prod; 1302 1303 for (i = 0; i < iov_cnt; i++) 1304 hv_ring_put(wrd, iov[i].iov_base, iov[i].iov_len); 1305 1306 indices = (uint64_t)oprod << 32; 1307 hv_ring_put(wrd, (uint8_t *)&indices, sizeof(indices)); 1308 1309 membar_sync(); 1310 wrd->rd_ring->br_windex = wrd->rd_prod; 1311 membar_sync(); 1312 1313 /* Signal when the ring transitions from being empty to non-empty */ 1314 if (wrd->rd_ring->br_imask == 0 && 1315 wrd->rd_ring->br_rindex == oprod) 1316 *needsig = 1; 1317 else 1318 *needsig = 0; 1319 1320 return (0); 1321 } 1322 1323 int 1324 hv_channel_send(struct hv_channel *ch, void *data, uint32_t datalen, 1325 uint64_t rid, int type, uint32_t flags) 1326 { 1327 struct hv_softc *sc = ch->ch_sc; 1328 struct vmbus_chanpkt cp; 1329 struct iovec iov[3]; 1330 uint32_t pktlen, pktlen_aligned; 1331 uint64_t zeropad = 0; 1332 int rv, needsig = 0; 1333 1334 pktlen = sizeof(cp) + datalen; 1335 pktlen_aligned = roundup(pktlen, sizeof(uint64_t)); 1336 1337 cp.cp_hdr.cph_type = type; 1338 cp.cp_hdr.cph_flags = flags; 1339 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_hlen, sizeof(cp)); 1340 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_tlen, pktlen_aligned); 1341 cp.cp_hdr.cph_tid = rid; 1342 1343 iov[0].iov_base = &cp; 1344 iov[0].iov_len = sizeof(cp); 1345 1346 iov[1].iov_base = data; 1347 iov[1].iov_len = datalen; 1348 1349 iov[2].iov_base = &zeropad; 1350 iov[2].iov_len = pktlen_aligned - pktlen; 1351 1352 mtx_enter(&ch->ch_wrd.rd_lock); 1353 rv = hv_ring_write(&ch->ch_wrd, iov, 3, &needsig); 1354 mtx_leave(&ch->ch_wrd.rd_lock); 1355 if (rv == 0 && needsig) 1356 hv_channel_setevent(sc, ch); 1357 1358 return (rv); 1359 } 1360 1361 int 1362 hv_channel_send_sgl(struct hv_channel *ch, struct vmbus_gpa *sgl, 1363 uint32_t nsge, void *data, uint32_t datalen, uint64_t rid) 1364 { 1365 struct hv_softc *sc = ch->ch_sc; 1366 struct vmbus_chanpkt_sglist cp; 1367 struct iovec iov[4]; 1368 uint32_t buflen, pktlen, pktlen_aligned; 1369 uint64_t zeropad = 0; 1370 int rv, needsig = 0; 1371 1372 buflen = sizeof(struct vmbus_gpa) * nsge; 1373 pktlen = sizeof(cp) + datalen + buflen; 1374 pktlen_aligned = roundup(pktlen, sizeof(uint64_t)); 1375 1376 cp.cp_hdr.cph_type = VMBUS_CHANPKT_TYPE_GPA; 1377 cp.cp_hdr.cph_flags = VMBUS_CHANPKT_FLAG_RC; 1378 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_hlen, sizeof(cp) + buflen); 1379 VMBUS_CHANPKT_SETLEN(cp.cp_hdr.cph_tlen, pktlen_aligned); 1380 cp.cp_hdr.cph_tid = rid; 1381 cp.cp_gpa_cnt = nsge; 1382 cp.cp_rsvd = 0; 1383 1384 iov[0].iov_base = &cp; 1385 iov[0].iov_len = sizeof(cp); 1386 1387 iov[1].iov_base = sgl; 1388 iov[1].iov_len = buflen; 1389 1390 iov[2].iov_base = data; 1391 iov[2].iov_len = datalen; 1392 1393 iov[3].iov_base = &zeropad; 1394 iov[3].iov_len = pktlen_aligned - pktlen; 1395 1396 mtx_enter(&ch->ch_wrd.rd_lock); 1397 rv = hv_ring_write(&ch->ch_wrd, iov, 4, &needsig); 1398 mtx_leave(&ch->ch_wrd.rd_lock); 1399 if (rv == 0 && needsig) 1400 hv_channel_setevent(sc, ch); 1401 1402 return (rv); 1403 } 1404 1405 int 1406 hv_ring_peek(struct hv_ring_data *rrd, void *data, uint32_t datalen) 1407 { 1408 uint32_t avail; 1409 1410 KASSERT(datalen <= rrd->rd_dsize); 1411 1412 hv_ring_avail(rrd, NULL, &avail); 1413 if (avail < datalen) 1414 return (EAGAIN); 1415 1416 hv_ring_get(rrd, (uint8_t *)data, datalen, 1); 1417 return (0); 1418 } 1419 1420 int 1421 hv_ring_read(struct hv_ring_data *rrd, void *data, uint32_t datalen, 1422 uint32_t offset) 1423 { 1424 uint64_t indices; 1425 uint32_t avail; 1426 1427 KASSERT(datalen <= rrd->rd_dsize); 1428 1429 hv_ring_avail(rrd, NULL, &avail); 1430 if (avail < datalen) { 1431 DPRINTF("%s: avail %u datalen %u\n", __func__, avail, datalen); 1432 return (EAGAIN); 1433 } 1434 1435 if (offset) { 1436 rrd->rd_cons += offset; 1437 if (rrd->rd_cons >= rrd->rd_dsize) 1438 rrd->rd_cons -= rrd->rd_dsize; 1439 } 1440 1441 hv_ring_get(rrd, (uint8_t *)data, datalen, 0); 1442 hv_ring_get(rrd, (uint8_t *)&indices, sizeof(indices), 0); 1443 1444 membar_sync(); 1445 rrd->rd_ring->br_rindex = rrd->rd_cons; 1446 1447 return (0); 1448 } 1449 1450 int 1451 hv_channel_recv(struct hv_channel *ch, void *data, uint32_t datalen, 1452 uint32_t *rlen, uint64_t *rid, int raw) 1453 { 1454 struct vmbus_chanpkt_hdr cph; 1455 uint32_t offset, pktlen; 1456 int rv; 1457 1458 *rlen = 0; 1459 1460 mtx_enter(&ch->ch_rrd.rd_lock); 1461 1462 if ((rv = hv_ring_peek(&ch->ch_rrd, &cph, sizeof(cph))) != 0) { 1463 mtx_leave(&ch->ch_rrd.rd_lock); 1464 return (rv); 1465 } 1466 1467 offset = raw ? 0 : VMBUS_CHANPKT_GETLEN(cph.cph_hlen); 1468 pktlen = VMBUS_CHANPKT_GETLEN(cph.cph_tlen) - offset; 1469 if (pktlen > datalen) { 1470 mtx_leave(&ch->ch_rrd.rd_lock); 1471 printf("%s: pktlen %u datalen %u\n", __func__, pktlen, datalen); 1472 return (EINVAL); 1473 } 1474 1475 rv = hv_ring_read(&ch->ch_rrd, data, pktlen, offset); 1476 if (rv == 0) { 1477 *rlen = pktlen; 1478 *rid = cph.cph_tid; 1479 } 1480 1481 mtx_leave(&ch->ch_rrd.rd_lock); 1482 1483 return (rv); 1484 } 1485 1486 /* How many PFNs can be referenced by the header */ 1487 #define HV_NPFNHDR ((VMBUS_MSG_DSIZE_MAX - \ 1488 sizeof(struct vmbus_chanmsg_gpadl_conn)) / sizeof(uint64_t)) 1489 1490 /* How many PFNs can be referenced by the body */ 1491 #define HV_NPFNBODY ((VMBUS_MSG_DSIZE_MAX - \ 1492 sizeof(struct vmbus_chanmsg_gpadl_subconn)) / sizeof(uint64_t)) 1493 1494 int 1495 hv_handle_alloc(struct hv_channel *ch, void *buffer, uint32_t buflen, 1496 uint32_t *handle) 1497 { 1498 struct hv_softc *sc = ch->ch_sc; 1499 struct vmbus_chanmsg_gpadl_conn *hdr; 1500 struct vmbus_chanmsg_gpadl_subconn *cmd; 1501 struct vmbus_chanmsg_gpadl_connresp rsp; 1502 struct hv_msg *msg; 1503 int i, j, last, left, rv; 1504 int bodylen = 0, ncmds = 0, pfn = 0; 1505 int waitflag = cold ? M_NOWAIT : M_WAITOK; 1506 uint64_t *frames; 1507 paddr_t pa; 1508 caddr_t body; 1509 /* Total number of pages to reference */ 1510 int total = atop(buflen); 1511 /* Number of pages that will fit the header */ 1512 int inhdr = MIN(total, HV_NPFNHDR); 1513 1514 KASSERT((buflen & (PAGE_SIZE - 1)) == 0); 1515 1516 if ((msg = malloc(sizeof(*msg), M_DEVBUF, M_ZERO | waitflag)) == NULL) 1517 return (ENOMEM); 1518 1519 /* Prepare array of frame addresses */ 1520 if ((frames = mallocarray(total, sizeof(*frames), M_DEVBUF, M_ZERO | 1521 waitflag)) == NULL) { 1522 free(msg, M_DEVBUF, sizeof(*msg)); 1523 return (ENOMEM); 1524 } 1525 for (i = 0; i < total; i++) { 1526 if (!pmap_extract(pmap_kernel(), (vaddr_t)buffer + 1527 PAGE_SIZE * i, &pa)) { 1528 free(msg, M_DEVBUF, sizeof(*msg)); 1529 free(frames, M_DEVBUF, total * sizeof(*frames)); 1530 return (EFAULT); 1531 } 1532 frames[i] = atop(pa); 1533 } 1534 1535 msg->msg_req.hc_dsize = sizeof(struct vmbus_chanmsg_gpadl_conn) + 1536 inhdr * sizeof(uint64_t); 1537 hdr = (struct vmbus_chanmsg_gpadl_conn *)msg->msg_req.hc_data; 1538 msg->msg_rsp = &rsp; 1539 msg->msg_rsplen = sizeof(rsp); 1540 if (waitflag == M_NOWAIT) 1541 msg->msg_flags = MSGF_NOSLEEP; 1542 1543 left = total - inhdr; 1544 1545 /* Allocate additional gpadl_body structures if required */ 1546 if (left > 0) { 1547 ncmds = MAX(1, left / HV_NPFNBODY + left % HV_NPFNBODY); 1548 bodylen = ncmds * VMBUS_MSG_DSIZE_MAX; 1549 body = malloc(bodylen, M_DEVBUF, M_ZERO | waitflag); 1550 if (body == NULL) { 1551 free(msg, M_DEVBUF, sizeof(*msg)); 1552 free(frames, M_DEVBUF, atop(buflen) * sizeof(*frames)); 1553 return (ENOMEM); 1554 } 1555 } 1556 1557 *handle = atomic_inc_int_nv(&sc->sc_handle); 1558 1559 hdr->chm_hdr.chm_type = VMBUS_CHANMSG_GPADL_CONN; 1560 hdr->chm_chanid = ch->ch_id; 1561 hdr->chm_gpadl = *handle; 1562 1563 /* Single range for a contiguous buffer */ 1564 hdr->chm_range_cnt = 1; 1565 hdr->chm_range_len = sizeof(struct vmbus_gpa_range) + total * 1566 sizeof(uint64_t); 1567 hdr->chm_range.gpa_ofs = 0; 1568 hdr->chm_range.gpa_len = buflen; 1569 1570 /* Fit as many pages as possible into the header */ 1571 for (i = 0; i < inhdr; i++) 1572 hdr->chm_range.gpa_page[i] = frames[pfn++]; 1573 1574 for (i = 0; i < ncmds; i++) { 1575 cmd = (struct vmbus_chanmsg_gpadl_subconn *)(body + 1576 VMBUS_MSG_DSIZE_MAX * i); 1577 cmd->chm_hdr.chm_type = VMBUS_CHANMSG_GPADL_SUBCONN; 1578 cmd->chm_gpadl = *handle; 1579 last = MIN(left, HV_NPFNBODY); 1580 for (j = 0; j < last; j++) 1581 cmd->chm_gpa_page[j] = frames[pfn++]; 1582 left -= last; 1583 } 1584 1585 rv = hv_start(sc, msg); 1586 if (rv != 0) { 1587 DPRINTF("%s: GPADL_CONN failed\n", sc->sc_dev.dv_xname); 1588 goto out; 1589 } 1590 for (i = 0; i < ncmds; i++) { 1591 int cmdlen = sizeof(*cmd); 1592 cmd = (struct vmbus_chanmsg_gpadl_subconn *)(body + 1593 VMBUS_MSG_DSIZE_MAX * i); 1594 /* Last element can be short */ 1595 if (i == ncmds - 1) 1596 cmdlen += last * sizeof(uint64_t); 1597 else 1598 cmdlen += HV_NPFNBODY * sizeof(uint64_t); 1599 rv = hv_cmd(sc, cmd, cmdlen, NULL, 0, waitflag | HCF_NOREPLY); 1600 if (rv != 0) { 1601 DPRINTF("%s: GPADL_SUBCONN (iteration %d/%d) failed " 1602 "with %d\n", sc->sc_dev.dv_xname, i, ncmds, rv); 1603 goto out; 1604 } 1605 } 1606 rv = hv_reply(sc, msg); 1607 if (rv != 0) 1608 DPRINTF("%s: GPADL allocation failed with %d\n", 1609 sc->sc_dev.dv_xname, rv); 1610 1611 out: 1612 free(msg, M_DEVBUF, sizeof(*msg)); 1613 free(frames, M_DEVBUF, total * sizeof(*frames)); 1614 if (bodylen > 0) 1615 free(body, M_DEVBUF, bodylen); 1616 if (rv != 0) 1617 return (rv); 1618 1619 KASSERT(*handle == rsp.chm_gpadl); 1620 1621 return (0); 1622 } 1623 1624 void 1625 hv_handle_free(struct hv_channel *ch, uint32_t handle) 1626 { 1627 struct hv_softc *sc = ch->ch_sc; 1628 struct vmbus_chanmsg_gpadl_disconn cmd; 1629 struct vmbus_chanmsg_gpadl_disconn rsp; 1630 int rv; 1631 1632 memset(&cmd, 0, sizeof(cmd)); 1633 cmd.chm_hdr.chm_type = VMBUS_CHANMSG_GPADL_DISCONN; 1634 cmd.chm_chanid = ch->ch_id; 1635 cmd.chm_gpadl = handle; 1636 1637 rv = hv_cmd(sc, &cmd, sizeof(cmd), &rsp, sizeof(rsp), 0); 1638 if (rv) 1639 DPRINTF("%s: GPADL_DISCONN failed with %d\n", 1640 sc->sc_dev.dv_xname, rv); 1641 } 1642 1643 static int 1644 hv_attach_print(void *aux, const char *name) 1645 { 1646 struct hv_attach_args *aa = aux; 1647 1648 if (name) 1649 printf("\"%s\" at %s", aa->aa_ident, name); 1650 1651 return (UNCONF); 1652 } 1653 1654 int 1655 hv_attach_devices(struct hv_softc *sc) 1656 { 1657 struct hv_dev *dv; 1658 struct hv_channel *ch; 1659 1660 SLIST_INIT(&sc->sc_devs); 1661 mtx_init(&sc->sc_devlck, IPL_NET); 1662 1663 TAILQ_FOREACH(ch, &sc->sc_channels, ch_entry) { 1664 if (ch->ch_state != HV_CHANSTATE_OFFERED) 1665 continue; 1666 if (!(ch->ch_flags & CHF_MONITOR)) 1667 continue; 1668 dv = malloc(sizeof(*dv), M_DEVBUF, M_ZERO | M_NOWAIT); 1669 if (dv == NULL) { 1670 printf("%s: failed to allocate device object\n", 1671 sc->sc_dev.dv_xname); 1672 return (-1); 1673 } 1674 dv->dv_aa.aa_parent = sc; 1675 dv->dv_aa.aa_type = &ch->ch_type; 1676 dv->dv_aa.aa_inst = &ch->ch_inst; 1677 dv->dv_aa.aa_ident = ch->ch_ident; 1678 dv->dv_aa.aa_chan = ch; 1679 dv->dv_aa.aa_dmat = sc->sc_dmat; 1680 mtx_enter(&sc->sc_devlck); 1681 SLIST_INSERT_HEAD(&sc->sc_devs, dv, dv_entry); 1682 mtx_leave(&sc->sc_devlck); 1683 config_found((struct device *)sc, &dv->dv_aa, hv_attach_print); 1684 } 1685 return (0); 1686 } 1687 1688 void 1689 hv_evcount_attach(struct hv_channel *ch, const char *name) 1690 { 1691 struct hv_softc *sc = ch->ch_sc; 1692 1693 evcount_attach(&ch->ch_evcnt, name, &sc->sc_idtvec); 1694 } 1695