1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright 2008 Sun Microsystems, Inc. All rights reserved. 23 * Use is subject to license terms. 24 */ 25 26 /* 27 * x86 root nexus driver 28 */ 29 30 #include <sys/sysmacros.h> 31 #include <sys/conf.h> 32 #include <sys/autoconf.h> 33 #include <sys/sysmacros.h> 34 #include <sys/debug.h> 35 #include <sys/psw.h> 36 #include <sys/ddidmareq.h> 37 #include <sys/promif.h> 38 #include <sys/devops.h> 39 #include <sys/kmem.h> 40 #include <sys/cmn_err.h> 41 #include <vm/seg.h> 42 #include <vm/seg_kmem.h> 43 #include <vm/seg_dev.h> 44 #include <sys/vmem.h> 45 #include <sys/mman.h> 46 #include <vm/hat.h> 47 #include <vm/as.h> 48 #include <vm/page.h> 49 #include <sys/avintr.h> 50 #include <sys/errno.h> 51 #include <sys/modctl.h> 52 #include <sys/ddi_impldefs.h> 53 #include <sys/sunddi.h> 54 #include <sys/sunndi.h> 55 #include <sys/mach_intr.h> 56 #include <sys/psm.h> 57 #include <sys/ontrap.h> 58 #include <sys/atomic.h> 59 #include <sys/sdt.h> 60 #include <sys/rootnex.h> 61 #include <vm/hat_i86.h> 62 #include <sys/ddifm.h> 63 #include <sys/ddi_isa.h> 64 65 #ifdef __xpv 66 #include <sys/bootinfo.h> 67 #include <sys/hypervisor.h> 68 #include <sys/bootconf.h> 69 #include <vm/kboot_mmu.h> 70 #else 71 #include <sys/intel_iommu.h> 72 #endif 73 74 75 /* 76 * enable/disable extra checking of function parameters. Useful for debugging 77 * drivers. 78 */ 79 #ifdef DEBUG 80 int rootnex_alloc_check_parms = 1; 81 int rootnex_bind_check_parms = 1; 82 int rootnex_bind_check_inuse = 1; 83 int rootnex_unbind_verify_buffer = 0; 84 int rootnex_sync_check_parms = 1; 85 #else 86 int rootnex_alloc_check_parms = 0; 87 int rootnex_bind_check_parms = 0; 88 int rootnex_bind_check_inuse = 0; 89 int rootnex_unbind_verify_buffer = 0; 90 int rootnex_sync_check_parms = 0; 91 #endif 92 93 /* Master Abort and Target Abort panic flag */ 94 int rootnex_fm_ma_ta_panic_flag = 0; 95 96 /* Semi-temporary patchables to phase in bug fixes, test drivers, etc. */ 97 int rootnex_bind_fail = 1; 98 int rootnex_bind_warn = 1; 99 uint8_t *rootnex_warn_list; 100 /* bitmasks for rootnex_warn_list. Up to 8 different warnings with uint8_t */ 101 #define ROOTNEX_BIND_WARNING (0x1 << 0) 102 103 /* 104 * revert back to old broken behavior of always sync'ing entire copy buffer. 105 * This is useful if be have a buggy driver which doesn't correctly pass in 106 * the offset and size into ddi_dma_sync(). 107 */ 108 int rootnex_sync_ignore_params = 0; 109 110 /* 111 * For the 64-bit kernel, pre-alloc enough cookies for a 256K buffer plus 1 112 * page for alignment. For the 32-bit kernel, pre-alloc enough cookies for a 113 * 64K buffer plus 1 page for alignment (we have less kernel space in a 32-bit 114 * kernel). Allocate enough windows to handle a 256K buffer w/ at least 65 115 * sgllen DMA engine, and enough copybuf buffer state pages to handle 2 pages 116 * (< 8K). We will still need to allocate the copy buffer during bind though 117 * (if we need one). These can only be modified in /etc/system before rootnex 118 * attach. 119 */ 120 #if defined(__amd64) 121 int rootnex_prealloc_cookies = 65; 122 int rootnex_prealloc_windows = 4; 123 int rootnex_prealloc_copybuf = 2; 124 #else 125 int rootnex_prealloc_cookies = 33; 126 int rootnex_prealloc_windows = 4; 127 int rootnex_prealloc_copybuf = 2; 128 #endif 129 130 /* driver global state */ 131 static rootnex_state_t *rootnex_state; 132 133 /* shortcut to rootnex counters */ 134 static uint64_t *rootnex_cnt; 135 136 /* 137 * XXX - does x86 even need these or are they left over from the SPARC days? 138 */ 139 /* statically defined integer/boolean properties for the root node */ 140 static rootnex_intprop_t rootnex_intprp[] = { 141 { "PAGESIZE", PAGESIZE }, 142 { "MMU_PAGESIZE", MMU_PAGESIZE }, 143 { "MMU_PAGEOFFSET", MMU_PAGEOFFSET }, 144 { DDI_RELATIVE_ADDRESSING, 1 }, 145 }; 146 #define NROOT_INTPROPS (sizeof (rootnex_intprp) / sizeof (rootnex_intprop_t)) 147 148 #ifdef __xpv 149 typedef maddr_t rootnex_addr_t; 150 #define ROOTNEX_PADDR_TO_RBASE(xinfo, pa) \ 151 (DOMAIN_IS_INITDOMAIN(xinfo) ? pa_to_ma(pa) : (pa)) 152 #else 153 typedef paddr_t rootnex_addr_t; 154 #endif 155 156 #if !defined(__xpv) 157 char _depends_on[] = "mach/pcplusmp misc/iommulib"; 158 #endif 159 160 static struct cb_ops rootnex_cb_ops = { 161 nodev, /* open */ 162 nodev, /* close */ 163 nodev, /* strategy */ 164 nodev, /* print */ 165 nodev, /* dump */ 166 nodev, /* read */ 167 nodev, /* write */ 168 nodev, /* ioctl */ 169 nodev, /* devmap */ 170 nodev, /* mmap */ 171 nodev, /* segmap */ 172 nochpoll, /* chpoll */ 173 ddi_prop_op, /* cb_prop_op */ 174 NULL, /* struct streamtab */ 175 D_NEW | D_MP | D_HOTPLUG, /* compatibility flags */ 176 CB_REV, /* Rev */ 177 nodev, /* cb_aread */ 178 nodev /* cb_awrite */ 179 }; 180 181 static int rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, 182 off_t offset, off_t len, caddr_t *vaddrp); 183 static int rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, 184 struct hat *hat, struct seg *seg, caddr_t addr, 185 struct devpage *dp, pfn_t pfn, uint_t prot, uint_t lock); 186 static int rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 187 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 188 static int rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, 189 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 190 ddi_dma_handle_t *handlep); 191 static int rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, 192 ddi_dma_handle_t handle); 193 static int rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 194 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 195 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 196 static int rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 197 ddi_dma_handle_t handle); 198 static int rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, 199 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 200 static int rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, 201 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 202 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 203 static int rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, 204 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, 205 off_t *offp, size_t *lenp, caddr_t *objp, uint_t cache_flags); 206 static int rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, 207 ddi_ctl_enum_t ctlop, void *arg, void *result); 208 static int rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 209 ddi_iblock_cookie_t *ibc); 210 static int rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, 211 ddi_intr_op_t intr_op, ddi_intr_handle_impl_t *hdlp, void *result); 212 213 static int rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 214 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 215 ddi_dma_handle_t *handlep); 216 static int rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 217 ddi_dma_handle_t handle); 218 static int rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 219 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 220 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 221 static int rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 222 ddi_dma_handle_t handle); 223 static void rootnex_coredma_reset_cookies(dev_info_t *dip, 224 ddi_dma_handle_t handle); 225 static int rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 226 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 227 static int rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, 228 ddi_dma_handle_t handle, off_t off, size_t len, uint_t cache_flags); 229 static int rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, 230 ddi_dma_handle_t handle, uint_t win, off_t *offp, size_t *lenp, 231 ddi_dma_cookie_t *cookiep, uint_t *ccountp); 232 static int rootnex_coredma_map(dev_info_t *dip, dev_info_t *rdip, 233 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep); 234 static int rootnex_coredma_mctl(dev_info_t *dip, dev_info_t *rdip, 235 ddi_dma_handle_t handle, enum ddi_dma_ctlops request, off_t *offp, 236 size_t *lenp, caddr_t *objpp, uint_t cache_flags); 237 238 static struct bus_ops rootnex_bus_ops = { 239 BUSO_REV, 240 rootnex_map, 241 NULL, 242 NULL, 243 NULL, 244 rootnex_map_fault, 245 rootnex_dma_map, 246 rootnex_dma_allochdl, 247 rootnex_dma_freehdl, 248 rootnex_dma_bindhdl, 249 rootnex_dma_unbindhdl, 250 rootnex_dma_sync, 251 rootnex_dma_win, 252 rootnex_dma_mctl, 253 rootnex_ctlops, 254 ddi_bus_prop_op, 255 i_ddi_rootnex_get_eventcookie, 256 i_ddi_rootnex_add_eventcall, 257 i_ddi_rootnex_remove_eventcall, 258 i_ddi_rootnex_post_event, 259 0, /* bus_intr_ctl */ 260 0, /* bus_config */ 261 0, /* bus_unconfig */ 262 rootnex_fm_init, /* bus_fm_init */ 263 NULL, /* bus_fm_fini */ 264 NULL, /* bus_fm_access_enter */ 265 NULL, /* bus_fm_access_exit */ 266 NULL, /* bus_powr */ 267 rootnex_intr_ops /* bus_intr_op */ 268 }; 269 270 static int rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd); 271 static int rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd); 272 273 static struct dev_ops rootnex_ops = { 274 DEVO_REV, 275 0, 276 ddi_no_info, 277 nulldev, 278 nulldev, 279 rootnex_attach, 280 rootnex_detach, 281 nulldev, 282 &rootnex_cb_ops, 283 &rootnex_bus_ops 284 }; 285 286 static struct modldrv rootnex_modldrv = { 287 &mod_driverops, 288 "i86pc root nexus", 289 &rootnex_ops 290 }; 291 292 static struct modlinkage rootnex_modlinkage = { 293 MODREV_1, 294 (void *)&rootnex_modldrv, 295 NULL 296 }; 297 298 static iommulib_nexops_t iommulib_nexops = { 299 IOMMU_NEXOPS_VERSION, 300 "Rootnex IOMMU ops Vers 1.1", 301 NULL, 302 rootnex_coredma_allochdl, 303 rootnex_coredma_freehdl, 304 rootnex_coredma_bindhdl, 305 rootnex_coredma_unbindhdl, 306 rootnex_coredma_reset_cookies, 307 rootnex_coredma_get_cookies, 308 rootnex_coredma_sync, 309 rootnex_coredma_win, 310 rootnex_coredma_map, 311 rootnex_coredma_mctl 312 }; 313 314 /* 315 * extern hacks 316 */ 317 extern struct seg_ops segdev_ops; 318 extern int ignore_hardware_nodes; /* force flag from ddi_impl.c */ 319 #ifdef DDI_MAP_DEBUG 320 extern int ddi_map_debug_flag; 321 #define ddi_map_debug if (ddi_map_debug_flag) prom_printf 322 #endif 323 extern void i86_pp_map(page_t *pp, caddr_t kaddr); 324 extern void i86_va_map(caddr_t vaddr, struct as *asp, caddr_t kaddr); 325 extern int (*psm_intr_ops)(dev_info_t *, ddi_intr_handle_impl_t *, 326 psm_intr_op_t, int *); 327 extern int impl_ddi_sunbus_initchild(dev_info_t *dip); 328 extern void impl_ddi_sunbus_removechild(dev_info_t *dip); 329 330 /* 331 * Use device arena to use for device control register mappings. 332 * Various kernel memory walkers (debugger, dtrace) need to know 333 * to avoid this address range to prevent undesired device activity. 334 */ 335 extern void *device_arena_alloc(size_t size, int vm_flag); 336 extern void device_arena_free(void * vaddr, size_t size); 337 338 339 /* 340 * Internal functions 341 */ 342 static int rootnex_dma_init(); 343 static void rootnex_add_props(dev_info_t *); 344 static int rootnex_ctl_reportdev(dev_info_t *dip); 345 static struct intrspec *rootnex_get_ispec(dev_info_t *rdip, int inum); 346 static int rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 347 static int rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp); 348 static int rootnex_map_handle(ddi_map_req_t *mp); 349 static void rootnex_clean_dmahdl(ddi_dma_impl_t *hp); 350 static int rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegsize); 351 static int rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, 352 ddi_dma_attr_t *attr); 353 static void rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 354 rootnex_sglinfo_t *sglinfo); 355 static int rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 356 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag); 357 static int rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 358 rootnex_dma_t *dma, ddi_dma_attr_t *attr); 359 static void rootnex_teardown_copybuf(rootnex_dma_t *dma); 360 static int rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 361 ddi_dma_attr_t *attr, int kmflag); 362 static void rootnex_teardown_windows(rootnex_dma_t *dma); 363 static void rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 364 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset); 365 static void rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, 366 rootnex_dma_t *dma, ddi_dma_cookie_t *cookie, off_t cur_offset, 367 size_t *copybuf_used, page_t **cur_pp); 368 static int rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, 369 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, 370 ddi_dma_attr_t *attr, off_t cur_offset); 371 static int rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, 372 rootnex_dma_t *dma, rootnex_window_t **windowp, 373 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used); 374 static int rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, 375 rootnex_dma_t *dma, rootnex_window_t **windowp, ddi_dma_cookie_t *cookie); 376 static int rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 377 off_t offset, size_t size, uint_t cache_flags); 378 static int rootnex_verify_buffer(rootnex_dma_t *dma); 379 static int rootnex_dma_check(dev_info_t *dip, const void *handle, 380 const void *comp_addr, const void *not_used); 381 382 /* 383 * _init() 384 * 385 */ 386 int 387 _init(void) 388 { 389 390 rootnex_state = NULL; 391 return (mod_install(&rootnex_modlinkage)); 392 } 393 394 395 /* 396 * _info() 397 * 398 */ 399 int 400 _info(struct modinfo *modinfop) 401 { 402 return (mod_info(&rootnex_modlinkage, modinfop)); 403 } 404 405 406 /* 407 * _fini() 408 * 409 */ 410 int 411 _fini(void) 412 { 413 return (EBUSY); 414 } 415 416 417 /* 418 * rootnex_attach() 419 * 420 */ 421 static int 422 rootnex_attach(dev_info_t *dip, ddi_attach_cmd_t cmd) 423 { 424 int fmcap; 425 int e; 426 427 switch (cmd) { 428 case DDI_ATTACH: 429 break; 430 case DDI_RESUME: 431 return (DDI_SUCCESS); 432 default: 433 return (DDI_FAILURE); 434 } 435 436 /* 437 * We should only have one instance of rootnex. Save it away since we 438 * don't have an easy way to get it back later. 439 */ 440 ASSERT(rootnex_state == NULL); 441 rootnex_state = kmem_zalloc(sizeof (rootnex_state_t), KM_SLEEP); 442 443 rootnex_state->r_dip = dip; 444 rootnex_state->r_err_ibc = (ddi_iblock_cookie_t)ipltospl(15); 445 rootnex_state->r_reserved_msg_printed = B_FALSE; 446 rootnex_cnt = &rootnex_state->r_counters[0]; 447 rootnex_state->r_intel_iommu_enabled = B_FALSE; 448 449 /* 450 * Set minimum fm capability level for i86pc platforms and then 451 * initialize error handling. Since we're the rootnex, we don't 452 * care what's returned in the fmcap field. 453 */ 454 ddi_system_fmcap = DDI_FM_EREPORT_CAPABLE | DDI_FM_ERRCB_CAPABLE | 455 DDI_FM_ACCCHK_CAPABLE | DDI_FM_DMACHK_CAPABLE; 456 fmcap = ddi_system_fmcap; 457 ddi_fm_init(dip, &fmcap, &rootnex_state->r_err_ibc); 458 459 /* initialize DMA related state */ 460 e = rootnex_dma_init(); 461 if (e != DDI_SUCCESS) { 462 kmem_free(rootnex_state, sizeof (rootnex_state_t)); 463 return (DDI_FAILURE); 464 } 465 466 /* Add static root node properties */ 467 rootnex_add_props(dip); 468 469 /* since we can't call ddi_report_dev() */ 470 cmn_err(CE_CONT, "?root nexus = %s\n", ddi_get_name(dip)); 471 472 /* Initialize rootnex event handle */ 473 i_ddi_rootnex_init_events(dip); 474 475 #if !defined(__xpv) 476 #if defined(__amd64) 477 /* probe intel iommu */ 478 intel_iommu_probe_and_parse(); 479 480 /* attach the iommu nodes */ 481 if (intel_iommu_support) { 482 if (intel_iommu_attach_dmar_nodes() == DDI_SUCCESS) { 483 rootnex_state->r_intel_iommu_enabled = B_TRUE; 484 } else { 485 intel_iommu_release_dmar_info(); 486 } 487 } 488 #endif 489 490 e = iommulib_nexus_register(dip, &iommulib_nexops, 491 &rootnex_state->r_iommulib_handle); 492 493 ASSERT(e == DDI_SUCCESS); 494 #endif 495 496 return (DDI_SUCCESS); 497 } 498 499 500 /* 501 * rootnex_detach() 502 * 503 */ 504 /*ARGSUSED*/ 505 static int 506 rootnex_detach(dev_info_t *dip, ddi_detach_cmd_t cmd) 507 { 508 switch (cmd) { 509 case DDI_SUSPEND: 510 break; 511 default: 512 return (DDI_FAILURE); 513 } 514 515 return (DDI_SUCCESS); 516 } 517 518 519 /* 520 * rootnex_dma_init() 521 * 522 */ 523 /*ARGSUSED*/ 524 static int 525 rootnex_dma_init() 526 { 527 size_t bufsize; 528 529 530 /* 531 * size of our cookie/window/copybuf state needed in dma bind that we 532 * pre-alloc in dma_alloc_handle 533 */ 534 rootnex_state->r_prealloc_cookies = rootnex_prealloc_cookies; 535 rootnex_state->r_prealloc_size = 536 (rootnex_state->r_prealloc_cookies * sizeof (ddi_dma_cookie_t)) + 537 (rootnex_prealloc_windows * sizeof (rootnex_window_t)) + 538 (rootnex_prealloc_copybuf * sizeof (rootnex_pgmap_t)); 539 540 /* 541 * setup DDI DMA handle kmem cache, align each handle on 64 bytes, 542 * allocate 16 extra bytes for struct pointer alignment 543 * (p->dmai_private & dma->dp_prealloc_buffer) 544 */ 545 bufsize = sizeof (ddi_dma_impl_t) + sizeof (rootnex_dma_t) + 546 rootnex_state->r_prealloc_size + 0x10; 547 rootnex_state->r_dmahdl_cache = kmem_cache_create("rootnex_dmahdl", 548 bufsize, 64, NULL, NULL, NULL, NULL, NULL, 0); 549 if (rootnex_state->r_dmahdl_cache == NULL) { 550 return (DDI_FAILURE); 551 } 552 553 /* 554 * allocate array to track which major numbers we have printed warnings 555 * for. 556 */ 557 rootnex_warn_list = kmem_zalloc(devcnt * sizeof (*rootnex_warn_list), 558 KM_SLEEP); 559 560 return (DDI_SUCCESS); 561 } 562 563 564 /* 565 * rootnex_add_props() 566 * 567 */ 568 static void 569 rootnex_add_props(dev_info_t *dip) 570 { 571 rootnex_intprop_t *rpp; 572 int i; 573 574 /* Add static integer/boolean properties to the root node */ 575 rpp = rootnex_intprp; 576 for (i = 0; i < NROOT_INTPROPS; i++) { 577 (void) e_ddi_prop_update_int(DDI_DEV_T_NONE, dip, 578 rpp[i].prop_name, rpp[i].prop_value); 579 } 580 } 581 582 583 584 /* 585 * ************************* 586 * ctlops related routines 587 * ************************* 588 */ 589 590 /* 591 * rootnex_ctlops() 592 * 593 */ 594 /*ARGSUSED*/ 595 static int 596 rootnex_ctlops(dev_info_t *dip, dev_info_t *rdip, ddi_ctl_enum_t ctlop, 597 void *arg, void *result) 598 { 599 int n, *ptr; 600 struct ddi_parent_private_data *pdp; 601 602 switch (ctlop) { 603 case DDI_CTLOPS_DMAPMAPC: 604 /* 605 * Return 'partial' to indicate that dma mapping 606 * has to be done in the main MMU. 607 */ 608 return (DDI_DMA_PARTIAL); 609 610 case DDI_CTLOPS_BTOP: 611 /* 612 * Convert byte count input to physical page units. 613 * (byte counts that are not a page-size multiple 614 * are rounded down) 615 */ 616 *(ulong_t *)result = btop(*(ulong_t *)arg); 617 return (DDI_SUCCESS); 618 619 case DDI_CTLOPS_PTOB: 620 /* 621 * Convert size in physical pages to bytes 622 */ 623 *(ulong_t *)result = ptob(*(ulong_t *)arg); 624 return (DDI_SUCCESS); 625 626 case DDI_CTLOPS_BTOPR: 627 /* 628 * Convert byte count input to physical page units 629 * (byte counts that are not a page-size multiple 630 * are rounded up) 631 */ 632 *(ulong_t *)result = btopr(*(ulong_t *)arg); 633 return (DDI_SUCCESS); 634 635 case DDI_CTLOPS_INITCHILD: 636 return (impl_ddi_sunbus_initchild(arg)); 637 638 case DDI_CTLOPS_UNINITCHILD: 639 impl_ddi_sunbus_removechild(arg); 640 return (DDI_SUCCESS); 641 642 case DDI_CTLOPS_REPORTDEV: 643 return (rootnex_ctl_reportdev(rdip)); 644 645 case DDI_CTLOPS_IOMIN: 646 /* 647 * Nothing to do here but reflect back.. 648 */ 649 return (DDI_SUCCESS); 650 651 case DDI_CTLOPS_REGSIZE: 652 case DDI_CTLOPS_NREGS: 653 break; 654 655 case DDI_CTLOPS_SIDDEV: 656 if (ndi_dev_is_prom_node(rdip)) 657 return (DDI_SUCCESS); 658 if (ndi_dev_is_persistent_node(rdip)) 659 return (DDI_SUCCESS); 660 return (DDI_FAILURE); 661 662 case DDI_CTLOPS_POWER: 663 return ((*pm_platform_power)((power_req_t *)arg)); 664 665 case DDI_CTLOPS_RESERVED0: /* Was DDI_CTLOPS_NINTRS, obsolete */ 666 case DDI_CTLOPS_RESERVED1: /* Was DDI_CTLOPS_POKE_INIT, obsolete */ 667 case DDI_CTLOPS_RESERVED2: /* Was DDI_CTLOPS_POKE_FLUSH, obsolete */ 668 case DDI_CTLOPS_RESERVED3: /* Was DDI_CTLOPS_POKE_FINI, obsolete */ 669 case DDI_CTLOPS_RESERVED4: /* Was DDI_CTLOPS_INTR_HILEVEL, obsolete */ 670 case DDI_CTLOPS_RESERVED5: /* Was DDI_CTLOPS_XLATE_INTRS, obsolete */ 671 if (!rootnex_state->r_reserved_msg_printed) { 672 rootnex_state->r_reserved_msg_printed = B_TRUE; 673 cmn_err(CE_WARN, "Failing ddi_ctlops call(s) for " 674 "1 or more reserved/obsolete operations."); 675 } 676 return (DDI_FAILURE); 677 678 default: 679 return (DDI_FAILURE); 680 } 681 /* 682 * The rest are for "hardware" properties 683 */ 684 if ((pdp = ddi_get_parent_data(rdip)) == NULL) 685 return (DDI_FAILURE); 686 687 if (ctlop == DDI_CTLOPS_NREGS) { 688 ptr = (int *)result; 689 *ptr = pdp->par_nreg; 690 } else { 691 off_t *size = (off_t *)result; 692 693 ptr = (int *)arg; 694 n = *ptr; 695 if (n >= pdp->par_nreg) { 696 return (DDI_FAILURE); 697 } 698 *size = (off_t)pdp->par_reg[n].regspec_size; 699 } 700 return (DDI_SUCCESS); 701 } 702 703 704 /* 705 * rootnex_ctl_reportdev() 706 * 707 */ 708 static int 709 rootnex_ctl_reportdev(dev_info_t *dev) 710 { 711 int i, n, len, f_len = 0; 712 char *buf; 713 714 buf = kmem_alloc(REPORTDEV_BUFSIZE, KM_SLEEP); 715 f_len += snprintf(buf, REPORTDEV_BUFSIZE, 716 "%s%d at root", ddi_driver_name(dev), ddi_get_instance(dev)); 717 len = strlen(buf); 718 719 for (i = 0; i < sparc_pd_getnreg(dev); i++) { 720 721 struct regspec *rp = sparc_pd_getreg(dev, i); 722 723 if (i == 0) 724 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 725 ": "); 726 else 727 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 728 " and "); 729 len = strlen(buf); 730 731 switch (rp->regspec_bustype) { 732 733 case BTEISA: 734 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 735 "%s 0x%x", DEVI_EISA_NEXNAME, rp->regspec_addr); 736 break; 737 738 case BTISA: 739 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 740 "%s 0x%x", DEVI_ISA_NEXNAME, rp->regspec_addr); 741 break; 742 743 default: 744 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 745 "space %x offset %x", 746 rp->regspec_bustype, rp->regspec_addr); 747 break; 748 } 749 len = strlen(buf); 750 } 751 for (i = 0, n = sparc_pd_getnintr(dev); i < n; i++) { 752 int pri; 753 754 if (i != 0) { 755 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 756 ","); 757 len = strlen(buf); 758 } 759 pri = INT_IPL(sparc_pd_getintr(dev, i)->intrspec_pri); 760 f_len += snprintf(buf + len, REPORTDEV_BUFSIZE - len, 761 " sparc ipl %d", pri); 762 len = strlen(buf); 763 } 764 #ifdef DEBUG 765 if (f_len + 1 >= REPORTDEV_BUFSIZE) { 766 cmn_err(CE_NOTE, "next message is truncated: " 767 "printed length 1024, real length %d", f_len); 768 } 769 #endif /* DEBUG */ 770 cmn_err(CE_CONT, "?%s\n", buf); 771 kmem_free(buf, REPORTDEV_BUFSIZE); 772 return (DDI_SUCCESS); 773 } 774 775 776 /* 777 * ****************** 778 * map related code 779 * ****************** 780 */ 781 782 /* 783 * rootnex_map() 784 * 785 */ 786 static int 787 rootnex_map(dev_info_t *dip, dev_info_t *rdip, ddi_map_req_t *mp, off_t offset, 788 off_t len, caddr_t *vaddrp) 789 { 790 struct regspec *rp, tmp_reg; 791 ddi_map_req_t mr = *mp; /* Get private copy of request */ 792 int error; 793 794 mp = &mr; 795 796 switch (mp->map_op) { 797 case DDI_MO_MAP_LOCKED: 798 case DDI_MO_UNMAP: 799 case DDI_MO_MAP_HANDLE: 800 break; 801 default: 802 #ifdef DDI_MAP_DEBUG 803 cmn_err(CE_WARN, "rootnex_map: unimplemented map op %d.", 804 mp->map_op); 805 #endif /* DDI_MAP_DEBUG */ 806 return (DDI_ME_UNIMPLEMENTED); 807 } 808 809 if (mp->map_flags & DDI_MF_USER_MAPPING) { 810 #ifdef DDI_MAP_DEBUG 811 cmn_err(CE_WARN, "rootnex_map: unimplemented map type: user."); 812 #endif /* DDI_MAP_DEBUG */ 813 return (DDI_ME_UNIMPLEMENTED); 814 } 815 816 /* 817 * First, if given an rnumber, convert it to a regspec... 818 * (Presumably, this is on behalf of a child of the root node?) 819 */ 820 821 if (mp->map_type == DDI_MT_RNUMBER) { 822 823 int rnumber = mp->map_obj.rnumber; 824 #ifdef DDI_MAP_DEBUG 825 static char *out_of_range = 826 "rootnex_map: Out of range rnumber <%d>, device <%s>"; 827 #endif /* DDI_MAP_DEBUG */ 828 829 rp = i_ddi_rnumber_to_regspec(rdip, rnumber); 830 if (rp == NULL) { 831 #ifdef DDI_MAP_DEBUG 832 cmn_err(CE_WARN, out_of_range, rnumber, 833 ddi_get_name(rdip)); 834 #endif /* DDI_MAP_DEBUG */ 835 return (DDI_ME_RNUMBER_RANGE); 836 } 837 838 /* 839 * Convert the given ddi_map_req_t from rnumber to regspec... 840 */ 841 842 mp->map_type = DDI_MT_REGSPEC; 843 mp->map_obj.rp = rp; 844 } 845 846 /* 847 * Adjust offset and length correspnding to called values... 848 * XXX: A non-zero length means override the one in the regspec 849 * XXX: (regardless of what's in the parent's range?) 850 */ 851 852 tmp_reg = *(mp->map_obj.rp); /* Preserve underlying data */ 853 rp = mp->map_obj.rp = &tmp_reg; /* Use tmp_reg in request */ 854 855 #ifdef DDI_MAP_DEBUG 856 cmn_err(CE_CONT, "rootnex: <%s,%s> <0x%x, 0x%x, 0x%d> offset %d len %d " 857 "handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 858 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, offset, 859 len, mp->map_handlep); 860 #endif /* DDI_MAP_DEBUG */ 861 862 /* 863 * I/O or memory mapping: 864 * 865 * <bustype=0, addr=x, len=x>: memory 866 * <bustype=1, addr=x, len=x>: i/o 867 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 868 */ 869 870 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 871 cmn_err(CE_WARN, "<%s,%s> invalid register spec" 872 " <0x%x, 0x%x, 0x%x>", ddi_get_name(dip), 873 ddi_get_name(rdip), rp->regspec_bustype, 874 rp->regspec_addr, rp->regspec_size); 875 return (DDI_ME_INVAL); 876 } 877 878 if (rp->regspec_bustype > 1 && rp->regspec_addr == 0) { 879 /* 880 * compatibility i/o mapping 881 */ 882 rp->regspec_bustype += (uint_t)offset; 883 } else { 884 /* 885 * Normal memory or i/o mapping 886 */ 887 rp->regspec_addr += (uint_t)offset; 888 } 889 890 if (len != 0) 891 rp->regspec_size = (uint_t)len; 892 893 #ifdef DDI_MAP_DEBUG 894 cmn_err(CE_CONT, " <%s,%s> <0x%x, 0x%x, 0x%d> offset %d " 895 "len %d handle 0x%x\n", ddi_get_name(dip), ddi_get_name(rdip), 896 rp->regspec_bustype, rp->regspec_addr, rp->regspec_size, 897 offset, len, mp->map_handlep); 898 #endif /* DDI_MAP_DEBUG */ 899 900 /* 901 * Apply any parent ranges at this level, if applicable. 902 * (This is where nexus specific regspec translation takes place. 903 * Use of this function is implicit agreement that translation is 904 * provided via ddi_apply_range.) 905 */ 906 907 #ifdef DDI_MAP_DEBUG 908 ddi_map_debug("applying range of parent <%s> to child <%s>...\n", 909 ddi_get_name(dip), ddi_get_name(rdip)); 910 #endif /* DDI_MAP_DEBUG */ 911 912 if ((error = i_ddi_apply_range(dip, rdip, mp->map_obj.rp)) != 0) 913 return (error); 914 915 switch (mp->map_op) { 916 case DDI_MO_MAP_LOCKED: 917 918 /* 919 * Set up the locked down kernel mapping to the regspec... 920 */ 921 922 return (rootnex_map_regspec(mp, vaddrp)); 923 924 case DDI_MO_UNMAP: 925 926 /* 927 * Release mapping... 928 */ 929 930 return (rootnex_unmap_regspec(mp, vaddrp)); 931 932 case DDI_MO_MAP_HANDLE: 933 934 return (rootnex_map_handle(mp)); 935 936 default: 937 return (DDI_ME_UNIMPLEMENTED); 938 } 939 } 940 941 942 /* 943 * rootnex_map_fault() 944 * 945 * fault in mappings for requestors 946 */ 947 /*ARGSUSED*/ 948 static int 949 rootnex_map_fault(dev_info_t *dip, dev_info_t *rdip, struct hat *hat, 950 struct seg *seg, caddr_t addr, struct devpage *dp, pfn_t pfn, uint_t prot, 951 uint_t lock) 952 { 953 954 #ifdef DDI_MAP_DEBUG 955 ddi_map_debug("rootnex_map_fault: address <%x> pfn <%x>", addr, pfn); 956 ddi_map_debug(" Seg <%s>\n", 957 seg->s_ops == &segdev_ops ? "segdev" : 958 seg == &kvseg ? "segkmem" : "NONE!"); 959 #endif /* DDI_MAP_DEBUG */ 960 961 /* 962 * This is all terribly broken, but it is a start 963 * 964 * XXX Note that this test means that segdev_ops 965 * must be exported from seg_dev.c. 966 * XXX What about devices with their own segment drivers? 967 */ 968 if (seg->s_ops == &segdev_ops) { 969 struct segdev_data *sdp = (struct segdev_data *)seg->s_data; 970 971 if (hat == NULL) { 972 /* 973 * This is one plausible interpretation of 974 * a null hat i.e. use the first hat on the 975 * address space hat list which by convention is 976 * the hat of the system MMU. At alternative 977 * would be to panic .. this might well be better .. 978 */ 979 ASSERT(AS_READ_HELD(seg->s_as, &seg->s_as->a_lock)); 980 hat = seg->s_as->a_hat; 981 cmn_err(CE_NOTE, "rootnex_map_fault: nil hat"); 982 } 983 hat_devload(hat, addr, MMU_PAGESIZE, pfn, prot | sdp->hat_attr, 984 (lock ? HAT_LOAD_LOCK : HAT_LOAD)); 985 } else if (seg == &kvseg && dp == NULL) { 986 hat_devload(kas.a_hat, addr, MMU_PAGESIZE, pfn, prot, 987 HAT_LOAD_LOCK); 988 } else 989 return (DDI_FAILURE); 990 return (DDI_SUCCESS); 991 } 992 993 994 /* 995 * rootnex_map_regspec() 996 * we don't support mapping of I/O cards above 4Gb 997 */ 998 static int 999 rootnex_map_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1000 { 1001 rootnex_addr_t rbase; 1002 void *cvaddr; 1003 uint_t npages, pgoffset; 1004 struct regspec *rp; 1005 ddi_acc_hdl_t *hp; 1006 ddi_acc_impl_t *ap; 1007 uint_t hat_acc_flags; 1008 paddr_t pbase; 1009 1010 rp = mp->map_obj.rp; 1011 hp = mp->map_handlep; 1012 1013 #ifdef DDI_MAP_DEBUG 1014 ddi_map_debug( 1015 "rootnex_map_regspec: <0x%x 0x%x 0x%x> handle 0x%x\n", 1016 rp->regspec_bustype, rp->regspec_addr, 1017 rp->regspec_size, mp->map_handlep); 1018 #endif /* DDI_MAP_DEBUG */ 1019 1020 /* 1021 * I/O or memory mapping 1022 * 1023 * <bustype=0, addr=x, len=x>: memory 1024 * <bustype=1, addr=x, len=x>: i/o 1025 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1026 */ 1027 1028 if (rp->regspec_bustype > 1 && rp->regspec_addr != 0) { 1029 cmn_err(CE_WARN, "rootnex: invalid register spec" 1030 " <0x%x, 0x%x, 0x%x>", rp->regspec_bustype, 1031 rp->regspec_addr, rp->regspec_size); 1032 return (DDI_FAILURE); 1033 } 1034 1035 if (rp->regspec_bustype != 0) { 1036 /* 1037 * I/O space - needs a handle. 1038 */ 1039 if (hp == NULL) { 1040 return (DDI_FAILURE); 1041 } 1042 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1043 ap->ahi_acc_attr |= DDI_ACCATTR_IO_SPACE; 1044 impl_acc_hdl_init(hp); 1045 1046 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1047 #ifdef DDI_MAP_DEBUG 1048 ddi_map_debug("rootnex_map_regspec: mmap() " 1049 "to I/O space is not supported.\n"); 1050 #endif /* DDI_MAP_DEBUG */ 1051 return (DDI_ME_INVAL); 1052 } else { 1053 /* 1054 * 1275-compliant vs. compatibility i/o mapping 1055 */ 1056 *vaddrp = 1057 (rp->regspec_bustype > 1 && rp->regspec_addr == 0) ? 1058 ((caddr_t)(uintptr_t)rp->regspec_bustype) : 1059 ((caddr_t)(uintptr_t)rp->regspec_addr); 1060 #ifdef __xpv 1061 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1062 hp->ah_pfn = xen_assign_pfn( 1063 mmu_btop((ulong_t)rp->regspec_addr & 1064 MMU_PAGEMASK)); 1065 } else { 1066 hp->ah_pfn = mmu_btop( 1067 (ulong_t)rp->regspec_addr & MMU_PAGEMASK); 1068 } 1069 #else 1070 hp->ah_pfn = mmu_btop((ulong_t)rp->regspec_addr & 1071 MMU_PAGEMASK); 1072 #endif 1073 hp->ah_pnum = mmu_btopr(rp->regspec_size + 1074 (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET); 1075 } 1076 1077 #ifdef DDI_MAP_DEBUG 1078 ddi_map_debug( 1079 "rootnex_map_regspec: \"Mapping\" %d bytes I/O space at 0x%x\n", 1080 rp->regspec_size, *vaddrp); 1081 #endif /* DDI_MAP_DEBUG */ 1082 return (DDI_SUCCESS); 1083 } 1084 1085 /* 1086 * Memory space 1087 */ 1088 1089 if (hp != NULL) { 1090 /* 1091 * hat layer ignores 1092 * hp->ah_acc.devacc_attr_endian_flags. 1093 */ 1094 switch (hp->ah_acc.devacc_attr_dataorder) { 1095 case DDI_STRICTORDER_ACC: 1096 hat_acc_flags = HAT_STRICTORDER; 1097 break; 1098 case DDI_UNORDERED_OK_ACC: 1099 hat_acc_flags = HAT_UNORDERED_OK; 1100 break; 1101 case DDI_MERGING_OK_ACC: 1102 hat_acc_flags = HAT_MERGING_OK; 1103 break; 1104 case DDI_LOADCACHING_OK_ACC: 1105 hat_acc_flags = HAT_LOADCACHING_OK; 1106 break; 1107 case DDI_STORECACHING_OK_ACC: 1108 hat_acc_flags = HAT_STORECACHING_OK; 1109 break; 1110 } 1111 ap = (ddi_acc_impl_t *)hp->ah_platform_private; 1112 ap->ahi_acc_attr |= DDI_ACCATTR_CPU_VADDR; 1113 impl_acc_hdl_init(hp); 1114 hp->ah_hat_flags = hat_acc_flags; 1115 } else { 1116 hat_acc_flags = HAT_STRICTORDER; 1117 } 1118 1119 rbase = (rootnex_addr_t)(rp->regspec_addr & MMU_PAGEMASK); 1120 #ifdef __xpv 1121 /* 1122 * If we're dom0, we're using a real device so we need to translate 1123 * the MA to a PA. 1124 */ 1125 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1126 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))); 1127 } else { 1128 pbase = rbase; 1129 } 1130 #else 1131 pbase = rbase; 1132 #endif 1133 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1134 1135 if (rp->regspec_size == 0) { 1136 #ifdef DDI_MAP_DEBUG 1137 ddi_map_debug("rootnex_map_regspec: zero regspec_size\n"); 1138 #endif /* DDI_MAP_DEBUG */ 1139 return (DDI_ME_INVAL); 1140 } 1141 1142 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) { 1143 /* extra cast to make gcc happy */ 1144 *vaddrp = (caddr_t)((uintptr_t)mmu_btop(pbase)); 1145 } else { 1146 npages = mmu_btopr(rp->regspec_size + pgoffset); 1147 1148 #ifdef DDI_MAP_DEBUG 1149 ddi_map_debug("rootnex_map_regspec: Mapping %d pages " 1150 "physical %llx", npages, pbase); 1151 #endif /* DDI_MAP_DEBUG */ 1152 1153 cvaddr = device_arena_alloc(ptob(npages), VM_NOSLEEP); 1154 if (cvaddr == NULL) 1155 return (DDI_ME_NORESOURCES); 1156 1157 /* 1158 * Now map in the pages we've allocated... 1159 */ 1160 hat_devload(kas.a_hat, cvaddr, mmu_ptob(npages), 1161 mmu_btop(pbase), mp->map_prot | hat_acc_flags, 1162 HAT_LOAD_LOCK); 1163 *vaddrp = (caddr_t)cvaddr + pgoffset; 1164 1165 /* save away pfn and npages for FMA */ 1166 hp = mp->map_handlep; 1167 if (hp) { 1168 hp->ah_pfn = mmu_btop(pbase); 1169 hp->ah_pnum = npages; 1170 } 1171 } 1172 1173 #ifdef DDI_MAP_DEBUG 1174 ddi_map_debug("at virtual 0x%x\n", *vaddrp); 1175 #endif /* DDI_MAP_DEBUG */ 1176 return (DDI_SUCCESS); 1177 } 1178 1179 1180 /* 1181 * rootnex_unmap_regspec() 1182 * 1183 */ 1184 static int 1185 rootnex_unmap_regspec(ddi_map_req_t *mp, caddr_t *vaddrp) 1186 { 1187 caddr_t addr = (caddr_t)*vaddrp; 1188 uint_t npages, pgoffset; 1189 struct regspec *rp; 1190 1191 if (mp->map_flags & DDI_MF_DEVICE_MAPPING) 1192 return (0); 1193 1194 rp = mp->map_obj.rp; 1195 1196 if (rp->regspec_size == 0) { 1197 #ifdef DDI_MAP_DEBUG 1198 ddi_map_debug("rootnex_unmap_regspec: zero regspec_size\n"); 1199 #endif /* DDI_MAP_DEBUG */ 1200 return (DDI_ME_INVAL); 1201 } 1202 1203 /* 1204 * I/O or memory mapping: 1205 * 1206 * <bustype=0, addr=x, len=x>: memory 1207 * <bustype=1, addr=x, len=x>: i/o 1208 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1209 */ 1210 if (rp->regspec_bustype != 0) { 1211 /* 1212 * This is I/O space, which requires no particular 1213 * processing on unmap since it isn't mapped in the 1214 * first place. 1215 */ 1216 return (DDI_SUCCESS); 1217 } 1218 1219 /* 1220 * Memory space 1221 */ 1222 pgoffset = (uintptr_t)addr & MMU_PAGEOFFSET; 1223 npages = mmu_btopr(rp->regspec_size + pgoffset); 1224 hat_unload(kas.a_hat, addr - pgoffset, ptob(npages), HAT_UNLOAD_UNLOCK); 1225 device_arena_free(addr - pgoffset, ptob(npages)); 1226 1227 /* 1228 * Destroy the pointer - the mapping has logically gone 1229 */ 1230 *vaddrp = NULL; 1231 1232 return (DDI_SUCCESS); 1233 } 1234 1235 1236 /* 1237 * rootnex_map_handle() 1238 * 1239 */ 1240 static int 1241 rootnex_map_handle(ddi_map_req_t *mp) 1242 { 1243 rootnex_addr_t rbase; 1244 ddi_acc_hdl_t *hp; 1245 uint_t pgoffset; 1246 struct regspec *rp; 1247 paddr_t pbase; 1248 1249 rp = mp->map_obj.rp; 1250 1251 #ifdef DDI_MAP_DEBUG 1252 ddi_map_debug( 1253 "rootnex_map_handle: <0x%x 0x%x 0x%x> handle 0x%x\n", 1254 rp->regspec_bustype, rp->regspec_addr, 1255 rp->regspec_size, mp->map_handlep); 1256 #endif /* DDI_MAP_DEBUG */ 1257 1258 /* 1259 * I/O or memory mapping: 1260 * 1261 * <bustype=0, addr=x, len=x>: memory 1262 * <bustype=1, addr=x, len=x>: i/o 1263 * <bustype>1, addr=0, len=x>: x86-compatibility i/o 1264 */ 1265 if (rp->regspec_bustype != 0) { 1266 /* 1267 * This refers to I/O space, and we don't support "mapping" 1268 * I/O space to a user. 1269 */ 1270 return (DDI_FAILURE); 1271 } 1272 1273 /* 1274 * Set up the hat_flags for the mapping. 1275 */ 1276 hp = mp->map_handlep; 1277 1278 switch (hp->ah_acc.devacc_attr_endian_flags) { 1279 case DDI_NEVERSWAP_ACC: 1280 hp->ah_hat_flags = HAT_NEVERSWAP | HAT_STRICTORDER; 1281 break; 1282 case DDI_STRUCTURE_LE_ACC: 1283 hp->ah_hat_flags = HAT_STRUCTURE_LE; 1284 break; 1285 case DDI_STRUCTURE_BE_ACC: 1286 return (DDI_FAILURE); 1287 default: 1288 return (DDI_REGS_ACC_CONFLICT); 1289 } 1290 1291 switch (hp->ah_acc.devacc_attr_dataorder) { 1292 case DDI_STRICTORDER_ACC: 1293 break; 1294 case DDI_UNORDERED_OK_ACC: 1295 hp->ah_hat_flags |= HAT_UNORDERED_OK; 1296 break; 1297 case DDI_MERGING_OK_ACC: 1298 hp->ah_hat_flags |= HAT_MERGING_OK; 1299 break; 1300 case DDI_LOADCACHING_OK_ACC: 1301 hp->ah_hat_flags |= HAT_LOADCACHING_OK; 1302 break; 1303 case DDI_STORECACHING_OK_ACC: 1304 hp->ah_hat_flags |= HAT_STORECACHING_OK; 1305 break; 1306 default: 1307 return (DDI_FAILURE); 1308 } 1309 1310 rbase = (rootnex_addr_t)rp->regspec_addr & 1311 (~(rootnex_addr_t)MMU_PAGEOFFSET); 1312 pgoffset = (ulong_t)rp->regspec_addr & MMU_PAGEOFFSET; 1313 1314 if (rp->regspec_size == 0) 1315 return (DDI_ME_INVAL); 1316 1317 #ifdef __xpv 1318 /* 1319 * If we're dom0, we're using a real device so we need to translate 1320 * the MA to a PA. 1321 */ 1322 if (DOMAIN_IS_INITDOMAIN(xen_info)) { 1323 pbase = pfn_to_pa(xen_assign_pfn(mmu_btop(rbase))) | 1324 (rbase & MMU_PAGEOFFSET); 1325 } else { 1326 pbase = rbase; 1327 } 1328 #else 1329 pbase = rbase; 1330 #endif 1331 1332 hp->ah_pfn = mmu_btop(pbase); 1333 hp->ah_pnum = mmu_btopr(rp->regspec_size + pgoffset); 1334 1335 return (DDI_SUCCESS); 1336 } 1337 1338 1339 1340 /* 1341 * ************************ 1342 * interrupt related code 1343 * ************************ 1344 */ 1345 1346 /* 1347 * rootnex_intr_ops() 1348 * bus_intr_op() function for interrupt support 1349 */ 1350 /* ARGSUSED */ 1351 static int 1352 rootnex_intr_ops(dev_info_t *pdip, dev_info_t *rdip, ddi_intr_op_t intr_op, 1353 ddi_intr_handle_impl_t *hdlp, void *result) 1354 { 1355 struct intrspec *ispec; 1356 struct ddi_parent_private_data *pdp; 1357 1358 DDI_INTR_NEXDBG((CE_CONT, 1359 "rootnex_intr_ops: pdip = %p, rdip = %p, intr_op = %x, hdlp = %p\n", 1360 (void *)pdip, (void *)rdip, intr_op, (void *)hdlp)); 1361 1362 /* Process the interrupt operation */ 1363 switch (intr_op) { 1364 case DDI_INTROP_GETCAP: 1365 /* First check with pcplusmp */ 1366 if (psm_intr_ops == NULL) 1367 return (DDI_FAILURE); 1368 1369 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_CAP, result)) { 1370 *(int *)result = 0; 1371 return (DDI_FAILURE); 1372 } 1373 break; 1374 case DDI_INTROP_SETCAP: 1375 if (psm_intr_ops == NULL) 1376 return (DDI_FAILURE); 1377 1378 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_CAP, result)) 1379 return (DDI_FAILURE); 1380 break; 1381 case DDI_INTROP_ALLOC: 1382 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1383 return (DDI_FAILURE); 1384 hdlp->ih_pri = ispec->intrspec_pri; 1385 *(int *)result = hdlp->ih_scratch1; 1386 break; 1387 case DDI_INTROP_FREE: 1388 pdp = ddi_get_parent_data(rdip); 1389 /* 1390 * Special case for 'pcic' driver' only. 1391 * If an intrspec was created for it, clean it up here 1392 * See detailed comments on this in the function 1393 * rootnex_get_ispec(). 1394 */ 1395 if (pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1396 kmem_free(pdp->par_intr, sizeof (struct intrspec) * 1397 pdp->par_nintr); 1398 /* 1399 * Set it to zero; so that 1400 * DDI framework doesn't free it again 1401 */ 1402 pdp->par_intr = NULL; 1403 pdp->par_nintr = 0; 1404 } 1405 break; 1406 case DDI_INTROP_GETPRI: 1407 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1408 return (DDI_FAILURE); 1409 *(int *)result = ispec->intrspec_pri; 1410 break; 1411 case DDI_INTROP_SETPRI: 1412 /* Validate the interrupt priority passed to us */ 1413 if (*(int *)result > LOCK_LEVEL) 1414 return (DDI_FAILURE); 1415 1416 /* Ensure that PSM is all initialized and ispec is ok */ 1417 if ((psm_intr_ops == NULL) || 1418 ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL)) 1419 return (DDI_FAILURE); 1420 1421 /* Change the priority */ 1422 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_PRI, result) == 1423 PSM_FAILURE) 1424 return (DDI_FAILURE); 1425 1426 /* update the ispec with the new priority */ 1427 ispec->intrspec_pri = *(int *)result; 1428 break; 1429 case DDI_INTROP_ADDISR: 1430 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1431 return (DDI_FAILURE); 1432 ispec->intrspec_func = hdlp->ih_cb_func; 1433 break; 1434 case DDI_INTROP_REMISR: 1435 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1436 return (DDI_FAILURE); 1437 ispec->intrspec_func = (uint_t (*)()) 0; 1438 break; 1439 case DDI_INTROP_ENABLE: 1440 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1441 return (DDI_FAILURE); 1442 1443 /* Call psmi to translate irq with the dip */ 1444 if (psm_intr_ops == NULL) 1445 return (DDI_FAILURE); 1446 1447 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1448 (void) (*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_XLATE_VECTOR, 1449 (int *)&hdlp->ih_vector); 1450 1451 /* Add the interrupt handler */ 1452 if (!add_avintr((void *)hdlp, ispec->intrspec_pri, 1453 hdlp->ih_cb_func, DEVI(rdip)->devi_name, hdlp->ih_vector, 1454 hdlp->ih_cb_arg1, hdlp->ih_cb_arg2, NULL, rdip)) 1455 return (DDI_FAILURE); 1456 break; 1457 case DDI_INTROP_DISABLE: 1458 if ((ispec = rootnex_get_ispec(rdip, hdlp->ih_inum)) == NULL) 1459 return (DDI_FAILURE); 1460 1461 /* Call psm_ops() to translate irq with the dip */ 1462 if (psm_intr_ops == NULL) 1463 return (DDI_FAILURE); 1464 1465 ((ihdl_plat_t *)hdlp->ih_private)->ip_ispecp = ispec; 1466 (void) (*psm_intr_ops)(rdip, hdlp, 1467 PSM_INTR_OP_XLATE_VECTOR, (int *)&hdlp->ih_vector); 1468 1469 /* Remove the interrupt handler */ 1470 rem_avintr((void *)hdlp, ispec->intrspec_pri, 1471 hdlp->ih_cb_func, hdlp->ih_vector); 1472 break; 1473 case DDI_INTROP_SETMASK: 1474 if (psm_intr_ops == NULL) 1475 return (DDI_FAILURE); 1476 1477 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_SET_MASK, NULL)) 1478 return (DDI_FAILURE); 1479 break; 1480 case DDI_INTROP_CLRMASK: 1481 if (psm_intr_ops == NULL) 1482 return (DDI_FAILURE); 1483 1484 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_CLEAR_MASK, NULL)) 1485 return (DDI_FAILURE); 1486 break; 1487 case DDI_INTROP_GETPENDING: 1488 if (psm_intr_ops == NULL) 1489 return (DDI_FAILURE); 1490 1491 if ((*psm_intr_ops)(rdip, hdlp, PSM_INTR_OP_GET_PENDING, 1492 result)) { 1493 *(int *)result = 0; 1494 return (DDI_FAILURE); 1495 } 1496 break; 1497 case DDI_INTROP_NAVAIL: 1498 case DDI_INTROP_NINTRS: 1499 *(int *)result = i_ddi_get_intx_nintrs(rdip); 1500 if (*(int *)result == 0) { 1501 /* 1502 * Special case for 'pcic' driver' only. This driver 1503 * driver is a child of 'isa' and 'rootnex' drivers. 1504 * 1505 * See detailed comments on this in the function 1506 * rootnex_get_ispec(). 1507 * 1508 * Children of 'pcic' send 'NINITR' request all the 1509 * way to rootnex driver. But, the 'pdp->par_nintr' 1510 * field may not initialized. So, we fake it here 1511 * to return 1 (a la what PCMCIA nexus does). 1512 */ 1513 if (strcmp(ddi_get_name(rdip), "pcic") == 0) 1514 *(int *)result = 1; 1515 else 1516 return (DDI_FAILURE); 1517 } 1518 break; 1519 case DDI_INTROP_SUPPORTED_TYPES: 1520 *(int *)result = DDI_INTR_TYPE_FIXED; /* Always ... */ 1521 break; 1522 default: 1523 return (DDI_FAILURE); 1524 } 1525 1526 return (DDI_SUCCESS); 1527 } 1528 1529 1530 /* 1531 * rootnex_get_ispec() 1532 * convert an interrupt number to an interrupt specification. 1533 * The interrupt number determines which interrupt spec will be 1534 * returned if more than one exists. 1535 * 1536 * Look into the parent private data area of the 'rdip' to find out 1537 * the interrupt specification. First check to make sure there is 1538 * one that matchs "inumber" and then return a pointer to it. 1539 * 1540 * Return NULL if one could not be found. 1541 * 1542 * NOTE: This is needed for rootnex_intr_ops() 1543 */ 1544 static struct intrspec * 1545 rootnex_get_ispec(dev_info_t *rdip, int inum) 1546 { 1547 struct ddi_parent_private_data *pdp = ddi_get_parent_data(rdip); 1548 1549 /* 1550 * Special case handling for drivers that provide their own 1551 * intrspec structures instead of relying on the DDI framework. 1552 * 1553 * A broken hardware driver in ON could potentially provide its 1554 * own intrspec structure, instead of relying on the hardware. 1555 * If these drivers are children of 'rootnex' then we need to 1556 * continue to provide backward compatibility to them here. 1557 * 1558 * Following check is a special case for 'pcic' driver which 1559 * was found to have broken hardwre andby provides its own intrspec. 1560 * 1561 * Verbatim comments from this driver are shown here: 1562 * "Don't use the ddi_add_intr since we don't have a 1563 * default intrspec in all cases." 1564 * 1565 * Since an 'ispec' may not be always created for it, 1566 * check for that and create one if so. 1567 * 1568 * NOTE: Currently 'pcic' is the only driver found to do this. 1569 */ 1570 if (!pdp->par_intr && strcmp(ddi_get_name(rdip), "pcic") == 0) { 1571 pdp->par_nintr = 1; 1572 pdp->par_intr = kmem_zalloc(sizeof (struct intrspec) * 1573 pdp->par_nintr, KM_SLEEP); 1574 } 1575 1576 /* Validate the interrupt number */ 1577 if (inum >= pdp->par_nintr) 1578 return (NULL); 1579 1580 /* Get the interrupt structure pointer and return that */ 1581 return ((struct intrspec *)&pdp->par_intr[inum]); 1582 } 1583 1584 1585 /* 1586 * ****************** 1587 * dma related code 1588 * ****************** 1589 */ 1590 1591 /*ARGSUSED*/ 1592 static int 1593 rootnex_coredma_allochdl(dev_info_t *dip, dev_info_t *rdip, 1594 ddi_dma_attr_t *attr, int (*waitfp)(caddr_t), caddr_t arg, 1595 ddi_dma_handle_t *handlep) 1596 { 1597 uint64_t maxsegmentsize_ll; 1598 uint_t maxsegmentsize; 1599 ddi_dma_impl_t *hp; 1600 rootnex_dma_t *dma; 1601 uint64_t count_max; 1602 uint64_t seg; 1603 int kmflag; 1604 int e; 1605 1606 1607 /* convert our sleep flags */ 1608 if (waitfp == DDI_DMA_SLEEP) { 1609 kmflag = KM_SLEEP; 1610 } else { 1611 kmflag = KM_NOSLEEP; 1612 } 1613 1614 /* 1615 * We try to do only one memory allocation here. We'll do a little 1616 * pointer manipulation later. If the bind ends up taking more than 1617 * our prealloc's space, we'll have to allocate more memory in the 1618 * bind operation. Not great, but much better than before and the 1619 * best we can do with the current bind interfaces. 1620 */ 1621 hp = kmem_cache_alloc(rootnex_state->r_dmahdl_cache, kmflag); 1622 if (hp == NULL) { 1623 if (waitfp != DDI_DMA_DONTWAIT) { 1624 ddi_set_callback(waitfp, arg, 1625 &rootnex_state->r_dvma_call_list_id); 1626 } 1627 return (DDI_DMA_NORESOURCES); 1628 } 1629 1630 /* Do our pointer manipulation now, align the structures */ 1631 hp->dmai_private = (void *)(((uintptr_t)hp + 1632 (uintptr_t)sizeof (ddi_dma_impl_t) + 0x7) & ~0x7); 1633 dma = (rootnex_dma_t *)hp->dmai_private; 1634 dma->dp_prealloc_buffer = (uchar_t *)(((uintptr_t)dma + 1635 sizeof (rootnex_dma_t) + 0x7) & ~0x7); 1636 1637 /* setup the handle */ 1638 rootnex_clean_dmahdl(hp); 1639 dma->dp_dip = rdip; 1640 dma->dp_sglinfo.si_min_addr = attr->dma_attr_addr_lo; 1641 dma->dp_sglinfo.si_max_addr = attr->dma_attr_addr_hi; 1642 hp->dmai_minxfer = attr->dma_attr_minxfer; 1643 hp->dmai_burstsizes = attr->dma_attr_burstsizes; 1644 hp->dmai_rdip = rdip; 1645 hp->dmai_attr = *attr; 1646 1647 /* we don't need to worry about the SPL since we do a tryenter */ 1648 mutex_init(&dma->dp_mutex, NULL, MUTEX_DRIVER, NULL); 1649 1650 /* 1651 * Figure out our maximum segment size. If the segment size is greater 1652 * than 4G, we will limit it to (4G - 1) since the max size of a dma 1653 * object (ddi_dma_obj_t.dmao_size) is 32 bits. dma_attr_seg and 1654 * dma_attr_count_max are size-1 type values. 1655 * 1656 * Maximum segment size is the largest physically contiguous chunk of 1657 * memory that we can return from a bind (i.e. the maximum size of a 1658 * single cookie). 1659 */ 1660 1661 /* handle the rollover cases */ 1662 seg = attr->dma_attr_seg + 1; 1663 if (seg < attr->dma_attr_seg) { 1664 seg = attr->dma_attr_seg; 1665 } 1666 count_max = attr->dma_attr_count_max + 1; 1667 if (count_max < attr->dma_attr_count_max) { 1668 count_max = attr->dma_attr_count_max; 1669 } 1670 1671 /* 1672 * granularity may or may not be a power of two. If it isn't, we can't 1673 * use a simple mask. 1674 */ 1675 if (attr->dma_attr_granular & (attr->dma_attr_granular - 1)) { 1676 dma->dp_granularity_power_2 = B_FALSE; 1677 } else { 1678 dma->dp_granularity_power_2 = B_TRUE; 1679 } 1680 1681 /* 1682 * maxxfer should be a whole multiple of granularity. If we're going to 1683 * break up a window because we're greater than maxxfer, we might as 1684 * well make sure it's maxxfer is a whole multiple so we don't have to 1685 * worry about triming the window later on for this case. 1686 */ 1687 if (attr->dma_attr_granular > 1) { 1688 if (dma->dp_granularity_power_2) { 1689 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1690 (attr->dma_attr_maxxfer & 1691 (attr->dma_attr_granular - 1)); 1692 } else { 1693 dma->dp_maxxfer = attr->dma_attr_maxxfer - 1694 (attr->dma_attr_maxxfer % attr->dma_attr_granular); 1695 } 1696 } else { 1697 dma->dp_maxxfer = attr->dma_attr_maxxfer; 1698 } 1699 1700 maxsegmentsize_ll = MIN(seg, dma->dp_maxxfer); 1701 maxsegmentsize_ll = MIN(maxsegmentsize_ll, count_max); 1702 if (maxsegmentsize_ll == 0 || (maxsegmentsize_ll > 0xFFFFFFFF)) { 1703 maxsegmentsize = 0xFFFFFFFF; 1704 } else { 1705 maxsegmentsize = maxsegmentsize_ll; 1706 } 1707 dma->dp_sglinfo.si_max_cookie_size = maxsegmentsize; 1708 dma->dp_sglinfo.si_segmask = attr->dma_attr_seg; 1709 1710 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1711 if (rootnex_alloc_check_parms) { 1712 e = rootnex_valid_alloc_parms(attr, maxsegmentsize); 1713 if (e != DDI_SUCCESS) { 1714 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ALLOC_FAIL]); 1715 (void) rootnex_dma_freehdl(dip, rdip, 1716 (ddi_dma_handle_t)hp); 1717 return (e); 1718 } 1719 } 1720 1721 *handlep = (ddi_dma_handle_t)hp; 1722 1723 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1724 DTRACE_PROBE1(rootnex__alloc__handle, uint64_t, 1725 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1726 1727 return (DDI_SUCCESS); 1728 } 1729 1730 1731 /* 1732 * rootnex_dma_allochdl() 1733 * called from ddi_dma_alloc_handle(). 1734 */ 1735 static int 1736 rootnex_dma_allochdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_attr_t *attr, 1737 int (*waitfp)(caddr_t), caddr_t arg, ddi_dma_handle_t *handlep) 1738 { 1739 #if !defined(__xpv) 1740 uint_t error = ENOTSUP; 1741 int retval; 1742 1743 retval = iommulib_nex_open(rdip, &error); 1744 1745 if (retval != DDI_SUCCESS && error == ENOTSUP) { 1746 /* No IOMMU */ 1747 return (rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1748 handlep)); 1749 } else if (retval != DDI_SUCCESS) { 1750 return (DDI_FAILURE); 1751 } 1752 1753 ASSERT(IOMMU_USED(rdip)); 1754 1755 /* has an IOMMU */ 1756 return (iommulib_nexdma_allochdl(dip, rdip, attr, 1757 waitfp, arg, handlep)); 1758 #else 1759 return (rootnex_coredma_allochdl(dip, rdip, attr, waitfp, arg, 1760 handlep)); 1761 #endif 1762 } 1763 1764 /*ARGSUSED*/ 1765 static int 1766 rootnex_coredma_freehdl(dev_info_t *dip, dev_info_t *rdip, 1767 ddi_dma_handle_t handle) 1768 { 1769 ddi_dma_impl_t *hp; 1770 rootnex_dma_t *dma; 1771 1772 1773 hp = (ddi_dma_impl_t *)handle; 1774 dma = (rootnex_dma_t *)hp->dmai_private; 1775 1776 /* unbind should have been called first */ 1777 ASSERT(!dma->dp_inuse); 1778 1779 mutex_destroy(&dma->dp_mutex); 1780 kmem_cache_free(rootnex_state->r_dmahdl_cache, hp); 1781 1782 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1783 DTRACE_PROBE1(rootnex__free__handle, uint64_t, 1784 rootnex_cnt[ROOTNEX_CNT_ACTIVE_HDLS]); 1785 1786 if (rootnex_state->r_dvma_call_list_id) 1787 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 1788 1789 return (DDI_SUCCESS); 1790 } 1791 1792 /* 1793 * rootnex_dma_freehdl() 1794 * called from ddi_dma_free_handle(). 1795 */ 1796 static int 1797 rootnex_dma_freehdl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle) 1798 { 1799 #if !defined(__xpv) 1800 if (IOMMU_USED(rdip)) { 1801 return (iommulib_nexdma_freehdl(dip, rdip, handle)); 1802 } 1803 #endif 1804 return (rootnex_coredma_freehdl(dip, rdip, handle)); 1805 } 1806 1807 1808 /*ARGSUSED*/ 1809 static int 1810 rootnex_coredma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 1811 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 1812 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 1813 { 1814 rootnex_sglinfo_t *sinfo; 1815 ddi_dma_attr_t *attr; 1816 ddi_dma_impl_t *hp; 1817 rootnex_dma_t *dma; 1818 int kmflag; 1819 int e; 1820 1821 1822 hp = (ddi_dma_impl_t *)handle; 1823 dma = (rootnex_dma_t *)hp->dmai_private; 1824 sinfo = &dma->dp_sglinfo; 1825 attr = &hp->dmai_attr; 1826 1827 hp->dmai_rflags = dmareq->dmar_flags & DMP_DDIFLAGS; 1828 1829 /* 1830 * This is useful for debugging a driver. Not as useful in a production 1831 * system. The only time this will fail is if you have a driver bug. 1832 */ 1833 if (rootnex_bind_check_inuse) { 1834 /* 1835 * No one else should ever have this lock unless someone else 1836 * is trying to use this handle. So contention on the lock 1837 * is the same as inuse being set. 1838 */ 1839 e = mutex_tryenter(&dma->dp_mutex); 1840 if (e == 0) { 1841 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1842 return (DDI_DMA_INUSE); 1843 } 1844 if (dma->dp_inuse) { 1845 mutex_exit(&dma->dp_mutex); 1846 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1847 return (DDI_DMA_INUSE); 1848 } 1849 dma->dp_inuse = B_TRUE; 1850 mutex_exit(&dma->dp_mutex); 1851 } 1852 1853 /* check the ddi_dma_attr arg to make sure it makes a little sense */ 1854 if (rootnex_bind_check_parms) { 1855 e = rootnex_valid_bind_parms(dmareq, attr); 1856 if (e != DDI_SUCCESS) { 1857 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1858 rootnex_clean_dmahdl(hp); 1859 return (e); 1860 } 1861 } 1862 1863 /* save away the original bind info */ 1864 dma->dp_dma = dmareq->dmar_object; 1865 1866 #if !defined(__xpv) 1867 if (rootnex_state->r_intel_iommu_enabled) { 1868 e = intel_iommu_map_sgl(handle, dmareq, 1869 rootnex_state->r_prealloc_cookies); 1870 1871 switch (e) { 1872 case IOMMU_SGL_SUCCESS: 1873 goto rootnex_sgl_end; 1874 1875 case IOMMU_SGL_DISABLE: 1876 goto rootnex_sgl_start; 1877 1878 case IOMMU_SGL_NORESOURCES: 1879 cmn_err(CE_WARN, "iommu map sgl failed for %s", 1880 ddi_node_name(dma->dp_dip)); 1881 rootnex_clean_dmahdl(hp); 1882 return (DDI_DMA_NORESOURCES); 1883 1884 default: 1885 cmn_err(CE_WARN, 1886 "undefined value returned from" 1887 " intel_iommu_map_sgl: %d", 1888 e); 1889 rootnex_clean_dmahdl(hp); 1890 return (DDI_DMA_NORESOURCES); 1891 } 1892 } 1893 #endif 1894 1895 rootnex_sgl_start: 1896 /* 1897 * Figure out a rough estimate of what maximum number of pages this 1898 * buffer could use (a high estimate of course). 1899 */ 1900 sinfo->si_max_pages = mmu_btopr(dma->dp_dma.dmao_size) + 1; 1901 1902 /* 1903 * We'll use the pre-allocated cookies for any bind that will *always* 1904 * fit (more important to be consistent, we don't want to create 1905 * additional degenerate cases). 1906 */ 1907 if (sinfo->si_max_pages <= rootnex_state->r_prealloc_cookies) { 1908 dma->dp_cookies = (ddi_dma_cookie_t *)dma->dp_prealloc_buffer; 1909 dma->dp_need_to_free_cookie = B_FALSE; 1910 DTRACE_PROBE2(rootnex__bind__prealloc, dev_info_t *, rdip, 1911 uint_t, sinfo->si_max_pages); 1912 1913 /* 1914 * For anything larger than that, we'll go ahead and allocate the 1915 * maximum number of pages we expect to see. Hopefuly, we won't be 1916 * seeing this path in the fast path for high performance devices very 1917 * frequently. 1918 * 1919 * a ddi bind interface that allowed the driver to provide storage to 1920 * the bind interface would speed this case up. 1921 */ 1922 } else { 1923 /* convert the sleep flags */ 1924 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 1925 kmflag = KM_SLEEP; 1926 } else { 1927 kmflag = KM_NOSLEEP; 1928 } 1929 1930 /* 1931 * Save away how much memory we allocated. If we're doing a 1932 * nosleep, the alloc could fail... 1933 */ 1934 dma->dp_cookie_size = sinfo->si_max_pages * 1935 sizeof (ddi_dma_cookie_t); 1936 dma->dp_cookies = kmem_alloc(dma->dp_cookie_size, kmflag); 1937 if (dma->dp_cookies == NULL) { 1938 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 1939 rootnex_clean_dmahdl(hp); 1940 return (DDI_DMA_NORESOURCES); 1941 } 1942 dma->dp_need_to_free_cookie = B_TRUE; 1943 DTRACE_PROBE2(rootnex__bind__alloc, dev_info_t *, rdip, uint_t, 1944 sinfo->si_max_pages); 1945 } 1946 hp->dmai_cookie = dma->dp_cookies; 1947 1948 /* 1949 * Get the real sgl. rootnex_get_sgl will fill in cookie array while 1950 * looking at the contraints in the dma structure. It will then put some 1951 * additional state about the sgl in the dma struct (i.e. is the sgl 1952 * clean, or do we need to do some munging; how many pages need to be 1953 * copied, etc.) 1954 */ 1955 rootnex_get_sgl(&dmareq->dmar_object, dma->dp_cookies, 1956 &dma->dp_sglinfo); 1957 1958 rootnex_sgl_end: 1959 ASSERT(sinfo->si_sgl_size <= sinfo->si_max_pages); 1960 /* if we don't need a copy buffer, we don't need to sync */ 1961 if (sinfo->si_copybuf_req == 0) { 1962 hp->dmai_rflags |= DMP_NOSYNC; 1963 } 1964 1965 /* 1966 * if we don't need the copybuf and we don't need to do a partial, we 1967 * hit the fast path. All the high performance devices should be trying 1968 * to hit this path. To hit this path, a device should be able to reach 1969 * all of memory, shouldn't try to bind more than it can transfer, and 1970 * the buffer shouldn't require more cookies than the driver/device can 1971 * handle [sgllen]). 1972 */ 1973 if ((sinfo->si_copybuf_req == 0) && 1974 (sinfo->si_sgl_size <= attr->dma_attr_sgllen) && 1975 (dma->dp_dma.dmao_size < dma->dp_maxxfer)) { 1976 /* 1977 * If the driver supports FMA, insert the handle in the FMA DMA 1978 * handle cache. 1979 */ 1980 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) { 1981 hp->dmai_error.err_cf = rootnex_dma_check; 1982 (void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL); 1983 } 1984 1985 /* 1986 * copy out the first cookie and ccountp, set the cookie 1987 * pointer to the second cookie. The first cookie is passed 1988 * back on the stack. Additional cookies are accessed via 1989 * ddi_dma_nextcookie() 1990 */ 1991 *cookiep = dma->dp_cookies[0]; 1992 *ccountp = sinfo->si_sgl_size; 1993 hp->dmai_cookie++; 1994 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 1995 hp->dmai_nwin = 1; 1996 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 1997 DTRACE_PROBE3(rootnex__bind__fast, dev_info_t *, rdip, uint64_t, 1998 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 1999 dma->dp_dma.dmao_size); 2000 return (DDI_DMA_MAPPED); 2001 } 2002 2003 /* 2004 * go to the slow path, we may need to alloc more memory, create 2005 * multiple windows, and munge up a sgl to make the device happy. 2006 */ 2007 e = rootnex_bind_slowpath(hp, dmareq, dma, attr, kmflag); 2008 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 2009 if (dma->dp_need_to_free_cookie) { 2010 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2011 } 2012 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_BIND_FAIL]); 2013 rootnex_clean_dmahdl(hp); /* must be after free cookie */ 2014 return (e); 2015 } 2016 2017 /* 2018 * If the driver supports FMA, insert the handle in the FMA DMA handle 2019 * cache. 2020 */ 2021 if (attr->dma_attr_flags & DDI_DMA_FLAGERR) { 2022 hp->dmai_error.err_cf = rootnex_dma_check; 2023 (void) ndi_fmc_insert(rdip, DMA_HANDLE, hp, NULL); 2024 } 2025 2026 /* if the first window uses the copy buffer, sync it for the device */ 2027 if ((dma->dp_window[dma->dp_current_win].wd_dosync) && 2028 (hp->dmai_rflags & DDI_DMA_WRITE)) { 2029 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 2030 DDI_DMA_SYNC_FORDEV); 2031 } 2032 2033 /* 2034 * copy out the first cookie and ccountp, set the cookie pointer to the 2035 * second cookie. Make sure the partial flag is set/cleared correctly. 2036 * If we have a partial map (i.e. multiple windows), the number of 2037 * cookies we return is the number of cookies in the first window. 2038 */ 2039 if (e == DDI_DMA_MAPPED) { 2040 hp->dmai_rflags &= ~DDI_DMA_PARTIAL; 2041 *ccountp = sinfo->si_sgl_size; 2042 } else { 2043 hp->dmai_rflags |= DDI_DMA_PARTIAL; 2044 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2045 ASSERT(hp->dmai_nwin <= dma->dp_max_win); 2046 } 2047 *cookiep = dma->dp_cookies[0]; 2048 hp->dmai_cookie++; 2049 2050 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2051 DTRACE_PROBE3(rootnex__bind__slow, dev_info_t *, rdip, uint64_t, 2052 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS], uint_t, 2053 dma->dp_dma.dmao_size); 2054 return (e); 2055 } 2056 2057 2058 /* 2059 * rootnex_dma_bindhdl() 2060 * called from ddi_dma_addr_bind_handle() and ddi_dma_buf_bind_handle(). 2061 */ 2062 static int 2063 rootnex_dma_bindhdl(dev_info_t *dip, dev_info_t *rdip, 2064 ddi_dma_handle_t handle, struct ddi_dma_req *dmareq, 2065 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2066 { 2067 #if !defined(__xpv) 2068 if (IOMMU_USED(rdip)) { 2069 return (iommulib_nexdma_bindhdl(dip, rdip, handle, dmareq, 2070 cookiep, ccountp)); 2071 } 2072 #endif 2073 return (rootnex_coredma_bindhdl(dip, rdip, handle, dmareq, 2074 cookiep, ccountp)); 2075 } 2076 2077 /*ARGSUSED*/ 2078 static int 2079 rootnex_coredma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2080 ddi_dma_handle_t handle) 2081 { 2082 ddi_dma_impl_t *hp; 2083 rootnex_dma_t *dma; 2084 int e; 2085 2086 hp = (ddi_dma_impl_t *)handle; 2087 dma = (rootnex_dma_t *)hp->dmai_private; 2088 2089 /* make sure the buffer wasn't free'd before calling unbind */ 2090 if (rootnex_unbind_verify_buffer) { 2091 e = rootnex_verify_buffer(dma); 2092 if (e != DDI_SUCCESS) { 2093 ASSERT(0); 2094 return (DDI_FAILURE); 2095 } 2096 } 2097 2098 /* sync the current window before unbinding the buffer */ 2099 if (dma->dp_window && dma->dp_window[dma->dp_current_win].wd_dosync && 2100 (hp->dmai_rflags & DDI_DMA_READ)) { 2101 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 2102 DDI_DMA_SYNC_FORCPU); 2103 } 2104 2105 /* 2106 * If the driver supports FMA, remove the handle in the FMA DMA handle 2107 * cache. 2108 */ 2109 if (hp->dmai_attr.dma_attr_flags & DDI_DMA_FLAGERR) { 2110 if ((DEVI(rdip)->devi_fmhdl != NULL) && 2111 (DDI_FM_DMA_ERR_CAP(DEVI(rdip)->devi_fmhdl->fh_cap))) { 2112 (void) ndi_fmc_remove(rdip, DMA_HANDLE, hp); 2113 } 2114 } 2115 2116 /* 2117 * cleanup and copy buffer or window state. if we didn't use the copy 2118 * buffer or windows, there won't be much to do :-) 2119 */ 2120 rootnex_teardown_copybuf(dma); 2121 rootnex_teardown_windows(dma); 2122 2123 #if !defined(__xpv) 2124 /* 2125 * If intel iommu enabled, clean up the page tables and free the dvma 2126 */ 2127 if (rootnex_state->r_intel_iommu_enabled) { 2128 intel_iommu_unmap_sgl(handle); 2129 } 2130 #endif 2131 2132 /* 2133 * If we had to allocate space to for the worse case sgl (it didn't 2134 * fit into our pre-allocate buffer), free that up now 2135 */ 2136 if (dma->dp_need_to_free_cookie) { 2137 kmem_free(dma->dp_cookies, dma->dp_cookie_size); 2138 } 2139 2140 /* 2141 * clean up the handle so it's ready for the next bind (i.e. if the 2142 * handle is reused). 2143 */ 2144 rootnex_clean_dmahdl(hp); 2145 2146 if (rootnex_state->r_dvma_call_list_id) 2147 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 2148 2149 ROOTNEX_PROF_DEC(&rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2150 DTRACE_PROBE1(rootnex__unbind, uint64_t, 2151 rootnex_cnt[ROOTNEX_CNT_ACTIVE_BINDS]); 2152 2153 return (DDI_SUCCESS); 2154 } 2155 2156 /* 2157 * rootnex_dma_unbindhdl() 2158 * called from ddi_dma_unbind_handle() 2159 */ 2160 /*ARGSUSED*/ 2161 static int 2162 rootnex_dma_unbindhdl(dev_info_t *dip, dev_info_t *rdip, 2163 ddi_dma_handle_t handle) 2164 { 2165 #if !defined(__xpv) 2166 if (IOMMU_USED(rdip)) { 2167 return (iommulib_nexdma_unbindhdl(dip, rdip, handle)); 2168 } 2169 #endif 2170 return (rootnex_coredma_unbindhdl(dip, rdip, handle)); 2171 } 2172 2173 /*ARGSUSED*/ 2174 static void 2175 rootnex_coredma_reset_cookies(dev_info_t *dip, ddi_dma_handle_t handle) 2176 { 2177 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2178 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2179 2180 hp->dmai_cookie = &dma->dp_cookies[0]; 2181 hp->dmai_cookie++; 2182 } 2183 2184 /*ARGSUSED*/ 2185 static int 2186 rootnex_coredma_get_cookies(dev_info_t *dip, ddi_dma_handle_t handle, 2187 ddi_dma_cookie_t *cookiep, uint_t *ccountp) 2188 { 2189 ddi_dma_impl_t *hp = (ddi_dma_impl_t *)handle; 2190 rootnex_dma_t *dma = (rootnex_dma_t *)hp->dmai_private; 2191 2192 2193 if (hp->dmai_rflags & DDI_DMA_PARTIAL) { 2194 *ccountp = dma->dp_window[dma->dp_current_win].wd_cookie_cnt; 2195 } else { 2196 *ccountp = dma->dp_sglinfo.si_sgl_size; 2197 } 2198 *cookiep = dma->dp_cookies[0]; 2199 2200 /* reset the cookies */ 2201 hp->dmai_cookie = &dma->dp_cookies[0]; 2202 hp->dmai_cookie++; 2203 2204 return (DDI_SUCCESS); 2205 } 2206 2207 /* 2208 * rootnex_verify_buffer() 2209 * verify buffer wasn't free'd 2210 */ 2211 static int 2212 rootnex_verify_buffer(rootnex_dma_t *dma) 2213 { 2214 page_t **pplist; 2215 caddr_t vaddr; 2216 uint_t pcnt; 2217 uint_t poff; 2218 page_t *pp; 2219 char b; 2220 int i; 2221 2222 /* Figure out how many pages this buffer occupies */ 2223 if (dma->dp_dma.dmao_type == DMA_OTYP_PAGES) { 2224 poff = dma->dp_dma.dmao_obj.pp_obj.pp_offset & MMU_PAGEOFFSET; 2225 } else { 2226 vaddr = dma->dp_dma.dmao_obj.virt_obj.v_addr; 2227 poff = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2228 } 2229 pcnt = mmu_btopr(dma->dp_dma.dmao_size + poff); 2230 2231 switch (dma->dp_dma.dmao_type) { 2232 case DMA_OTYP_PAGES: 2233 /* 2234 * for a linked list of pp's walk through them to make sure 2235 * they're locked and not free. 2236 */ 2237 pp = dma->dp_dma.dmao_obj.pp_obj.pp_pp; 2238 for (i = 0; i < pcnt; i++) { 2239 if (PP_ISFREE(pp) || !PAGE_LOCKED(pp)) { 2240 return (DDI_FAILURE); 2241 } 2242 pp = pp->p_next; 2243 } 2244 break; 2245 2246 case DMA_OTYP_VADDR: 2247 case DMA_OTYP_BUFVADDR: 2248 pplist = dma->dp_dma.dmao_obj.virt_obj.v_priv; 2249 /* 2250 * for an array of pp's walk through them to make sure they're 2251 * not free. It's possible that they may not be locked. 2252 */ 2253 if (pplist) { 2254 for (i = 0; i < pcnt; i++) { 2255 if (PP_ISFREE(pplist[i])) { 2256 return (DDI_FAILURE); 2257 } 2258 } 2259 2260 /* For a virtual address, try to peek at each page */ 2261 } else { 2262 if (dma->dp_sglinfo.si_asp == &kas) { 2263 for (i = 0; i < pcnt; i++) { 2264 if (ddi_peek8(NULL, vaddr, &b) == 2265 DDI_FAILURE) 2266 return (DDI_FAILURE); 2267 vaddr += MMU_PAGESIZE; 2268 } 2269 } 2270 } 2271 break; 2272 2273 default: 2274 ASSERT(0); 2275 break; 2276 } 2277 2278 return (DDI_SUCCESS); 2279 } 2280 2281 2282 /* 2283 * rootnex_clean_dmahdl() 2284 * Clean the dma handle. This should be called on a handle alloc and an 2285 * unbind handle. Set the handle state to the default settings. 2286 */ 2287 static void 2288 rootnex_clean_dmahdl(ddi_dma_impl_t *hp) 2289 { 2290 rootnex_dma_t *dma; 2291 2292 2293 dma = (rootnex_dma_t *)hp->dmai_private; 2294 2295 hp->dmai_nwin = 0; 2296 dma->dp_current_cookie = 0; 2297 dma->dp_copybuf_size = 0; 2298 dma->dp_window = NULL; 2299 dma->dp_cbaddr = NULL; 2300 dma->dp_inuse = B_FALSE; 2301 dma->dp_need_to_free_cookie = B_FALSE; 2302 dma->dp_need_to_free_window = B_FALSE; 2303 dma->dp_partial_required = B_FALSE; 2304 dma->dp_trim_required = B_FALSE; 2305 dma->dp_sglinfo.si_copybuf_req = 0; 2306 #if !defined(__amd64) 2307 dma->dp_cb_remaping = B_FALSE; 2308 dma->dp_kva = NULL; 2309 #endif 2310 2311 /* FMA related initialization */ 2312 hp->dmai_fault = 0; 2313 hp->dmai_fault_check = NULL; 2314 hp->dmai_fault_notify = NULL; 2315 hp->dmai_error.err_ena = 0; 2316 hp->dmai_error.err_status = DDI_FM_OK; 2317 hp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED; 2318 hp->dmai_error.err_ontrap = NULL; 2319 hp->dmai_error.err_fep = NULL; 2320 hp->dmai_error.err_cf = NULL; 2321 } 2322 2323 2324 /* 2325 * rootnex_valid_alloc_parms() 2326 * Called in ddi_dma_alloc_handle path to validate its parameters. 2327 */ 2328 static int 2329 rootnex_valid_alloc_parms(ddi_dma_attr_t *attr, uint_t maxsegmentsize) 2330 { 2331 if ((attr->dma_attr_seg < MMU_PAGEOFFSET) || 2332 (attr->dma_attr_count_max < MMU_PAGEOFFSET) || 2333 (attr->dma_attr_granular > MMU_PAGESIZE) || 2334 (attr->dma_attr_maxxfer < MMU_PAGESIZE)) { 2335 return (DDI_DMA_BADATTR); 2336 } 2337 2338 if (attr->dma_attr_addr_hi <= attr->dma_attr_addr_lo) { 2339 return (DDI_DMA_BADATTR); 2340 } 2341 2342 if ((attr->dma_attr_seg & MMU_PAGEOFFSET) != MMU_PAGEOFFSET || 2343 MMU_PAGESIZE & (attr->dma_attr_granular - 1) || 2344 attr->dma_attr_sgllen <= 0) { 2345 return (DDI_DMA_BADATTR); 2346 } 2347 2348 /* We should be able to DMA into every byte offset in a page */ 2349 if (maxsegmentsize < MMU_PAGESIZE) { 2350 return (DDI_DMA_BADATTR); 2351 } 2352 2353 return (DDI_SUCCESS); 2354 } 2355 2356 2357 /* 2358 * rootnex_valid_bind_parms() 2359 * Called in ddi_dma_*_bind_handle path to validate its parameters. 2360 */ 2361 /* ARGSUSED */ 2362 static int 2363 rootnex_valid_bind_parms(ddi_dma_req_t *dmareq, ddi_dma_attr_t *attr) 2364 { 2365 #if !defined(__amd64) 2366 /* 2367 * we only support up to a 2G-1 transfer size on 32-bit kernels so 2368 * we can track the offset for the obsoleted interfaces. 2369 */ 2370 if (dmareq->dmar_object.dmao_size > 0x7FFFFFFF) { 2371 return (DDI_DMA_TOOBIG); 2372 } 2373 #endif 2374 2375 return (DDI_SUCCESS); 2376 } 2377 2378 2379 /* 2380 * rootnex_get_sgl() 2381 * Called in bind fastpath to get the sgl. Most of this will be replaced 2382 * with a call to the vm layer when vm2.0 comes around... 2383 */ 2384 static void 2385 rootnex_get_sgl(ddi_dma_obj_t *dmar_object, ddi_dma_cookie_t *sgl, 2386 rootnex_sglinfo_t *sglinfo) 2387 { 2388 ddi_dma_atyp_t buftype; 2389 rootnex_addr_t raddr; 2390 uint64_t last_page; 2391 uint64_t offset; 2392 uint64_t addrhi; 2393 uint64_t addrlo; 2394 uint64_t maxseg; 2395 page_t **pplist; 2396 uint64_t paddr; 2397 uint32_t psize; 2398 uint32_t size; 2399 caddr_t vaddr; 2400 uint_t pcnt; 2401 page_t *pp; 2402 uint_t cnt; 2403 2404 2405 /* shortcuts */ 2406 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 2407 vaddr = dmar_object->dmao_obj.virt_obj.v_addr; 2408 maxseg = sglinfo->si_max_cookie_size; 2409 buftype = dmar_object->dmao_type; 2410 addrhi = sglinfo->si_max_addr; 2411 addrlo = sglinfo->si_min_addr; 2412 size = dmar_object->dmao_size; 2413 2414 pcnt = 0; 2415 cnt = 0; 2416 2417 /* 2418 * if we were passed down a linked list of pages, i.e. pointer to 2419 * page_t, use this to get our physical address and buf offset. 2420 */ 2421 if (buftype == DMA_OTYP_PAGES) { 2422 pp = dmar_object->dmao_obj.pp_obj.pp_pp; 2423 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2424 offset = dmar_object->dmao_obj.pp_obj.pp_offset & 2425 MMU_PAGEOFFSET; 2426 paddr = pfn_to_pa(pp->p_pagenum) + offset; 2427 psize = MIN(size, (MMU_PAGESIZE - offset)); 2428 pp = pp->p_next; 2429 sglinfo->si_asp = NULL; 2430 2431 /* 2432 * We weren't passed down a linked list of pages, but if we were passed 2433 * down an array of pages, use this to get our physical address and buf 2434 * offset. 2435 */ 2436 } else if (pplist != NULL) { 2437 ASSERT((buftype == DMA_OTYP_VADDR) || 2438 (buftype == DMA_OTYP_BUFVADDR)); 2439 2440 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2441 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2442 if (sglinfo->si_asp == NULL) { 2443 sglinfo->si_asp = &kas; 2444 } 2445 2446 ASSERT(!PP_ISFREE(pplist[pcnt])); 2447 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2448 paddr += offset; 2449 psize = MIN(size, (MMU_PAGESIZE - offset)); 2450 pcnt++; 2451 2452 /* 2453 * All we have is a virtual address, we'll need to call into the VM 2454 * to get the physical address. 2455 */ 2456 } else { 2457 ASSERT((buftype == DMA_OTYP_VADDR) || 2458 (buftype == DMA_OTYP_BUFVADDR)); 2459 2460 offset = (uintptr_t)vaddr & MMU_PAGEOFFSET; 2461 sglinfo->si_asp = dmar_object->dmao_obj.virt_obj.v_as; 2462 if (sglinfo->si_asp == NULL) { 2463 sglinfo->si_asp = &kas; 2464 } 2465 2466 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, vaddr)); 2467 paddr += offset; 2468 psize = MIN(size, (MMU_PAGESIZE - offset)); 2469 vaddr += psize; 2470 } 2471 2472 #ifdef __xpv 2473 /* 2474 * If we're dom0, we're using a real device so we need to load 2475 * the cookies with MFNs instead of PFNs. 2476 */ 2477 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2478 #else 2479 raddr = paddr; 2480 #endif 2481 2482 /* 2483 * Setup the first cookie with the physical address of the page and the 2484 * size of the page (which takes into account the initial offset into 2485 * the page. 2486 */ 2487 sgl[cnt].dmac_laddress = raddr; 2488 sgl[cnt].dmac_size = psize; 2489 sgl[cnt].dmac_type = 0; 2490 2491 /* 2492 * Save away the buffer offset into the page. We'll need this later in 2493 * the copy buffer code to help figure out the page index within the 2494 * buffer and the offset into the current page. 2495 */ 2496 sglinfo->si_buf_offset = offset; 2497 2498 /* 2499 * If the DMA engine can't reach the physical address, increase how 2500 * much copy buffer we need. We always increase by pagesize so we don't 2501 * have to worry about converting offsets. Set a flag in the cookies 2502 * dmac_type to indicate that it uses the copy buffer. If this isn't the 2503 * last cookie, go to the next cookie (since we separate each page which 2504 * uses the copy buffer in case the copy buffer is not physically 2505 * contiguous. 2506 */ 2507 if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2508 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2509 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2510 if ((cnt + 1) < sglinfo->si_max_pages) { 2511 cnt++; 2512 sgl[cnt].dmac_laddress = 0; 2513 sgl[cnt].dmac_size = 0; 2514 sgl[cnt].dmac_type = 0; 2515 } 2516 } 2517 2518 /* 2519 * save this page's physical address so we can figure out if the next 2520 * page is physically contiguous. Keep decrementing size until we are 2521 * done with the buffer. 2522 */ 2523 last_page = raddr & MMU_PAGEMASK; 2524 size -= psize; 2525 2526 while (size > 0) { 2527 /* Get the size for this page (i.e. partial or full page) */ 2528 psize = MIN(size, MMU_PAGESIZE); 2529 2530 if (buftype == DMA_OTYP_PAGES) { 2531 /* get the paddr from the page_t */ 2532 ASSERT(!PP_ISFREE(pp) && PAGE_LOCKED(pp)); 2533 paddr = pfn_to_pa(pp->p_pagenum); 2534 pp = pp->p_next; 2535 } else if (pplist != NULL) { 2536 /* index into the array of page_t's to get the paddr */ 2537 ASSERT(!PP_ISFREE(pplist[pcnt])); 2538 paddr = pfn_to_pa(pplist[pcnt]->p_pagenum); 2539 pcnt++; 2540 } else { 2541 /* call into the VM to get the paddr */ 2542 paddr = pfn_to_pa(hat_getpfnum(sglinfo->si_asp->a_hat, 2543 vaddr)); 2544 vaddr += psize; 2545 } 2546 2547 #ifdef __xpv 2548 /* 2549 * If we're dom0, we're using a real device so we need to load 2550 * the cookies with MFNs instead of PFNs. 2551 */ 2552 raddr = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 2553 #else 2554 raddr = paddr; 2555 #endif 2556 2557 /* check to see if this page needs the copy buffer */ 2558 if ((raddr < addrlo) || ((raddr + psize) > addrhi)) { 2559 sglinfo->si_copybuf_req += MMU_PAGESIZE; 2560 2561 /* 2562 * if there is something in the current cookie, go to 2563 * the next one. We only want one page in a cookie which 2564 * uses the copybuf since the copybuf doesn't have to 2565 * be physically contiguous. 2566 */ 2567 if (sgl[cnt].dmac_size != 0) { 2568 cnt++; 2569 } 2570 sgl[cnt].dmac_laddress = raddr; 2571 sgl[cnt].dmac_size = psize; 2572 #if defined(__amd64) 2573 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF; 2574 #else 2575 /* 2576 * save the buf offset for 32-bit kernel. used in the 2577 * obsoleted interfaces. 2578 */ 2579 sgl[cnt].dmac_type = ROOTNEX_USES_COPYBUF | 2580 (dmar_object->dmao_size - size); 2581 #endif 2582 /* if this isn't the last cookie, go to the next one */ 2583 if ((cnt + 1) < sglinfo->si_max_pages) { 2584 cnt++; 2585 sgl[cnt].dmac_laddress = 0; 2586 sgl[cnt].dmac_size = 0; 2587 sgl[cnt].dmac_type = 0; 2588 } 2589 2590 /* 2591 * this page didn't need the copy buffer, if it's not physically 2592 * contiguous, or it would put us over a segment boundary, or it 2593 * puts us over the max cookie size, or the current sgl doesn't 2594 * have anything in it. 2595 */ 2596 } else if (((last_page + MMU_PAGESIZE) != raddr) || 2597 !(raddr & sglinfo->si_segmask) || 2598 ((sgl[cnt].dmac_size + psize) > maxseg) || 2599 (sgl[cnt].dmac_size == 0)) { 2600 /* 2601 * if we're not already in a new cookie, go to the next 2602 * cookie. 2603 */ 2604 if (sgl[cnt].dmac_size != 0) { 2605 cnt++; 2606 } 2607 2608 /* save the cookie information */ 2609 sgl[cnt].dmac_laddress = raddr; 2610 sgl[cnt].dmac_size = psize; 2611 #if defined(__amd64) 2612 sgl[cnt].dmac_type = 0; 2613 #else 2614 /* 2615 * save the buf offset for 32-bit kernel. used in the 2616 * obsoleted interfaces. 2617 */ 2618 sgl[cnt].dmac_type = dmar_object->dmao_size - size; 2619 #endif 2620 2621 /* 2622 * this page didn't need the copy buffer, it is physically 2623 * contiguous with the last page, and it's <= the max cookie 2624 * size. 2625 */ 2626 } else { 2627 sgl[cnt].dmac_size += psize; 2628 2629 /* 2630 * if this exactly == the maximum cookie size, and 2631 * it isn't the last cookie, go to the next cookie. 2632 */ 2633 if (((sgl[cnt].dmac_size + psize) == maxseg) && 2634 ((cnt + 1) < sglinfo->si_max_pages)) { 2635 cnt++; 2636 sgl[cnt].dmac_laddress = 0; 2637 sgl[cnt].dmac_size = 0; 2638 sgl[cnt].dmac_type = 0; 2639 } 2640 } 2641 2642 /* 2643 * save this page's physical address so we can figure out if the 2644 * next page is physically contiguous. Keep decrementing size 2645 * until we are done with the buffer. 2646 */ 2647 last_page = raddr; 2648 size -= psize; 2649 } 2650 2651 /* we're done, save away how many cookies the sgl has */ 2652 if (sgl[cnt].dmac_size == 0) { 2653 ASSERT(cnt < sglinfo->si_max_pages); 2654 sglinfo->si_sgl_size = cnt; 2655 } else { 2656 sglinfo->si_sgl_size = cnt + 1; 2657 } 2658 } 2659 2660 2661 /* 2662 * rootnex_bind_slowpath() 2663 * Call in the bind path if the calling driver can't use the sgl without 2664 * modifying it. We either need to use the copy buffer and/or we will end up 2665 * with a partial bind. 2666 */ 2667 static int 2668 rootnex_bind_slowpath(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2669 rootnex_dma_t *dma, ddi_dma_attr_t *attr, int kmflag) 2670 { 2671 rootnex_sglinfo_t *sinfo; 2672 rootnex_window_t *window; 2673 ddi_dma_cookie_t *cookie; 2674 size_t copybuf_used; 2675 size_t dmac_size; 2676 boolean_t partial; 2677 off_t cur_offset; 2678 page_t *cur_pp; 2679 major_t mnum; 2680 int e; 2681 int i; 2682 2683 2684 sinfo = &dma->dp_sglinfo; 2685 copybuf_used = 0; 2686 partial = B_FALSE; 2687 2688 /* 2689 * If we're using the copybuf, set the copybuf state in dma struct. 2690 * Needs to be first since it sets the copy buffer size. 2691 */ 2692 if (sinfo->si_copybuf_req != 0) { 2693 e = rootnex_setup_copybuf(hp, dmareq, dma, attr); 2694 if (e != DDI_SUCCESS) { 2695 return (e); 2696 } 2697 } else { 2698 dma->dp_copybuf_size = 0; 2699 } 2700 2701 /* 2702 * Figure out if we need to do a partial mapping. If so, figure out 2703 * if we need to trim the buffers when we munge the sgl. 2704 */ 2705 if ((dma->dp_copybuf_size < sinfo->si_copybuf_req) || 2706 (dma->dp_dma.dmao_size > dma->dp_maxxfer) || 2707 (attr->dma_attr_sgllen < sinfo->si_sgl_size)) { 2708 dma->dp_partial_required = B_TRUE; 2709 if (attr->dma_attr_granular != 1) { 2710 dma->dp_trim_required = B_TRUE; 2711 } 2712 } else { 2713 dma->dp_partial_required = B_FALSE; 2714 dma->dp_trim_required = B_FALSE; 2715 } 2716 2717 /* If we need to do a partial bind, make sure the driver supports it */ 2718 if (dma->dp_partial_required && 2719 !(dmareq->dmar_flags & DDI_DMA_PARTIAL)) { 2720 2721 mnum = ddi_driver_major(dma->dp_dip); 2722 /* 2723 * patchable which allows us to print one warning per major 2724 * number. 2725 */ 2726 if ((rootnex_bind_warn) && 2727 ((rootnex_warn_list[mnum] & ROOTNEX_BIND_WARNING) == 0)) { 2728 rootnex_warn_list[mnum] |= ROOTNEX_BIND_WARNING; 2729 cmn_err(CE_WARN, "!%s: coding error detected, the " 2730 "driver is using ddi_dma_attr(9S) incorrectly. " 2731 "There is a small risk of data corruption in " 2732 "particular with large I/Os. The driver should be " 2733 "replaced with a corrected version for proper " 2734 "system operation. To disable this warning, add " 2735 "'set rootnex:rootnex_bind_warn=0' to " 2736 "/etc/system(4).", ddi_driver_name(dma->dp_dip)); 2737 } 2738 return (DDI_DMA_TOOBIG); 2739 } 2740 2741 /* 2742 * we might need multiple windows, setup state to handle them. In this 2743 * code path, we will have at least one window. 2744 */ 2745 e = rootnex_setup_windows(hp, dma, attr, kmflag); 2746 if (e != DDI_SUCCESS) { 2747 rootnex_teardown_copybuf(dma); 2748 return (e); 2749 } 2750 2751 window = &dma->dp_window[0]; 2752 cookie = &dma->dp_cookies[0]; 2753 cur_offset = 0; 2754 rootnex_init_win(hp, dma, window, cookie, cur_offset); 2755 if (dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) { 2756 cur_pp = dmareq->dmar_object.dmao_obj.pp_obj.pp_pp; 2757 } 2758 2759 /* loop though all the cookies we got back from get_sgl() */ 2760 for (i = 0; i < sinfo->si_sgl_size; i++) { 2761 /* 2762 * If we're using the copy buffer, check this cookie and setup 2763 * its associated copy buffer state. If this cookie uses the 2764 * copy buffer, make sure we sync this window during dma_sync. 2765 */ 2766 if (dma->dp_copybuf_size > 0) { 2767 rootnex_setup_cookie(&dmareq->dmar_object, dma, cookie, 2768 cur_offset, ©buf_used, &cur_pp); 2769 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2770 window->wd_dosync = B_TRUE; 2771 } 2772 } 2773 2774 /* 2775 * save away the cookie size, since it could be modified in 2776 * the windowing code. 2777 */ 2778 dmac_size = cookie->dmac_size; 2779 2780 /* if we went over max copybuf size */ 2781 if (dma->dp_copybuf_size && 2782 (copybuf_used > dma->dp_copybuf_size)) { 2783 partial = B_TRUE; 2784 e = rootnex_copybuf_window_boundary(hp, dma, &window, 2785 cookie, cur_offset, ©buf_used); 2786 if (e != DDI_SUCCESS) { 2787 rootnex_teardown_copybuf(dma); 2788 rootnex_teardown_windows(dma); 2789 return (e); 2790 } 2791 2792 /* 2793 * if the coookie uses the copy buffer, make sure the 2794 * new window we just moved to is set to sync. 2795 */ 2796 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2797 window->wd_dosync = B_TRUE; 2798 } 2799 DTRACE_PROBE1(rootnex__copybuf__window, dev_info_t *, 2800 dma->dp_dip); 2801 2802 /* if the cookie cnt == max sgllen, move to the next window */ 2803 } else if (window->wd_cookie_cnt >= attr->dma_attr_sgllen) { 2804 partial = B_TRUE; 2805 ASSERT(window->wd_cookie_cnt == attr->dma_attr_sgllen); 2806 e = rootnex_sgllen_window_boundary(hp, dma, &window, 2807 cookie, attr, cur_offset); 2808 if (e != DDI_SUCCESS) { 2809 rootnex_teardown_copybuf(dma); 2810 rootnex_teardown_windows(dma); 2811 return (e); 2812 } 2813 2814 /* 2815 * if the coookie uses the copy buffer, make sure the 2816 * new window we just moved to is set to sync. 2817 */ 2818 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2819 window->wd_dosync = B_TRUE; 2820 } 2821 DTRACE_PROBE1(rootnex__sgllen__window, dev_info_t *, 2822 dma->dp_dip); 2823 2824 /* else if we will be over maxxfer */ 2825 } else if ((window->wd_size + dmac_size) > 2826 dma->dp_maxxfer) { 2827 partial = B_TRUE; 2828 e = rootnex_maxxfer_window_boundary(hp, dma, &window, 2829 cookie); 2830 if (e != DDI_SUCCESS) { 2831 rootnex_teardown_copybuf(dma); 2832 rootnex_teardown_windows(dma); 2833 return (e); 2834 } 2835 2836 /* 2837 * if the coookie uses the copy buffer, make sure the 2838 * new window we just moved to is set to sync. 2839 */ 2840 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 2841 window->wd_dosync = B_TRUE; 2842 } 2843 DTRACE_PROBE1(rootnex__maxxfer__window, dev_info_t *, 2844 dma->dp_dip); 2845 2846 /* else this cookie fits in the current window */ 2847 } else { 2848 window->wd_cookie_cnt++; 2849 window->wd_size += dmac_size; 2850 } 2851 2852 /* track our offset into the buffer, go to the next cookie */ 2853 ASSERT(dmac_size <= dma->dp_dma.dmao_size); 2854 ASSERT(cookie->dmac_size <= dmac_size); 2855 cur_offset += dmac_size; 2856 cookie++; 2857 } 2858 2859 /* if we ended up with a zero sized window in the end, clean it up */ 2860 if (window->wd_size == 0) { 2861 hp->dmai_nwin--; 2862 window--; 2863 } 2864 2865 ASSERT(window->wd_trim.tr_trim_last == B_FALSE); 2866 2867 if (!partial) { 2868 return (DDI_DMA_MAPPED); 2869 } 2870 2871 ASSERT(dma->dp_partial_required); 2872 return (DDI_DMA_PARTIAL_MAP); 2873 } 2874 2875 2876 /* 2877 * rootnex_setup_copybuf() 2878 * Called in bind slowpath. Figures out if we're going to use the copy 2879 * buffer, and if we do, sets up the basic state to handle it. 2880 */ 2881 static int 2882 rootnex_setup_copybuf(ddi_dma_impl_t *hp, struct ddi_dma_req *dmareq, 2883 rootnex_dma_t *dma, ddi_dma_attr_t *attr) 2884 { 2885 rootnex_sglinfo_t *sinfo; 2886 ddi_dma_attr_t lattr; 2887 size_t max_copybuf; 2888 int cansleep; 2889 int e; 2890 #if !defined(__amd64) 2891 int vmflag; 2892 #endif 2893 2894 2895 sinfo = &dma->dp_sglinfo; 2896 2897 /* read this first so it's consistent through the routine */ 2898 max_copybuf = i_ddi_copybuf_size() & MMU_PAGEMASK; 2899 2900 /* We need to call into the rootnex on ddi_dma_sync() */ 2901 hp->dmai_rflags &= ~DMP_NOSYNC; 2902 2903 /* make sure the copybuf size <= the max size */ 2904 dma->dp_copybuf_size = MIN(sinfo->si_copybuf_req, max_copybuf); 2905 ASSERT((dma->dp_copybuf_size & MMU_PAGEOFFSET) == 0); 2906 2907 #if !defined(__amd64) 2908 /* 2909 * if we don't have kva space to copy to/from, allocate the KVA space 2910 * now. We only do this for the 32-bit kernel. We use seg kpm space for 2911 * the 64-bit kernel. 2912 */ 2913 if ((dmareq->dmar_object.dmao_type == DMA_OTYP_PAGES) || 2914 (dmareq->dmar_object.dmao_obj.virt_obj.v_as != NULL)) { 2915 2916 /* convert the sleep flags */ 2917 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2918 vmflag = VM_SLEEP; 2919 } else { 2920 vmflag = VM_NOSLEEP; 2921 } 2922 2923 /* allocate Kernel VA space that we can bcopy to/from */ 2924 dma->dp_kva = vmem_alloc(heap_arena, dma->dp_copybuf_size, 2925 vmflag); 2926 if (dma->dp_kva == NULL) { 2927 return (DDI_DMA_NORESOURCES); 2928 } 2929 } 2930 #endif 2931 2932 /* convert the sleep flags */ 2933 if (dmareq->dmar_fp == DDI_DMA_SLEEP) { 2934 cansleep = 1; 2935 } else { 2936 cansleep = 0; 2937 } 2938 2939 /* 2940 * Allocate the actual copy buffer. This needs to fit within the DMA 2941 * engine limits, so we can't use kmem_alloc... We don't need 2942 * contiguous memory (sgllen) since we will be forcing windows on 2943 * sgllen anyway. 2944 */ 2945 lattr = *attr; 2946 lattr.dma_attr_align = MMU_PAGESIZE; 2947 /* 2948 * this should be < 0 to indicate no limit, but due to a bug in 2949 * the rootnex, we'll set it to the maximum positive int. 2950 */ 2951 lattr.dma_attr_sgllen = 0x7fffffff; 2952 e = i_ddi_mem_alloc(dma->dp_dip, &lattr, dma->dp_copybuf_size, cansleep, 2953 0, NULL, &dma->dp_cbaddr, &dma->dp_cbsize, NULL); 2954 if (e != DDI_SUCCESS) { 2955 #if !defined(__amd64) 2956 if (dma->dp_kva != NULL) { 2957 vmem_free(heap_arena, dma->dp_kva, 2958 dma->dp_copybuf_size); 2959 } 2960 #endif 2961 return (DDI_DMA_NORESOURCES); 2962 } 2963 2964 DTRACE_PROBE2(rootnex__alloc__copybuf, dev_info_t *, dma->dp_dip, 2965 size_t, dma->dp_copybuf_size); 2966 2967 return (DDI_SUCCESS); 2968 } 2969 2970 2971 /* 2972 * rootnex_setup_windows() 2973 * Called in bind slowpath to setup the window state. We always have windows 2974 * in the slowpath. Even if the window count = 1. 2975 */ 2976 static int 2977 rootnex_setup_windows(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 2978 ddi_dma_attr_t *attr, int kmflag) 2979 { 2980 rootnex_window_t *windowp; 2981 rootnex_sglinfo_t *sinfo; 2982 size_t copy_state_size; 2983 size_t win_state_size; 2984 size_t state_available; 2985 size_t space_needed; 2986 uint_t copybuf_win; 2987 uint_t maxxfer_win; 2988 size_t space_used; 2989 uint_t sglwin; 2990 2991 2992 sinfo = &dma->dp_sglinfo; 2993 2994 dma->dp_current_win = 0; 2995 hp->dmai_nwin = 0; 2996 2997 /* If we don't need to do a partial, we only have one window */ 2998 if (!dma->dp_partial_required) { 2999 dma->dp_max_win = 1; 3000 3001 /* 3002 * we need multiple windows, need to figure out the worse case number 3003 * of windows. 3004 */ 3005 } else { 3006 /* 3007 * if we need windows because we need more copy buffer that 3008 * we allow, the worse case number of windows we could need 3009 * here would be (copybuf space required / copybuf space that 3010 * we have) plus one for remainder, and plus 2 to handle the 3011 * extra pages on the trim for the first and last pages of the 3012 * buffer (a page is the minimum window size so under the right 3013 * attr settings, you could have a window for each page). 3014 * The last page will only be hit here if the size is not a 3015 * multiple of the granularity (which theoretically shouldn't 3016 * be the case but never has been enforced, so we could have 3017 * broken things without it). 3018 */ 3019 if (sinfo->si_copybuf_req > dma->dp_copybuf_size) { 3020 ASSERT(dma->dp_copybuf_size > 0); 3021 copybuf_win = (sinfo->si_copybuf_req / 3022 dma->dp_copybuf_size) + 1 + 2; 3023 } else { 3024 copybuf_win = 0; 3025 } 3026 3027 /* 3028 * if we need windows because we have more cookies than the H/W 3029 * can handle, the number of windows we would need here would 3030 * be (cookie count / cookies count H/W supports) plus one for 3031 * remainder, and plus 2 to handle the extra pages on the trim 3032 * (see above comment about trim) 3033 */ 3034 if (attr->dma_attr_sgllen < sinfo->si_sgl_size) { 3035 sglwin = ((sinfo->si_sgl_size / attr->dma_attr_sgllen) 3036 + 1) + 2; 3037 } else { 3038 sglwin = 0; 3039 } 3040 3041 /* 3042 * if we need windows because we're binding more memory than the 3043 * H/W can transfer at once, the number of windows we would need 3044 * here would be (xfer count / max xfer H/W supports) plus one 3045 * for remainder, and plus 2 to handle the extra pages on the 3046 * trim (see above comment about trim) 3047 */ 3048 if (dma->dp_dma.dmao_size > dma->dp_maxxfer) { 3049 maxxfer_win = (dma->dp_dma.dmao_size / 3050 dma->dp_maxxfer) + 1 + 2; 3051 } else { 3052 maxxfer_win = 0; 3053 } 3054 dma->dp_max_win = copybuf_win + sglwin + maxxfer_win; 3055 ASSERT(dma->dp_max_win > 0); 3056 } 3057 win_state_size = dma->dp_max_win * sizeof (rootnex_window_t); 3058 3059 /* 3060 * Get space for window and potential copy buffer state. Before we 3061 * go and allocate memory, see if we can get away with using what's 3062 * left in the pre-allocted state or the dynamically allocated sgl. 3063 */ 3064 space_used = (uintptr_t)(sinfo->si_sgl_size * 3065 sizeof (ddi_dma_cookie_t)); 3066 3067 /* if we dynamically allocated space for the cookies */ 3068 if (dma->dp_need_to_free_cookie) { 3069 /* if we have more space in the pre-allocted buffer, use it */ 3070 ASSERT(space_used <= dma->dp_cookie_size); 3071 if ((dma->dp_cookie_size - space_used) <= 3072 rootnex_state->r_prealloc_size) { 3073 state_available = rootnex_state->r_prealloc_size; 3074 windowp = (rootnex_window_t *)dma->dp_prealloc_buffer; 3075 3076 /* 3077 * else, we have more free space in the dynamically allocated 3078 * buffer, i.e. the buffer wasn't worse case fragmented so we 3079 * didn't need a lot of cookies. 3080 */ 3081 } else { 3082 state_available = dma->dp_cookie_size - space_used; 3083 windowp = (rootnex_window_t *) 3084 &dma->dp_cookies[sinfo->si_sgl_size]; 3085 } 3086 3087 /* we used the pre-alloced buffer */ 3088 } else { 3089 ASSERT(space_used <= rootnex_state->r_prealloc_size); 3090 state_available = rootnex_state->r_prealloc_size - space_used; 3091 windowp = (rootnex_window_t *) 3092 &dma->dp_cookies[sinfo->si_sgl_size]; 3093 } 3094 3095 /* 3096 * figure out how much state we need to track the copy buffer. Add an 3097 * addition 8 bytes for pointer alignemnt later. 3098 */ 3099 if (dma->dp_copybuf_size > 0) { 3100 copy_state_size = sinfo->si_max_pages * 3101 sizeof (rootnex_pgmap_t); 3102 } else { 3103 copy_state_size = 0; 3104 } 3105 /* add an additional 8 bytes for pointer alignment */ 3106 space_needed = win_state_size + copy_state_size + 0x8; 3107 3108 /* if we have enough space already, use it */ 3109 if (state_available >= space_needed) { 3110 dma->dp_window = windowp; 3111 dma->dp_need_to_free_window = B_FALSE; 3112 3113 /* not enough space, need to allocate more. */ 3114 } else { 3115 dma->dp_window = kmem_alloc(space_needed, kmflag); 3116 if (dma->dp_window == NULL) { 3117 return (DDI_DMA_NORESOURCES); 3118 } 3119 dma->dp_need_to_free_window = B_TRUE; 3120 dma->dp_window_size = space_needed; 3121 DTRACE_PROBE2(rootnex__bind__sp__alloc, dev_info_t *, 3122 dma->dp_dip, size_t, space_needed); 3123 } 3124 3125 /* 3126 * we allocate copy buffer state and window state at the same time. 3127 * setup our copy buffer state pointers. Make sure it's aligned. 3128 */ 3129 if (dma->dp_copybuf_size > 0) { 3130 dma->dp_pgmap = (rootnex_pgmap_t *)(((uintptr_t) 3131 &dma->dp_window[dma->dp_max_win] + 0x7) & ~0x7); 3132 3133 #if !defined(__amd64) 3134 /* 3135 * make sure all pm_mapped, pm_vaddr, and pm_pp are set to 3136 * false/NULL. Should be quicker to bzero vs loop and set. 3137 */ 3138 bzero(dma->dp_pgmap, copy_state_size); 3139 #endif 3140 } else { 3141 dma->dp_pgmap = NULL; 3142 } 3143 3144 return (DDI_SUCCESS); 3145 } 3146 3147 3148 /* 3149 * rootnex_teardown_copybuf() 3150 * cleans up after rootnex_setup_copybuf() 3151 */ 3152 static void 3153 rootnex_teardown_copybuf(rootnex_dma_t *dma) 3154 { 3155 #if !defined(__amd64) 3156 int i; 3157 3158 /* 3159 * if we allocated kernel heap VMEM space, go through all the pages and 3160 * map out any of the ones that we're mapped into the kernel heap VMEM 3161 * arena. Then free the VMEM space. 3162 */ 3163 if (dma->dp_kva != NULL) { 3164 for (i = 0; i < dma->dp_sglinfo.si_max_pages; i++) { 3165 if (dma->dp_pgmap[i].pm_mapped) { 3166 hat_unload(kas.a_hat, dma->dp_pgmap[i].pm_kaddr, 3167 MMU_PAGESIZE, HAT_UNLOAD); 3168 dma->dp_pgmap[i].pm_mapped = B_FALSE; 3169 } 3170 } 3171 3172 vmem_free(heap_arena, dma->dp_kva, dma->dp_copybuf_size); 3173 } 3174 3175 #endif 3176 3177 /* if we allocated a copy buffer, free it */ 3178 if (dma->dp_cbaddr != NULL) { 3179 i_ddi_mem_free(dma->dp_cbaddr, NULL); 3180 } 3181 } 3182 3183 3184 /* 3185 * rootnex_teardown_windows() 3186 * cleans up after rootnex_setup_windows() 3187 */ 3188 static void 3189 rootnex_teardown_windows(rootnex_dma_t *dma) 3190 { 3191 /* 3192 * if we had to allocate window state on the last bind (because we 3193 * didn't have enough pre-allocated space in the handle), free it. 3194 */ 3195 if (dma->dp_need_to_free_window) { 3196 kmem_free(dma->dp_window, dma->dp_window_size); 3197 } 3198 } 3199 3200 3201 /* 3202 * rootnex_init_win() 3203 * Called in bind slow path during creation of a new window. Initializes 3204 * window state to default values. 3205 */ 3206 /*ARGSUSED*/ 3207 static void 3208 rootnex_init_win(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3209 rootnex_window_t *window, ddi_dma_cookie_t *cookie, off_t cur_offset) 3210 { 3211 hp->dmai_nwin++; 3212 window->wd_dosync = B_FALSE; 3213 window->wd_offset = cur_offset; 3214 window->wd_size = 0; 3215 window->wd_first_cookie = cookie; 3216 window->wd_cookie_cnt = 0; 3217 window->wd_trim.tr_trim_first = B_FALSE; 3218 window->wd_trim.tr_trim_last = B_FALSE; 3219 window->wd_trim.tr_first_copybuf_win = B_FALSE; 3220 window->wd_trim.tr_last_copybuf_win = B_FALSE; 3221 #if !defined(__amd64) 3222 window->wd_remap_copybuf = dma->dp_cb_remaping; 3223 #endif 3224 } 3225 3226 3227 /* 3228 * rootnex_setup_cookie() 3229 * Called in the bind slow path when the sgl uses the copy buffer. If any of 3230 * the sgl uses the copy buffer, we need to go through each cookie, figure 3231 * out if it uses the copy buffer, and if it does, save away everything we'll 3232 * need during sync. 3233 */ 3234 static void 3235 rootnex_setup_cookie(ddi_dma_obj_t *dmar_object, rootnex_dma_t *dma, 3236 ddi_dma_cookie_t *cookie, off_t cur_offset, size_t *copybuf_used, 3237 page_t **cur_pp) 3238 { 3239 boolean_t copybuf_sz_power_2; 3240 rootnex_sglinfo_t *sinfo; 3241 paddr_t paddr; 3242 uint_t pidx; 3243 uint_t pcnt; 3244 off_t poff; 3245 #if defined(__amd64) 3246 pfn_t pfn; 3247 #else 3248 page_t **pplist; 3249 #endif 3250 3251 sinfo = &dma->dp_sglinfo; 3252 3253 /* 3254 * Calculate the page index relative to the start of the buffer. The 3255 * index to the current page for our buffer is the offset into the 3256 * first page of the buffer plus our current offset into the buffer 3257 * itself, shifted of course... 3258 */ 3259 pidx = (sinfo->si_buf_offset + cur_offset) >> MMU_PAGESHIFT; 3260 ASSERT(pidx < sinfo->si_max_pages); 3261 3262 /* if this cookie uses the copy buffer */ 3263 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3264 /* 3265 * NOTE: we know that since this cookie uses the copy buffer, it 3266 * is <= MMU_PAGESIZE. 3267 */ 3268 3269 /* 3270 * get the offset into the page. For the 64-bit kernel, get the 3271 * pfn which we'll use with seg kpm. 3272 */ 3273 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3274 #if defined(__amd64) 3275 /* mfn_to_pfn() is a NOP on i86pc */ 3276 pfn = mfn_to_pfn(cookie->dmac_laddress >> MMU_PAGESHIFT); 3277 #endif /* __amd64 */ 3278 3279 /* figure out if the copybuf size is a power of 2 */ 3280 if (dma->dp_copybuf_size & (dma->dp_copybuf_size - 1)) { 3281 copybuf_sz_power_2 = B_FALSE; 3282 } else { 3283 copybuf_sz_power_2 = B_TRUE; 3284 } 3285 3286 /* This page uses the copy buffer */ 3287 dma->dp_pgmap[pidx].pm_uses_copybuf = B_TRUE; 3288 3289 /* 3290 * save the copy buffer KVA that we'll use with this page. 3291 * if we still fit within the copybuf, it's a simple add. 3292 * otherwise, we need to wrap over using & or % accordingly. 3293 */ 3294 if ((*copybuf_used + MMU_PAGESIZE) <= dma->dp_copybuf_size) { 3295 dma->dp_pgmap[pidx].pm_cbaddr = dma->dp_cbaddr + 3296 *copybuf_used; 3297 } else { 3298 if (copybuf_sz_power_2) { 3299 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3300 (uintptr_t)dma->dp_cbaddr + 3301 (*copybuf_used & 3302 (dma->dp_copybuf_size - 1))); 3303 } else { 3304 dma->dp_pgmap[pidx].pm_cbaddr = (caddr_t)( 3305 (uintptr_t)dma->dp_cbaddr + 3306 (*copybuf_used % dma->dp_copybuf_size)); 3307 } 3308 } 3309 3310 /* 3311 * over write the cookie physical address with the address of 3312 * the physical address of the copy buffer page that we will 3313 * use. 3314 */ 3315 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3316 dma->dp_pgmap[pidx].pm_cbaddr)) + poff; 3317 3318 #ifdef __xpv 3319 /* 3320 * If we're dom0, we're using a real device so we need to load 3321 * the cookies with MAs instead of PAs. 3322 */ 3323 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3324 #else 3325 cookie->dmac_laddress = paddr; 3326 #endif 3327 3328 /* if we have a kernel VA, it's easy, just save that address */ 3329 if ((dmar_object->dmao_type != DMA_OTYP_PAGES) && 3330 (sinfo->si_asp == &kas)) { 3331 /* 3332 * save away the page aligned virtual address of the 3333 * driver buffer. Offsets are handled in the sync code. 3334 */ 3335 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t)(((uintptr_t) 3336 dmar_object->dmao_obj.virt_obj.v_addr + cur_offset) 3337 & MMU_PAGEMASK); 3338 #if !defined(__amd64) 3339 /* 3340 * we didn't need to, and will never need to map this 3341 * page. 3342 */ 3343 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3344 #endif 3345 3346 /* we don't have a kernel VA. We need one for the bcopy. */ 3347 } else { 3348 #if defined(__amd64) 3349 /* 3350 * for the 64-bit kernel, it's easy. We use seg kpm to 3351 * get a Kernel VA for the corresponding pfn. 3352 */ 3353 dma->dp_pgmap[pidx].pm_kaddr = hat_kpm_pfn2va(pfn); 3354 #else 3355 /* 3356 * for the 32-bit kernel, this is a pain. First we'll 3357 * save away the page_t or user VA for this page. This 3358 * is needed in rootnex_dma_win() when we switch to a 3359 * new window which requires us to re-map the copy 3360 * buffer. 3361 */ 3362 pplist = dmar_object->dmao_obj.virt_obj.v_priv; 3363 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3364 dma->dp_pgmap[pidx].pm_pp = *cur_pp; 3365 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3366 } else if (pplist != NULL) { 3367 dma->dp_pgmap[pidx].pm_pp = pplist[pidx]; 3368 dma->dp_pgmap[pidx].pm_vaddr = NULL; 3369 } else { 3370 dma->dp_pgmap[pidx].pm_pp = NULL; 3371 dma->dp_pgmap[pidx].pm_vaddr = (caddr_t) 3372 (((uintptr_t) 3373 dmar_object->dmao_obj.virt_obj.v_addr + 3374 cur_offset) & MMU_PAGEMASK); 3375 } 3376 3377 /* 3378 * save away the page aligned virtual address which was 3379 * allocated from the kernel heap arena (taking into 3380 * account if we need more copy buffer than we alloced 3381 * and use multiple windows to handle this, i.e. &,%). 3382 * NOTE: there isn't and physical memory backing up this 3383 * virtual address space currently. 3384 */ 3385 if ((*copybuf_used + MMU_PAGESIZE) <= 3386 dma->dp_copybuf_size) { 3387 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3388 (((uintptr_t)dma->dp_kva + *copybuf_used) & 3389 MMU_PAGEMASK); 3390 } else { 3391 if (copybuf_sz_power_2) { 3392 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3393 (((uintptr_t)dma->dp_kva + 3394 (*copybuf_used & 3395 (dma->dp_copybuf_size - 1))) & 3396 MMU_PAGEMASK); 3397 } else { 3398 dma->dp_pgmap[pidx].pm_kaddr = (caddr_t) 3399 (((uintptr_t)dma->dp_kva + 3400 (*copybuf_used % 3401 dma->dp_copybuf_size)) & 3402 MMU_PAGEMASK); 3403 } 3404 } 3405 3406 /* 3407 * if we haven't used up the available copy buffer yet, 3408 * map the kva to the physical page. 3409 */ 3410 if (!dma->dp_cb_remaping && ((*copybuf_used + 3411 MMU_PAGESIZE) <= dma->dp_copybuf_size)) { 3412 dma->dp_pgmap[pidx].pm_mapped = B_TRUE; 3413 if (dma->dp_pgmap[pidx].pm_pp != NULL) { 3414 i86_pp_map(dma->dp_pgmap[pidx].pm_pp, 3415 dma->dp_pgmap[pidx].pm_kaddr); 3416 } else { 3417 i86_va_map(dma->dp_pgmap[pidx].pm_vaddr, 3418 sinfo->si_asp, 3419 dma->dp_pgmap[pidx].pm_kaddr); 3420 } 3421 3422 /* 3423 * we've used up the available copy buffer, this page 3424 * will have to be mapped during rootnex_dma_win() when 3425 * we switch to a new window which requires a re-map 3426 * the copy buffer. (32-bit kernel only) 3427 */ 3428 } else { 3429 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3430 } 3431 #endif 3432 /* go to the next page_t */ 3433 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3434 *cur_pp = (*cur_pp)->p_next; 3435 } 3436 } 3437 3438 /* add to the copy buffer count */ 3439 *copybuf_used += MMU_PAGESIZE; 3440 3441 /* 3442 * This cookie doesn't use the copy buffer. Walk through the pages this 3443 * cookie occupies to reflect this. 3444 */ 3445 } else { 3446 /* 3447 * figure out how many pages the cookie occupies. We need to 3448 * use the original page offset of the buffer and the cookies 3449 * offset in the buffer to do this. 3450 */ 3451 poff = (sinfo->si_buf_offset + cur_offset) & MMU_PAGEOFFSET; 3452 pcnt = mmu_btopr(cookie->dmac_size + poff); 3453 3454 while (pcnt > 0) { 3455 #if !defined(__amd64) 3456 /* 3457 * the 32-bit kernel doesn't have seg kpm, so we need 3458 * to map in the driver buffer (if it didn't come down 3459 * with a kernel VA) on the fly. Since this page doesn't 3460 * use the copy buffer, it's not, or will it ever, have 3461 * to be mapped in. 3462 */ 3463 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 3464 #endif 3465 dma->dp_pgmap[pidx].pm_uses_copybuf = B_FALSE; 3466 3467 /* 3468 * we need to update pidx and cur_pp or we'll loose 3469 * track of where we are. 3470 */ 3471 if (dmar_object->dmao_type == DMA_OTYP_PAGES) { 3472 *cur_pp = (*cur_pp)->p_next; 3473 } 3474 pidx++; 3475 pcnt--; 3476 } 3477 } 3478 } 3479 3480 3481 /* 3482 * rootnex_sgllen_window_boundary() 3483 * Called in the bind slow path when the next cookie causes us to exceed (in 3484 * this case == since we start at 0 and sgllen starts at 1) the maximum sgl 3485 * length supported by the DMA H/W. 3486 */ 3487 static int 3488 rootnex_sgllen_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3489 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, ddi_dma_attr_t *attr, 3490 off_t cur_offset) 3491 { 3492 off_t new_offset; 3493 size_t trim_sz; 3494 off_t coffset; 3495 3496 3497 /* 3498 * if we know we'll never have to trim, it's pretty easy. Just move to 3499 * the next window and init it. We're done. 3500 */ 3501 if (!dma->dp_trim_required) { 3502 (*windowp)++; 3503 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3504 (*windowp)->wd_cookie_cnt++; 3505 (*windowp)->wd_size = cookie->dmac_size; 3506 return (DDI_SUCCESS); 3507 } 3508 3509 /* figure out how much we need to trim from the window */ 3510 ASSERT(attr->dma_attr_granular != 0); 3511 if (dma->dp_granularity_power_2) { 3512 trim_sz = (*windowp)->wd_size & (attr->dma_attr_granular - 1); 3513 } else { 3514 trim_sz = (*windowp)->wd_size % attr->dma_attr_granular; 3515 } 3516 3517 /* The window's a whole multiple of granularity. We're done */ 3518 if (trim_sz == 0) { 3519 (*windowp)++; 3520 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3521 (*windowp)->wd_cookie_cnt++; 3522 (*windowp)->wd_size = cookie->dmac_size; 3523 return (DDI_SUCCESS); 3524 } 3525 3526 /* 3527 * The window's not a whole multiple of granularity, since we know this 3528 * is due to the sgllen, we need to go back to the last cookie and trim 3529 * that one, add the left over part of the old cookie into the new 3530 * window, and then add in the new cookie into the new window. 3531 */ 3532 3533 /* 3534 * make sure the driver isn't making us do something bad... Trimming and 3535 * sgllen == 1 don't go together. 3536 */ 3537 if (attr->dma_attr_sgllen == 1) { 3538 return (DDI_DMA_NOMAPPING); 3539 } 3540 3541 /* 3542 * first, setup the current window to account for the trim. Need to go 3543 * back to the last cookie for this. 3544 */ 3545 cookie--; 3546 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3547 (*windowp)->wd_trim.tr_last_cookie = cookie; 3548 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3549 ASSERT(cookie->dmac_size > trim_sz); 3550 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3551 (*windowp)->wd_size -= trim_sz; 3552 3553 /* save the buffer offsets for the next window */ 3554 coffset = cookie->dmac_size - trim_sz; 3555 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3556 3557 /* 3558 * set this now in case this is the first window. all other cases are 3559 * set in dma_win() 3560 */ 3561 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3562 3563 /* 3564 * initialize the next window using what's left over in the previous 3565 * cookie. 3566 */ 3567 (*windowp)++; 3568 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3569 (*windowp)->wd_cookie_cnt++; 3570 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3571 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3572 (*windowp)->wd_trim.tr_first_size = trim_sz; 3573 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3574 (*windowp)->wd_dosync = B_TRUE; 3575 } 3576 3577 /* 3578 * now go back to the current cookie and add it to the new window. set 3579 * the new window size to the what was left over from the previous 3580 * cookie and what's in the current cookie. 3581 */ 3582 cookie++; 3583 (*windowp)->wd_cookie_cnt++; 3584 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3585 3586 /* 3587 * trim plus the next cookie could put us over maxxfer (a cookie can be 3588 * a max size of maxxfer). Handle that case. 3589 */ 3590 if ((*windowp)->wd_size > dma->dp_maxxfer) { 3591 /* 3592 * maxxfer is already a whole multiple of granularity, and this 3593 * trim will be <= the previous trim (since a cookie can't be 3594 * larger than maxxfer). Make things simple here. 3595 */ 3596 trim_sz = (*windowp)->wd_size - dma->dp_maxxfer; 3597 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3598 (*windowp)->wd_trim.tr_last_cookie = cookie; 3599 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3600 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3601 (*windowp)->wd_size -= trim_sz; 3602 ASSERT((*windowp)->wd_size == dma->dp_maxxfer); 3603 3604 /* save the buffer offsets for the next window */ 3605 coffset = cookie->dmac_size - trim_sz; 3606 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3607 3608 /* setup the next window */ 3609 (*windowp)++; 3610 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3611 (*windowp)->wd_cookie_cnt++; 3612 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3613 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3614 coffset; 3615 (*windowp)->wd_trim.tr_first_size = trim_sz; 3616 } 3617 3618 return (DDI_SUCCESS); 3619 } 3620 3621 3622 /* 3623 * rootnex_copybuf_window_boundary() 3624 * Called in bind slowpath when we get to a window boundary because we used 3625 * up all the copy buffer that we have. 3626 */ 3627 static int 3628 rootnex_copybuf_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3629 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie, off_t cur_offset, 3630 size_t *copybuf_used) 3631 { 3632 rootnex_sglinfo_t *sinfo; 3633 off_t new_offset; 3634 size_t trim_sz; 3635 paddr_t paddr; 3636 off_t coffset; 3637 uint_t pidx; 3638 off_t poff; 3639 3640 3641 sinfo = &dma->dp_sglinfo; 3642 3643 /* 3644 * the copy buffer should be a whole multiple of page size. We know that 3645 * this cookie is <= MMU_PAGESIZE. 3646 */ 3647 ASSERT(cookie->dmac_size <= MMU_PAGESIZE); 3648 3649 /* 3650 * from now on, all new windows in this bind need to be re-mapped during 3651 * ddi_dma_getwin() (32-bit kernel only). i.e. we ran out out copybuf 3652 * space... 3653 */ 3654 #if !defined(__amd64) 3655 dma->dp_cb_remaping = B_TRUE; 3656 #endif 3657 3658 /* reset copybuf used */ 3659 *copybuf_used = 0; 3660 3661 /* 3662 * if we don't have to trim (since granularity is set to 1), go to the 3663 * next window and add the current cookie to it. We know the current 3664 * cookie uses the copy buffer since we're in this code path. 3665 */ 3666 if (!dma->dp_trim_required) { 3667 (*windowp)++; 3668 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3669 3670 /* Add this cookie to the new window */ 3671 (*windowp)->wd_cookie_cnt++; 3672 (*windowp)->wd_size += cookie->dmac_size; 3673 *copybuf_used += MMU_PAGESIZE; 3674 return (DDI_SUCCESS); 3675 } 3676 3677 /* 3678 * *** may need to trim, figure it out. 3679 */ 3680 3681 /* figure out how much we need to trim from the window */ 3682 if (dma->dp_granularity_power_2) { 3683 trim_sz = (*windowp)->wd_size & 3684 (hp->dmai_attr.dma_attr_granular - 1); 3685 } else { 3686 trim_sz = (*windowp)->wd_size % hp->dmai_attr.dma_attr_granular; 3687 } 3688 3689 /* 3690 * if the window's a whole multiple of granularity, go to the next 3691 * window, init it, then add in the current cookie. We know the current 3692 * cookie uses the copy buffer since we're in this code path. 3693 */ 3694 if (trim_sz == 0) { 3695 (*windowp)++; 3696 rootnex_init_win(hp, dma, *windowp, cookie, cur_offset); 3697 3698 /* Add this cookie to the new window */ 3699 (*windowp)->wd_cookie_cnt++; 3700 (*windowp)->wd_size += cookie->dmac_size; 3701 *copybuf_used += MMU_PAGESIZE; 3702 return (DDI_SUCCESS); 3703 } 3704 3705 /* 3706 * *** We figured it out, we definitly need to trim 3707 */ 3708 3709 /* 3710 * make sure the driver isn't making us do something bad... 3711 * Trimming and sgllen == 1 don't go together. 3712 */ 3713 if (hp->dmai_attr.dma_attr_sgllen == 1) { 3714 return (DDI_DMA_NOMAPPING); 3715 } 3716 3717 /* 3718 * first, setup the current window to account for the trim. Need to go 3719 * back to the last cookie for this. Some of the last cookie will be in 3720 * the current window, and some of the last cookie will be in the new 3721 * window. All of the current cookie will be in the new window. 3722 */ 3723 cookie--; 3724 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3725 (*windowp)->wd_trim.tr_last_cookie = cookie; 3726 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3727 ASSERT(cookie->dmac_size > trim_sz); 3728 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3729 (*windowp)->wd_size -= trim_sz; 3730 3731 /* 3732 * we're trimming the last cookie (not the current cookie). So that 3733 * last cookie may have or may not have been using the copy buffer ( 3734 * we know the cookie passed in uses the copy buffer since we're in 3735 * this code path). 3736 * 3737 * If the last cookie doesn't use the copy buffer, nothing special to 3738 * do. However, if it does uses the copy buffer, it will be both the 3739 * last page in the current window and the first page in the next 3740 * window. Since we are reusing the copy buffer (and KVA space on the 3741 * 32-bit kernel), this page will use the end of the copy buffer in the 3742 * current window, and the start of the copy buffer in the next window. 3743 * Track that info... The cookie physical address was already set to 3744 * the copy buffer physical address in setup_cookie.. 3745 */ 3746 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3747 pidx = (sinfo->si_buf_offset + (*windowp)->wd_offset + 3748 (*windowp)->wd_size) >> MMU_PAGESHIFT; 3749 (*windowp)->wd_trim.tr_last_copybuf_win = B_TRUE; 3750 (*windowp)->wd_trim.tr_last_pidx = pidx; 3751 (*windowp)->wd_trim.tr_last_cbaddr = 3752 dma->dp_pgmap[pidx].pm_cbaddr; 3753 #if !defined(__amd64) 3754 (*windowp)->wd_trim.tr_last_kaddr = 3755 dma->dp_pgmap[pidx].pm_kaddr; 3756 #endif 3757 } 3758 3759 /* save the buffer offsets for the next window */ 3760 coffset = cookie->dmac_size - trim_sz; 3761 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3762 3763 /* 3764 * set this now in case this is the first window. all other cases are 3765 * set in dma_win() 3766 */ 3767 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3768 3769 /* 3770 * initialize the next window using what's left over in the previous 3771 * cookie. 3772 */ 3773 (*windowp)++; 3774 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3775 (*windowp)->wd_cookie_cnt++; 3776 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3777 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + coffset; 3778 (*windowp)->wd_trim.tr_first_size = trim_sz; 3779 3780 /* 3781 * again, we're tracking if the last cookie uses the copy buffer. 3782 * read the comment above for more info on why we need to track 3783 * additional state. 3784 * 3785 * For the first cookie in the new window, we need reset the physical 3786 * address to DMA into to the start of the copy buffer plus any 3787 * initial page offset which may be present. 3788 */ 3789 if (cookie->dmac_type & ROOTNEX_USES_COPYBUF) { 3790 (*windowp)->wd_dosync = B_TRUE; 3791 (*windowp)->wd_trim.tr_first_copybuf_win = B_TRUE; 3792 (*windowp)->wd_trim.tr_first_pidx = pidx; 3793 (*windowp)->wd_trim.tr_first_cbaddr = dma->dp_cbaddr; 3794 poff = (*windowp)->wd_trim.tr_first_paddr & MMU_PAGEOFFSET; 3795 3796 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, dma->dp_cbaddr)) + 3797 poff; 3798 #ifdef __xpv 3799 /* 3800 * If we're dom0, we're using a real device so we need to load 3801 * the cookies with MAs instead of PAs. 3802 */ 3803 (*windowp)->wd_trim.tr_first_paddr = 3804 ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3805 #else 3806 (*windowp)->wd_trim.tr_first_paddr = paddr; 3807 #endif 3808 3809 #if !defined(__amd64) 3810 (*windowp)->wd_trim.tr_first_kaddr = dma->dp_kva; 3811 #endif 3812 /* account for the cookie copybuf usage in the new window */ 3813 *copybuf_used += MMU_PAGESIZE; 3814 3815 /* 3816 * every piece of code has to have a hack, and here is this 3817 * ones :-) 3818 * 3819 * There is a complex interaction between setup_cookie and the 3820 * copybuf window boundary. The complexity had to be in either 3821 * the maxxfer window, or the copybuf window, and I chose the 3822 * copybuf code. 3823 * 3824 * So in this code path, we have taken the last cookie, 3825 * virtually broken it in half due to the trim, and it happens 3826 * to use the copybuf which further complicates life. At the 3827 * same time, we have already setup the current cookie, which 3828 * is now wrong. More background info: the current cookie uses 3829 * the copybuf, so it is only a page long max. So we need to 3830 * fix the current cookies copy buffer address, physical 3831 * address, and kva for the 32-bit kernel. We due this by 3832 * bumping them by page size (of course, we can't due this on 3833 * the physical address since the copy buffer may not be 3834 * physically contiguous). 3835 */ 3836 cookie++; 3837 dma->dp_pgmap[pidx + 1].pm_cbaddr += MMU_PAGESIZE; 3838 poff = cookie->dmac_laddress & MMU_PAGEOFFSET; 3839 3840 paddr = pfn_to_pa(hat_getpfnum(kas.a_hat, 3841 dma->dp_pgmap[pidx + 1].pm_cbaddr)) + poff; 3842 #ifdef __xpv 3843 /* 3844 * If we're dom0, we're using a real device so we need to load 3845 * the cookies with MAs instead of PAs. 3846 */ 3847 cookie->dmac_laddress = ROOTNEX_PADDR_TO_RBASE(xen_info, paddr); 3848 #else 3849 cookie->dmac_laddress = paddr; 3850 #endif 3851 3852 #if !defined(__amd64) 3853 ASSERT(dma->dp_pgmap[pidx + 1].pm_mapped == B_FALSE); 3854 dma->dp_pgmap[pidx + 1].pm_kaddr += MMU_PAGESIZE; 3855 #endif 3856 } else { 3857 /* go back to the current cookie */ 3858 cookie++; 3859 } 3860 3861 /* 3862 * add the current cookie to the new window. set the new window size to 3863 * the what was left over from the previous cookie and what's in the 3864 * current cookie. 3865 */ 3866 (*windowp)->wd_cookie_cnt++; 3867 (*windowp)->wd_size = trim_sz + cookie->dmac_size; 3868 ASSERT((*windowp)->wd_size < dma->dp_maxxfer); 3869 3870 /* 3871 * we know that the cookie passed in always uses the copy buffer. We 3872 * wouldn't be here if it didn't. 3873 */ 3874 *copybuf_used += MMU_PAGESIZE; 3875 3876 return (DDI_SUCCESS); 3877 } 3878 3879 3880 /* 3881 * rootnex_maxxfer_window_boundary() 3882 * Called in bind slowpath when we get to a window boundary because we will 3883 * go over maxxfer. 3884 */ 3885 static int 3886 rootnex_maxxfer_window_boundary(ddi_dma_impl_t *hp, rootnex_dma_t *dma, 3887 rootnex_window_t **windowp, ddi_dma_cookie_t *cookie) 3888 { 3889 size_t dmac_size; 3890 off_t new_offset; 3891 size_t trim_sz; 3892 off_t coffset; 3893 3894 3895 /* 3896 * calculate how much we have to trim off of the current cookie to equal 3897 * maxxfer. We don't have to account for granularity here since our 3898 * maxxfer already takes that into account. 3899 */ 3900 trim_sz = ((*windowp)->wd_size + cookie->dmac_size) - dma->dp_maxxfer; 3901 ASSERT(trim_sz <= cookie->dmac_size); 3902 ASSERT(trim_sz <= dma->dp_maxxfer); 3903 3904 /* save cookie size since we need it later and we might change it */ 3905 dmac_size = cookie->dmac_size; 3906 3907 /* 3908 * if we're not trimming the entire cookie, setup the current window to 3909 * account for the trim. 3910 */ 3911 if (trim_sz < cookie->dmac_size) { 3912 (*windowp)->wd_cookie_cnt++; 3913 (*windowp)->wd_trim.tr_trim_last = B_TRUE; 3914 (*windowp)->wd_trim.tr_last_cookie = cookie; 3915 (*windowp)->wd_trim.tr_last_paddr = cookie->dmac_laddress; 3916 (*windowp)->wd_trim.tr_last_size = cookie->dmac_size - trim_sz; 3917 (*windowp)->wd_size = dma->dp_maxxfer; 3918 3919 /* 3920 * set the adjusted cookie size now in case this is the first 3921 * window. All other windows are taken care of in get win 3922 */ 3923 cookie->dmac_size = (*windowp)->wd_trim.tr_last_size; 3924 } 3925 3926 /* 3927 * coffset is the current offset within the cookie, new_offset is the 3928 * current offset with the entire buffer. 3929 */ 3930 coffset = dmac_size - trim_sz; 3931 new_offset = (*windowp)->wd_offset + (*windowp)->wd_size; 3932 3933 /* initialize the next window */ 3934 (*windowp)++; 3935 rootnex_init_win(hp, dma, *windowp, cookie, new_offset); 3936 (*windowp)->wd_cookie_cnt++; 3937 (*windowp)->wd_size = trim_sz; 3938 if (trim_sz < dmac_size) { 3939 (*windowp)->wd_trim.tr_trim_first = B_TRUE; 3940 (*windowp)->wd_trim.tr_first_paddr = cookie->dmac_laddress + 3941 coffset; 3942 (*windowp)->wd_trim.tr_first_size = trim_sz; 3943 } 3944 3945 return (DDI_SUCCESS); 3946 } 3947 3948 3949 /*ARGSUSED*/ 3950 static int 3951 rootnex_coredma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 3952 off_t off, size_t len, uint_t cache_flags) 3953 { 3954 rootnex_sglinfo_t *sinfo; 3955 rootnex_pgmap_t *cbpage; 3956 rootnex_window_t *win; 3957 ddi_dma_impl_t *hp; 3958 rootnex_dma_t *dma; 3959 caddr_t fromaddr; 3960 caddr_t toaddr; 3961 uint_t psize; 3962 off_t offset; 3963 uint_t pidx; 3964 size_t size; 3965 off_t poff; 3966 int e; 3967 3968 3969 hp = (ddi_dma_impl_t *)handle; 3970 dma = (rootnex_dma_t *)hp->dmai_private; 3971 sinfo = &dma->dp_sglinfo; 3972 3973 /* 3974 * if we don't have any windows, we don't need to sync. A copybuf 3975 * will cause us to have at least one window. 3976 */ 3977 if (dma->dp_window == NULL) { 3978 return (DDI_SUCCESS); 3979 } 3980 3981 /* This window may not need to be sync'd */ 3982 win = &dma->dp_window[dma->dp_current_win]; 3983 if (!win->wd_dosync) { 3984 return (DDI_SUCCESS); 3985 } 3986 3987 /* handle off and len special cases */ 3988 if ((off == 0) || (rootnex_sync_ignore_params)) { 3989 offset = win->wd_offset; 3990 } else { 3991 offset = off; 3992 } 3993 if ((len == 0) || (rootnex_sync_ignore_params)) { 3994 size = win->wd_size; 3995 } else { 3996 size = len; 3997 } 3998 3999 /* check the sync args to make sure they make a little sense */ 4000 if (rootnex_sync_check_parms) { 4001 e = rootnex_valid_sync_parms(hp, win, offset, size, 4002 cache_flags); 4003 if (e != DDI_SUCCESS) { 4004 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_SYNC_FAIL]); 4005 return (DDI_FAILURE); 4006 } 4007 } 4008 4009 /* 4010 * special case the first page to handle the offset into the page. The 4011 * offset to the current page for our buffer is the offset into the 4012 * first page of the buffer plus our current offset into the buffer 4013 * itself, masked of course. 4014 */ 4015 poff = (sinfo->si_buf_offset + offset) & MMU_PAGEOFFSET; 4016 psize = MIN((MMU_PAGESIZE - poff), size); 4017 4018 /* go through all the pages that we want to sync */ 4019 while (size > 0) { 4020 /* 4021 * Calculate the page index relative to the start of the buffer. 4022 * The index to the current page for our buffer is the offset 4023 * into the first page of the buffer plus our current offset 4024 * into the buffer itself, shifted of course... 4025 */ 4026 pidx = (sinfo->si_buf_offset + offset) >> MMU_PAGESHIFT; 4027 ASSERT(pidx < sinfo->si_max_pages); 4028 4029 /* 4030 * if this page uses the copy buffer, we need to sync it, 4031 * otherwise, go on to the next page. 4032 */ 4033 cbpage = &dma->dp_pgmap[pidx]; 4034 ASSERT((cbpage->pm_uses_copybuf == B_TRUE) || 4035 (cbpage->pm_uses_copybuf == B_FALSE)); 4036 if (cbpage->pm_uses_copybuf) { 4037 /* cbaddr and kaddr should be page aligned */ 4038 ASSERT(((uintptr_t)cbpage->pm_cbaddr & 4039 MMU_PAGEOFFSET) == 0); 4040 ASSERT(((uintptr_t)cbpage->pm_kaddr & 4041 MMU_PAGEOFFSET) == 0); 4042 4043 /* 4044 * if we're copying for the device, we are going to 4045 * copy from the drivers buffer and to the rootnex 4046 * allocated copy buffer. 4047 */ 4048 if (cache_flags == DDI_DMA_SYNC_FORDEV) { 4049 fromaddr = cbpage->pm_kaddr + poff; 4050 toaddr = cbpage->pm_cbaddr + poff; 4051 DTRACE_PROBE2(rootnex__sync__dev, 4052 dev_info_t *, dma->dp_dip, size_t, psize); 4053 4054 /* 4055 * if we're copying for the cpu/kernel, we are going to 4056 * copy from the rootnex allocated copy buffer to the 4057 * drivers buffer. 4058 */ 4059 } else { 4060 fromaddr = cbpage->pm_cbaddr + poff; 4061 toaddr = cbpage->pm_kaddr + poff; 4062 DTRACE_PROBE2(rootnex__sync__cpu, 4063 dev_info_t *, dma->dp_dip, size_t, psize); 4064 } 4065 4066 bcopy(fromaddr, toaddr, psize); 4067 } 4068 4069 /* 4070 * decrement size until we're done, update our offset into the 4071 * buffer, and get the next page size. 4072 */ 4073 size -= psize; 4074 offset += psize; 4075 psize = MIN(MMU_PAGESIZE, size); 4076 4077 /* page offset is zero for the rest of this loop */ 4078 poff = 0; 4079 } 4080 4081 return (DDI_SUCCESS); 4082 } 4083 4084 /* 4085 * rootnex_dma_sync() 4086 * called from ddi_dma_sync() if DMP_NOSYNC is not set in hp->dmai_rflags. 4087 * We set DMP_NOSYNC if we're not using the copy buffer. If DMP_NOSYNC 4088 * is set, ddi_dma_sync() returns immediately passing back success. 4089 */ 4090 /*ARGSUSED*/ 4091 static int 4092 rootnex_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4093 off_t off, size_t len, uint_t cache_flags) 4094 { 4095 #if !defined(__xpv) 4096 if (IOMMU_USED(rdip)) { 4097 return (iommulib_nexdma_sync(dip, rdip, handle, off, len, 4098 cache_flags)); 4099 } 4100 #endif 4101 return (rootnex_coredma_sync(dip, rdip, handle, off, len, 4102 cache_flags)); 4103 } 4104 4105 /* 4106 * rootnex_valid_sync_parms() 4107 * checks the parameters passed to sync to verify they are correct. 4108 */ 4109 static int 4110 rootnex_valid_sync_parms(ddi_dma_impl_t *hp, rootnex_window_t *win, 4111 off_t offset, size_t size, uint_t cache_flags) 4112 { 4113 off_t woffset; 4114 4115 4116 /* 4117 * the first part of the test to make sure the offset passed in is 4118 * within the window. 4119 */ 4120 if (offset < win->wd_offset) { 4121 return (DDI_FAILURE); 4122 } 4123 4124 /* 4125 * second and last part of the test to make sure the offset and length 4126 * passed in is within the window. 4127 */ 4128 woffset = offset - win->wd_offset; 4129 if ((woffset + size) > win->wd_size) { 4130 return (DDI_FAILURE); 4131 } 4132 4133 /* 4134 * if we are sync'ing for the device, the DDI_DMA_WRITE flag should 4135 * be set too. 4136 */ 4137 if ((cache_flags == DDI_DMA_SYNC_FORDEV) && 4138 (hp->dmai_rflags & DDI_DMA_WRITE)) { 4139 return (DDI_SUCCESS); 4140 } 4141 4142 /* 4143 * at this point, either DDI_DMA_SYNC_FORCPU or DDI_DMA_SYNC_FORKERNEL 4144 * should be set. Also DDI_DMA_READ should be set in the flags. 4145 */ 4146 if (((cache_flags == DDI_DMA_SYNC_FORCPU) || 4147 (cache_flags == DDI_DMA_SYNC_FORKERNEL)) && 4148 (hp->dmai_rflags & DDI_DMA_READ)) { 4149 return (DDI_SUCCESS); 4150 } 4151 4152 return (DDI_FAILURE); 4153 } 4154 4155 4156 /*ARGSUSED*/ 4157 static int 4158 rootnex_coredma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4159 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4160 uint_t *ccountp) 4161 { 4162 rootnex_window_t *window; 4163 rootnex_trim_t *trim; 4164 ddi_dma_impl_t *hp; 4165 rootnex_dma_t *dma; 4166 #if !defined(__amd64) 4167 rootnex_sglinfo_t *sinfo; 4168 rootnex_pgmap_t *pmap; 4169 uint_t pidx; 4170 uint_t pcnt; 4171 off_t poff; 4172 int i; 4173 #endif 4174 4175 4176 hp = (ddi_dma_impl_t *)handle; 4177 dma = (rootnex_dma_t *)hp->dmai_private; 4178 #if !defined(__amd64) 4179 sinfo = &dma->dp_sglinfo; 4180 #endif 4181 4182 /* If we try and get a window which doesn't exist, return failure */ 4183 if (win >= hp->dmai_nwin) { 4184 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4185 return (DDI_FAILURE); 4186 } 4187 4188 /* 4189 * if we don't have any windows, and they're asking for the first 4190 * window, setup the cookie pointer to the first cookie in the bind. 4191 * setup our return values, then increment the cookie since we return 4192 * the first cookie on the stack. 4193 */ 4194 if (dma->dp_window == NULL) { 4195 if (win != 0) { 4196 ROOTNEX_PROF_INC(&rootnex_cnt[ROOTNEX_CNT_GETWIN_FAIL]); 4197 return (DDI_FAILURE); 4198 } 4199 hp->dmai_cookie = dma->dp_cookies; 4200 *offp = 0; 4201 *lenp = dma->dp_dma.dmao_size; 4202 *ccountp = dma->dp_sglinfo.si_sgl_size; 4203 *cookiep = hp->dmai_cookie[0]; 4204 hp->dmai_cookie++; 4205 return (DDI_SUCCESS); 4206 } 4207 4208 /* sync the old window before moving on to the new one */ 4209 window = &dma->dp_window[dma->dp_current_win]; 4210 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_READ)) { 4211 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 4212 DDI_DMA_SYNC_FORCPU); 4213 } 4214 4215 #if !defined(__amd64) 4216 /* 4217 * before we move to the next window, if we need to re-map, unmap all 4218 * the pages in this window. 4219 */ 4220 if (dma->dp_cb_remaping) { 4221 /* 4222 * If we switch to this window again, we'll need to map in 4223 * on the fly next time. 4224 */ 4225 window->wd_remap_copybuf = B_TRUE; 4226 4227 /* 4228 * calculate the page index into the buffer where this window 4229 * starts, and the number of pages this window takes up. 4230 */ 4231 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4232 MMU_PAGESHIFT; 4233 poff = (sinfo->si_buf_offset + window->wd_offset) & 4234 MMU_PAGEOFFSET; 4235 pcnt = mmu_btopr(window->wd_size + poff); 4236 ASSERT((pidx + pcnt) <= sinfo->si_max_pages); 4237 4238 /* unmap pages which are currently mapped in this window */ 4239 for (i = 0; i < pcnt; i++) { 4240 if (dma->dp_pgmap[pidx].pm_mapped) { 4241 hat_unload(kas.a_hat, 4242 dma->dp_pgmap[pidx].pm_kaddr, MMU_PAGESIZE, 4243 HAT_UNLOAD); 4244 dma->dp_pgmap[pidx].pm_mapped = B_FALSE; 4245 } 4246 pidx++; 4247 } 4248 } 4249 #endif 4250 4251 /* 4252 * Move to the new window. 4253 * NOTE: current_win must be set for sync to work right 4254 */ 4255 dma->dp_current_win = win; 4256 window = &dma->dp_window[win]; 4257 4258 /* if needed, adjust the first and/or last cookies for trim */ 4259 trim = &window->wd_trim; 4260 if (trim->tr_trim_first) { 4261 window->wd_first_cookie->dmac_laddress = trim->tr_first_paddr; 4262 window->wd_first_cookie->dmac_size = trim->tr_first_size; 4263 #if !defined(__amd64) 4264 window->wd_first_cookie->dmac_type = 4265 (window->wd_first_cookie->dmac_type & 4266 ROOTNEX_USES_COPYBUF) + window->wd_offset; 4267 #endif 4268 if (trim->tr_first_copybuf_win) { 4269 dma->dp_pgmap[trim->tr_first_pidx].pm_cbaddr = 4270 trim->tr_first_cbaddr; 4271 #if !defined(__amd64) 4272 dma->dp_pgmap[trim->tr_first_pidx].pm_kaddr = 4273 trim->tr_first_kaddr; 4274 #endif 4275 } 4276 } 4277 if (trim->tr_trim_last) { 4278 trim->tr_last_cookie->dmac_laddress = trim->tr_last_paddr; 4279 trim->tr_last_cookie->dmac_size = trim->tr_last_size; 4280 if (trim->tr_last_copybuf_win) { 4281 dma->dp_pgmap[trim->tr_last_pidx].pm_cbaddr = 4282 trim->tr_last_cbaddr; 4283 #if !defined(__amd64) 4284 dma->dp_pgmap[trim->tr_last_pidx].pm_kaddr = 4285 trim->tr_last_kaddr; 4286 #endif 4287 } 4288 } 4289 4290 /* 4291 * setup the cookie pointer to the first cookie in the window. setup 4292 * our return values, then increment the cookie since we return the 4293 * first cookie on the stack. 4294 */ 4295 hp->dmai_cookie = window->wd_first_cookie; 4296 *offp = window->wd_offset; 4297 *lenp = window->wd_size; 4298 *ccountp = window->wd_cookie_cnt; 4299 *cookiep = hp->dmai_cookie[0]; 4300 hp->dmai_cookie++; 4301 4302 #if !defined(__amd64) 4303 /* re-map copybuf if required for this window */ 4304 if (dma->dp_cb_remaping) { 4305 /* 4306 * calculate the page index into the buffer where this 4307 * window starts. 4308 */ 4309 pidx = (sinfo->si_buf_offset + window->wd_offset) >> 4310 MMU_PAGESHIFT; 4311 ASSERT(pidx < sinfo->si_max_pages); 4312 4313 /* 4314 * the first page can get unmapped if it's shared with the 4315 * previous window. Even if the rest of this window is already 4316 * mapped in, we need to still check this one. 4317 */ 4318 pmap = &dma->dp_pgmap[pidx]; 4319 if ((pmap->pm_uses_copybuf) && (pmap->pm_mapped == B_FALSE)) { 4320 if (pmap->pm_pp != NULL) { 4321 pmap->pm_mapped = B_TRUE; 4322 i86_pp_map(pmap->pm_pp, pmap->pm_kaddr); 4323 } else if (pmap->pm_vaddr != NULL) { 4324 pmap->pm_mapped = B_TRUE; 4325 i86_va_map(pmap->pm_vaddr, sinfo->si_asp, 4326 pmap->pm_kaddr); 4327 } 4328 } 4329 pidx++; 4330 4331 /* map in the rest of the pages if required */ 4332 if (window->wd_remap_copybuf) { 4333 window->wd_remap_copybuf = B_FALSE; 4334 4335 /* figure out many pages this window takes up */ 4336 poff = (sinfo->si_buf_offset + window->wd_offset) & 4337 MMU_PAGEOFFSET; 4338 pcnt = mmu_btopr(window->wd_size + poff); 4339 ASSERT(((pidx - 1) + pcnt) <= sinfo->si_max_pages); 4340 4341 /* map pages which require it */ 4342 for (i = 1; i < pcnt; i++) { 4343 pmap = &dma->dp_pgmap[pidx]; 4344 if (pmap->pm_uses_copybuf) { 4345 ASSERT(pmap->pm_mapped == B_FALSE); 4346 if (pmap->pm_pp != NULL) { 4347 pmap->pm_mapped = B_TRUE; 4348 i86_pp_map(pmap->pm_pp, 4349 pmap->pm_kaddr); 4350 } else if (pmap->pm_vaddr != NULL) { 4351 pmap->pm_mapped = B_TRUE; 4352 i86_va_map(pmap->pm_vaddr, 4353 sinfo->si_asp, 4354 pmap->pm_kaddr); 4355 } 4356 } 4357 pidx++; 4358 } 4359 } 4360 } 4361 #endif 4362 4363 /* if the new window uses the copy buffer, sync it for the device */ 4364 if ((window->wd_dosync) && (hp->dmai_rflags & DDI_DMA_WRITE)) { 4365 (void) rootnex_dma_sync(dip, rdip, handle, 0, 0, 4366 DDI_DMA_SYNC_FORDEV); 4367 } 4368 4369 return (DDI_SUCCESS); 4370 } 4371 4372 /* 4373 * rootnex_dma_win() 4374 * called from ddi_dma_getwin() 4375 */ 4376 /*ARGSUSED*/ 4377 static int 4378 rootnex_dma_win(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4379 uint_t win, off_t *offp, size_t *lenp, ddi_dma_cookie_t *cookiep, 4380 uint_t *ccountp) 4381 { 4382 #if !defined(__xpv) 4383 if (IOMMU_USED(rdip)) { 4384 return (iommulib_nexdma_win(dip, rdip, handle, win, offp, lenp, 4385 cookiep, ccountp)); 4386 } 4387 #endif 4388 4389 return (rootnex_coredma_win(dip, rdip, handle, win, offp, lenp, 4390 cookiep, ccountp)); 4391 } 4392 4393 /* 4394 * ************************ 4395 * obsoleted dma routines 4396 * ************************ 4397 */ 4398 4399 /* ARGSUSED */ 4400 static int 4401 rootnex_coredma_map(dev_info_t *dip, dev_info_t *rdip, 4402 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep) 4403 { 4404 #if defined(__amd64) 4405 /* 4406 * this interface is not supported in 64-bit x86 kernel. See comment in 4407 * rootnex_dma_mctl() 4408 */ 4409 return (DDI_DMA_NORESOURCES); 4410 4411 #else /* 32-bit x86 kernel */ 4412 ddi_dma_handle_t *lhandlep; 4413 ddi_dma_handle_t lhandle; 4414 ddi_dma_cookie_t cookie; 4415 ddi_dma_attr_t dma_attr; 4416 ddi_dma_lim_t *dma_lim; 4417 uint_t ccnt; 4418 int e; 4419 4420 4421 /* 4422 * if the driver is just testing to see if it's possible to do the bind, 4423 * we'll use local state. Otherwise, use the handle pointer passed in. 4424 */ 4425 if (handlep == NULL) { 4426 lhandlep = &lhandle; 4427 } else { 4428 lhandlep = handlep; 4429 } 4430 4431 /* convert the limit structure to a dma_attr one */ 4432 dma_lim = dmareq->dmar_limits; 4433 dma_attr.dma_attr_version = DMA_ATTR_V0; 4434 dma_attr.dma_attr_addr_lo = dma_lim->dlim_addr_lo; 4435 dma_attr.dma_attr_addr_hi = dma_lim->dlim_addr_hi; 4436 dma_attr.dma_attr_minxfer = dma_lim->dlim_minxfer; 4437 dma_attr.dma_attr_seg = dma_lim->dlim_adreg_max; 4438 dma_attr.dma_attr_count_max = dma_lim->dlim_ctreg_max; 4439 dma_attr.dma_attr_granular = dma_lim->dlim_granular; 4440 dma_attr.dma_attr_sgllen = dma_lim->dlim_sgllen; 4441 dma_attr.dma_attr_maxxfer = dma_lim->dlim_reqsize; 4442 dma_attr.dma_attr_burstsizes = dma_lim->dlim_burstsizes; 4443 dma_attr.dma_attr_align = MMU_PAGESIZE; 4444 dma_attr.dma_attr_flags = 0; 4445 4446 e = rootnex_dma_allochdl(dip, rdip, &dma_attr, dmareq->dmar_fp, 4447 dmareq->dmar_arg, lhandlep); 4448 if (e != DDI_SUCCESS) { 4449 return (e); 4450 } 4451 4452 e = rootnex_dma_bindhdl(dip, rdip, *lhandlep, dmareq, &cookie, &ccnt); 4453 if ((e != DDI_DMA_MAPPED) && (e != DDI_DMA_PARTIAL_MAP)) { 4454 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4455 return (e); 4456 } 4457 4458 /* 4459 * if the driver is just testing to see if it's possible to do the bind, 4460 * free up the local state and return the result. 4461 */ 4462 if (handlep == NULL) { 4463 (void) rootnex_dma_unbindhdl(dip, rdip, *lhandlep); 4464 (void) rootnex_dma_freehdl(dip, rdip, *lhandlep); 4465 if (e == DDI_DMA_MAPPED) { 4466 return (DDI_DMA_MAPOK); 4467 } else { 4468 return (DDI_DMA_NOMAPPING); 4469 } 4470 } 4471 4472 return (e); 4473 #endif /* defined(__amd64) */ 4474 } 4475 4476 /* 4477 * rootnex_dma_map() 4478 * called from ddi_dma_setup() 4479 */ 4480 /* ARGSUSED */ 4481 static int 4482 rootnex_dma_map(dev_info_t *dip, dev_info_t *rdip, 4483 struct ddi_dma_req *dmareq, ddi_dma_handle_t *handlep) 4484 { 4485 #if !defined(__xpv) 4486 if (IOMMU_USED(rdip)) { 4487 return (iommulib_nexdma_map(dip, rdip, dmareq, handlep)); 4488 } 4489 #endif 4490 return (rootnex_coredma_map(dip, rdip, dmareq, handlep)); 4491 } 4492 4493 /* 4494 * rootnex_dma_mctl() 4495 * 4496 */ 4497 /* ARGSUSED */ 4498 static int 4499 rootnex_coredma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4500 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4501 uint_t cache_flags) 4502 { 4503 #if defined(__amd64) 4504 /* 4505 * DDI_DMA_SMEM_ALLOC & DDI_DMA_IOPB_ALLOC we're changed to have a 4506 * common implementation in genunix, so they no longer have x86 4507 * specific functionality which called into dma_ctl. 4508 * 4509 * The rest of the obsoleted interfaces were never supported in the 4510 * 64-bit x86 kernel. For s10, the obsoleted DDI_DMA_SEGTOC interface 4511 * was not ported to the x86 64-bit kernel do to serious x86 rootnex 4512 * implementation issues. 4513 * 4514 * If you can't use DDI_DMA_SEGTOC; DDI_DMA_NEXTSEG, DDI_DMA_FREE, and 4515 * DDI_DMA_NEXTWIN are useless since you can get to the cookie, so we 4516 * reflect that now too... 4517 * 4518 * Even though we fixed the pointer problem in DDI_DMA_SEGTOC, we are 4519 * not going to put this functionality into the 64-bit x86 kernel now. 4520 * It wasn't ported to the 64-bit kernel for s10, no reason to change 4521 * that in a future release. 4522 */ 4523 return (DDI_FAILURE); 4524 4525 #else /* 32-bit x86 kernel */ 4526 ddi_dma_cookie_t lcookie; 4527 ddi_dma_cookie_t *cookie; 4528 rootnex_window_t *window; 4529 ddi_dma_impl_t *hp; 4530 rootnex_dma_t *dma; 4531 uint_t nwin; 4532 uint_t ccnt; 4533 size_t len; 4534 off_t off; 4535 int e; 4536 4537 4538 /* 4539 * DDI_DMA_SEGTOC, DDI_DMA_NEXTSEG, and DDI_DMA_NEXTWIN are a little 4540 * hacky since were optimizing for the current interfaces and so we can 4541 * cleanup the mess in genunix. Hopefully we will remove the this 4542 * obsoleted routines someday soon. 4543 */ 4544 4545 switch (request) { 4546 4547 case DDI_DMA_SEGTOC: /* ddi_dma_segtocookie() */ 4548 hp = (ddi_dma_impl_t *)handle; 4549 cookie = (ddi_dma_cookie_t *)objpp; 4550 4551 /* 4552 * convert segment to cookie. We don't distinguish between the 4553 * two :-) 4554 */ 4555 *cookie = *hp->dmai_cookie; 4556 *lenp = cookie->dmac_size; 4557 *offp = cookie->dmac_type & ~ROOTNEX_USES_COPYBUF; 4558 return (DDI_SUCCESS); 4559 4560 case DDI_DMA_NEXTSEG: /* ddi_dma_nextseg() */ 4561 hp = (ddi_dma_impl_t *)handle; 4562 dma = (rootnex_dma_t *)hp->dmai_private; 4563 4564 if ((*lenp != NULL) && ((uintptr_t)*lenp != (uintptr_t)hp)) { 4565 return (DDI_DMA_STALE); 4566 } 4567 4568 /* handle the case where we don't have any windows */ 4569 if (dma->dp_window == NULL) { 4570 /* 4571 * if seg == NULL, and we don't have any windows, 4572 * return the first cookie in the sgl. 4573 */ 4574 if (*lenp == NULL) { 4575 dma->dp_current_cookie = 0; 4576 hp->dmai_cookie = dma->dp_cookies; 4577 *objpp = (caddr_t)handle; 4578 return (DDI_SUCCESS); 4579 4580 /* if we have more cookies, go to the next cookie */ 4581 } else { 4582 if ((dma->dp_current_cookie + 1) >= 4583 dma->dp_sglinfo.si_sgl_size) { 4584 return (DDI_DMA_DONE); 4585 } 4586 dma->dp_current_cookie++; 4587 hp->dmai_cookie++; 4588 return (DDI_SUCCESS); 4589 } 4590 } 4591 4592 /* We have one or more windows */ 4593 window = &dma->dp_window[dma->dp_current_win]; 4594 4595 /* 4596 * if seg == NULL, return the first cookie in the current 4597 * window 4598 */ 4599 if (*lenp == NULL) { 4600 dma->dp_current_cookie = 0; 4601 hp->dmai_cookie = window->wd_first_cookie; 4602 4603 /* 4604 * go to the next cookie in the window then see if we done with 4605 * this window. 4606 */ 4607 } else { 4608 if ((dma->dp_current_cookie + 1) >= 4609 window->wd_cookie_cnt) { 4610 return (DDI_DMA_DONE); 4611 } 4612 dma->dp_current_cookie++; 4613 hp->dmai_cookie++; 4614 } 4615 *objpp = (caddr_t)handle; 4616 return (DDI_SUCCESS); 4617 4618 case DDI_DMA_NEXTWIN: /* ddi_dma_nextwin() */ 4619 hp = (ddi_dma_impl_t *)handle; 4620 dma = (rootnex_dma_t *)hp->dmai_private; 4621 4622 if ((*offp != NULL) && ((uintptr_t)*offp != (uintptr_t)hp)) { 4623 return (DDI_DMA_STALE); 4624 } 4625 4626 /* if win == NULL, return the first window in the bind */ 4627 if (*offp == NULL) { 4628 nwin = 0; 4629 4630 /* 4631 * else, go to the next window then see if we're done with all 4632 * the windows. 4633 */ 4634 } else { 4635 nwin = dma->dp_current_win + 1; 4636 if (nwin >= hp->dmai_nwin) { 4637 return (DDI_DMA_DONE); 4638 } 4639 } 4640 4641 /* switch to the next window */ 4642 e = rootnex_dma_win(dip, rdip, handle, nwin, &off, &len, 4643 &lcookie, &ccnt); 4644 ASSERT(e == DDI_SUCCESS); 4645 if (e != DDI_SUCCESS) { 4646 return (DDI_DMA_STALE); 4647 } 4648 4649 /* reset the cookie back to the first cookie in the window */ 4650 if (dma->dp_window != NULL) { 4651 window = &dma->dp_window[dma->dp_current_win]; 4652 hp->dmai_cookie = window->wd_first_cookie; 4653 } else { 4654 hp->dmai_cookie = dma->dp_cookies; 4655 } 4656 4657 *objpp = (caddr_t)handle; 4658 return (DDI_SUCCESS); 4659 4660 case DDI_DMA_FREE: /* ddi_dma_free() */ 4661 (void) rootnex_dma_unbindhdl(dip, rdip, handle); 4662 (void) rootnex_dma_freehdl(dip, rdip, handle); 4663 if (rootnex_state->r_dvma_call_list_id) { 4664 ddi_run_callback(&rootnex_state->r_dvma_call_list_id); 4665 } 4666 return (DDI_SUCCESS); 4667 4668 case DDI_DMA_IOPB_ALLOC: /* get contiguous DMA-able memory */ 4669 case DDI_DMA_SMEM_ALLOC: /* get contiguous DMA-able memory */ 4670 /* should never get here, handled in genunix */ 4671 ASSERT(0); 4672 return (DDI_FAILURE); 4673 4674 case DDI_DMA_KVADDR: 4675 case DDI_DMA_GETERR: 4676 case DDI_DMA_COFF: 4677 return (DDI_FAILURE); 4678 } 4679 4680 return (DDI_FAILURE); 4681 #endif /* defined(__amd64) */ 4682 } 4683 4684 /* 4685 * rootnex_dma_mctl() 4686 * 4687 */ 4688 /* ARGSUSED */ 4689 static int 4690 rootnex_dma_mctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle, 4691 enum ddi_dma_ctlops request, off_t *offp, size_t *lenp, caddr_t *objpp, 4692 uint_t cache_flags) 4693 { 4694 #if !defined(__xpv) 4695 if (IOMMU_USED(rdip)) { 4696 return (iommulib_nexdma_mctl(dip, rdip, handle, request, offp, 4697 lenp, objpp, cache_flags)); 4698 } 4699 #endif 4700 4701 return (rootnex_coredma_mctl(dip, rdip, handle, request, offp, 4702 lenp, objpp, cache_flags)); 4703 } 4704 4705 /* 4706 * ********* 4707 * FMA Code 4708 * ********* 4709 */ 4710 4711 /* 4712 * rootnex_fm_init() 4713 * FMA init busop 4714 */ 4715 /* ARGSUSED */ 4716 static int 4717 rootnex_fm_init(dev_info_t *dip, dev_info_t *tdip, int tcap, 4718 ddi_iblock_cookie_t *ibc) 4719 { 4720 *ibc = rootnex_state->r_err_ibc; 4721 4722 return (ddi_system_fmcap); 4723 } 4724 4725 /* 4726 * rootnex_dma_check() 4727 * Function called after a dma fault occurred to find out whether the 4728 * fault address is associated with a driver that is able to handle faults 4729 * and recover from faults. 4730 */ 4731 /* ARGSUSED */ 4732 static int 4733 rootnex_dma_check(dev_info_t *dip, const void *handle, const void *addr, 4734 const void *not_used) 4735 { 4736 rootnex_window_t *window; 4737 uint64_t start_addr; 4738 uint64_t fault_addr; 4739 ddi_dma_impl_t *hp; 4740 rootnex_dma_t *dma; 4741 uint64_t end_addr; 4742 size_t csize; 4743 int i; 4744 int j; 4745 4746 4747 /* The driver has to set DDI_DMA_FLAGERR to recover from dma faults */ 4748 hp = (ddi_dma_impl_t *)handle; 4749 ASSERT(hp); 4750 4751 dma = (rootnex_dma_t *)hp->dmai_private; 4752 4753 /* Get the address that we need to search for */ 4754 fault_addr = *(uint64_t *)addr; 4755 4756 /* 4757 * if we don't have any windows, we can just walk through all the 4758 * cookies. 4759 */ 4760 if (dma->dp_window == NULL) { 4761 /* for each cookie */ 4762 for (i = 0; i < dma->dp_sglinfo.si_sgl_size; i++) { 4763 /* 4764 * if the faulted address is within the physical address 4765 * range of the cookie, return DDI_FM_NONFATAL. 4766 */ 4767 if ((fault_addr >= dma->dp_cookies[i].dmac_laddress) && 4768 (fault_addr <= (dma->dp_cookies[i].dmac_laddress + 4769 dma->dp_cookies[i].dmac_size))) { 4770 return (DDI_FM_NONFATAL); 4771 } 4772 } 4773 4774 /* fault_addr not within this DMA handle */ 4775 return (DDI_FM_UNKNOWN); 4776 } 4777 4778 /* we have mutiple windows, walk through each window */ 4779 for (i = 0; i < hp->dmai_nwin; i++) { 4780 window = &dma->dp_window[i]; 4781 4782 /* Go through all the cookies in the window */ 4783 for (j = 0; j < window->wd_cookie_cnt; j++) { 4784 4785 start_addr = window->wd_first_cookie[j].dmac_laddress; 4786 csize = window->wd_first_cookie[j].dmac_size; 4787 4788 /* 4789 * if we are trimming the first cookie in the window, 4790 * and this is the first cookie, adjust the start 4791 * address and size of the cookie to account for the 4792 * trim. 4793 */ 4794 if (window->wd_trim.tr_trim_first && (j == 0)) { 4795 start_addr = window->wd_trim.tr_first_paddr; 4796 csize = window->wd_trim.tr_first_size; 4797 } 4798 4799 /* 4800 * if we are trimming the last cookie in the window, 4801 * and this is the last cookie, adjust the start 4802 * address and size of the cookie to account for the 4803 * trim. 4804 */ 4805 if (window->wd_trim.tr_trim_last && 4806 (j == (window->wd_cookie_cnt - 1))) { 4807 start_addr = window->wd_trim.tr_last_paddr; 4808 csize = window->wd_trim.tr_last_size; 4809 } 4810 4811 end_addr = start_addr + csize; 4812 4813 /* 4814 * if the faulted address is within the physical address 4815 * range of the cookie, return DDI_FM_NONFATAL. 4816 */ 4817 if ((fault_addr >= start_addr) && 4818 (fault_addr <= end_addr)) { 4819 return (DDI_FM_NONFATAL); 4820 } 4821 } 4822 } 4823 4824 /* fault_addr not within this DMA handle */ 4825 return (DDI_FM_UNKNOWN); 4826 } 4827