1 /* $NetBSD: iommu.c,v 1.107 2012/03/25 03:51:33 mrg Exp $ */ 2 3 /* 4 * Copyright (c) 1999, 2000 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 16 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 17 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 18 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 19 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 20 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 21 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 22 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 23 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 24 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 25 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 26 * SUCH DAMAGE. 27 */ 28 29 /* 30 * Copyright (c) 2001, 2002 Eduardo Horvath 31 * All rights reserved. 32 * 33 * Redistribution and use in source and binary forms, with or without 34 * modification, are permitted provided that the following conditions 35 * are met: 36 * 1. Redistributions of source code must retain the above copyright 37 * notice, this list of conditions and the following disclaimer. 38 * 2. Redistributions in binary form must reproduce the above copyright 39 * notice, this list of conditions and the following disclaimer in the 40 * documentation and/or other materials provided with the distribution. 41 * 3. The name of the author may not be used to endorse or promote products 42 * derived from this software without specific prior written permission. 43 * 44 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 45 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 46 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 47 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 48 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 49 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 50 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 51 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 52 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 53 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 54 * SUCH DAMAGE. 55 */ 56 57 /* 58 * UltraSPARC IOMMU support; used by both the sbus and pci code. 59 */ 60 61 #include <sys/cdefs.h> 62 __KERNEL_RCSID(0, "$NetBSD: iommu.c,v 1.107 2012/03/25 03:51:33 mrg Exp $"); 63 64 #include "opt_ddb.h" 65 66 #include <sys/param.h> 67 #include <sys/extent.h> 68 #include <sys/malloc.h> 69 #include <sys/systm.h> 70 #include <sys/device.h> 71 #include <sys/proc.h> 72 73 #include <uvm/uvm.h> 74 75 #include <sys/bus.h> 76 #include <sparc64/dev/iommureg.h> 77 #include <sparc64/dev/iommuvar.h> 78 79 #include <machine/autoconf.h> 80 #include <machine/cpu.h> 81 82 #ifdef DEBUG 83 #define IDB_BUSDMA 0x1 84 #define IDB_IOMMU 0x2 85 #define IDB_INFO 0x4 86 #define IDB_SYNC 0x8 87 int iommudebug = 0x0; 88 #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0) 89 #define IOTTE_DEBUG(n) (n) 90 #else 91 #define DPRINTF(l, s) 92 #define IOTTE_DEBUG(n) 0 93 #endif 94 95 #define iommu_strbuf_flush(i, v) do { \ 96 if ((i)->sb_flush) \ 97 bus_space_write_8((i)->sb_is->is_bustag, (i)->sb_sb, \ 98 STRBUFREG(strbuf_pgflush), (v)); \ 99 } while (0) 100 101 static int iommu_strbuf_flush_done(struct strbuf_ctl *); 102 static void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t, 103 bus_size_t, int); 104 105 /* 106 * initialise the UltraSPARC IOMMU (SBUS or PCI): 107 * - allocate and setup the iotsb. 108 * - enable the IOMMU 109 * - initialise the streaming buffers (if they exist) 110 * - create a private DVMA map. 111 */ 112 void 113 iommu_init(char *name, struct iommu_state *is, int tsbsize, uint32_t iovabase) 114 { 115 psize_t size; 116 vaddr_t va; 117 paddr_t pa; 118 struct vm_page *pg; 119 struct pglist pglist; 120 121 /* 122 * Setup the iommu. 123 * 124 * The sun4u iommu is part of the SBUS or PCI controller so we will 125 * deal with it here.. 126 * 127 * For sysio and psycho/psycho+ the IOMMU address space always ends at 128 * 0xffffe000, but the starting address depends on the size of the 129 * map. The map size is 1024 * 2 ^ is->is_tsbsize entries, where each 130 * entry is 8 bytes. The start of the map can be calculated by 131 * (0xffffe000 << (8 + is->is_tsbsize)). 132 * 133 * But sabre and hummingbird use a different scheme that seems to 134 * be hard-wired, so we read the start and size from the PROM and 135 * just use those values. 136 */ 137 is->is_cr = IOMMUCR_EN; 138 is->is_tsbsize = tsbsize; 139 if (iovabase == -1) { 140 is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); 141 is->is_dvmaend = IOTSB_VEND - 1; 142 } else { 143 is->is_dvmabase = iovabase; 144 is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1; 145 } 146 147 /* 148 * Allocate memory for I/O pagetables. They need to be physically 149 * contiguous. 150 */ 151 152 size = PAGE_SIZE << is->is_tsbsize; 153 if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1, 154 (paddr_t)PAGE_SIZE, (paddr_t)0, &pglist, 1, 0) != 0) 155 panic("iommu_init: no memory"); 156 157 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY); 158 if (va == 0) 159 panic("iommu_init: no memory"); 160 is->is_tsb = (int64_t *)va; 161 162 is->is_ptsb = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)); 163 164 /* Map the pages */ 165 TAILQ_FOREACH(pg, &pglist, pageq.queue) { 166 pa = VM_PAGE_TO_PHYS(pg); 167 pmap_kenter_pa(va, pa | PMAP_NVC, 168 VM_PROT_READ | VM_PROT_WRITE, 0); 169 va += PAGE_SIZE; 170 } 171 pmap_update(pmap_kernel()); 172 memset(is->is_tsb, 0, size); 173 174 #ifdef DEBUG 175 if (iommudebug & IDB_INFO) 176 { 177 /* Probe the iommu */ 178 179 printf("iommu cr=%llx tsb=%llx\n", 180 (unsigned long long)bus_space_read_8(is->is_bustag, 181 is->is_iommu, 182 offsetof(struct iommureg, iommu_cr)), 183 (unsigned long long)bus_space_read_8(is->is_bustag, 184 is->is_iommu, 185 offsetof(struct iommureg, iommu_tsb))); 186 printf("TSB base %p phys %llx\n", (void *)is->is_tsb, 187 (unsigned long long)is->is_ptsb); 188 delay(1000000); /* 1 s */ 189 } 190 #endif 191 192 /* 193 * Now all the hardware's working we need to allocate a dvma map. 194 */ 195 aprint_debug("DVMA map: %x to %x\n", 196 (unsigned int)is->is_dvmabase, 197 (unsigned int)is->is_dvmaend); 198 aprint_debug("IOTSB: %llx to %llx\n", 199 (unsigned long long)is->is_ptsb, 200 (unsigned long long)(is->is_ptsb + size - 1)); 201 is->is_dvmamap = extent_create(name, 202 is->is_dvmabase, is->is_dvmaend, 203 0, 0, EX_NOWAIT); 204 /* XXXMRG Check is_dvmamap is valid. */ 205 206 mutex_init(&is->is_lock, MUTEX_DEFAULT, IPL_HIGH); 207 208 /* 209 * Set the TSB size. The relevant bits were moved to the TSB 210 * base register in the PCIe host bridges. 211 */ 212 if (is->is_flags & IOMMU_TSBSIZE_IN_PTSB) 213 is->is_ptsb |= is->is_tsbsize; 214 else 215 is->is_cr |= (is->is_tsbsize << 16); 216 217 /* 218 * now actually start up the IOMMU 219 */ 220 iommu_reset(is); 221 } 222 223 /* 224 * Streaming buffers don't exist on the UltraSPARC IIi; we should have 225 * detected that already and disabled them. If not, we will notice that 226 * they aren't there when the STRBUF_EN bit does not remain. 227 */ 228 void 229 iommu_reset(struct iommu_state *is) 230 { 231 int i; 232 struct strbuf_ctl *sb; 233 234 IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb); 235 236 /* Enable IOMMU in diagnostic mode */ 237 IOMMUREG_WRITE(is, iommu_cr, is->is_cr|IOMMUCR_DE); 238 239 for (i = 0; i < 2; i++) { 240 if ((sb = is->is_sb[i])) { 241 242 /* Enable diagnostics mode? */ 243 bus_space_write_8(is->is_bustag, is->is_sb[i]->sb_sb, 244 STRBUFREG(strbuf_ctl), STRBUF_EN); 245 246 membar_Lookaside(); 247 248 /* No streaming buffers? Disable them */ 249 if (bus_space_read_8(is->is_bustag, 250 is->is_sb[i]->sb_sb, 251 STRBUFREG(strbuf_ctl)) == 0) { 252 is->is_sb[i]->sb_flush = NULL; 253 } else { 254 255 /* 256 * locate the pa of the flush buffer. 257 */ 258 if (pmap_extract(pmap_kernel(), 259 (vaddr_t)is->is_sb[i]->sb_flush, 260 &is->is_sb[i]->sb_flushpa) == FALSE) 261 is->is_sb[i]->sb_flush = NULL; 262 } 263 } 264 } 265 266 if (is->is_flags & IOMMU_FLUSH_CACHE) 267 IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL); 268 } 269 270 /* 271 * Here are the iommu control routines. 272 */ 273 void 274 iommu_enter(struct strbuf_ctl *sb, vaddr_t va, int64_t pa, int flags) 275 { 276 struct iommu_state *is = sb->sb_is; 277 int strbuf = (flags & BUS_DMA_STREAMING); 278 int64_t tte; 279 280 #ifdef DIAGNOSTIC 281 if (va < is->is_dvmabase || va > is->is_dvmaend) 282 panic("iommu_enter: va %#lx not in DVMA space", va); 283 #endif 284 285 /* Is the streamcache flush really needed? */ 286 if (sb->sb_flush) 287 iommu_strbuf_flush(sb, va); 288 else 289 /* If we can't flush the strbuf don't enable it. */ 290 strbuf = 0; 291 292 tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE), 293 !(flags & BUS_DMA_NOCACHE), (strbuf)); 294 #ifdef DEBUG 295 tte |= (flags & 0xff000LL)<<(4*8); 296 #endif 297 298 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte; 299 bus_space_write_8(is->is_bustag, is->is_iommu, 300 IOMMUREG(iommu_flush), va); 301 DPRINTF(IDB_IOMMU, ("iommu_enter: slot %d va %lx pa %lx " 302 "TSB[%lx]@%p=%lx\n", (int)IOTSBSLOT(va,is->is_tsbsize), 303 va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize), 304 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 305 (u_long)tte)); 306 } 307 308 /* 309 * Find the value of a DVMA address (debug routine). 310 */ 311 paddr_t 312 iommu_extract(struct iommu_state *is, vaddr_t dva) 313 { 314 int64_t tte = 0; 315 316 if (dva >= is->is_dvmabase && dva <= is->is_dvmaend) 317 tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)]; 318 319 if ((tte & IOTTE_V) == 0) 320 return ((paddr_t)-1L); 321 return (tte & IOTTE_PAMASK); 322 } 323 324 /* 325 * iommu_remove: removes mappings created by iommu_enter 326 * 327 * Only demap from IOMMU if flag is set. 328 * 329 * XXX: this function needs better internal error checking. 330 */ 331 void 332 iommu_remove(struct iommu_state *is, vaddr_t va, size_t len) 333 { 334 int slot; 335 336 #ifdef DIAGNOSTIC 337 if (va < is->is_dvmabase || va > is->is_dvmaend) 338 panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va); 339 if ((long)(va + len) < (long)va) 340 panic("iommu_remove: va 0x%lx + len 0x%lx wraps", 341 (long) va, (long) len); 342 if (len & ~0xfffffff) 343 panic("iommu_remove: ridiculous len 0x%lx", (u_long)len); 344 #endif 345 346 va = trunc_page(va); 347 DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n", 348 va, (u_long)IOTSBSLOT(va, is->is_tsbsize), 349 &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)])); 350 while (len > 0) { 351 DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d " 352 "for va %p size %lx\n", 353 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va, 354 (u_long)len)); 355 if (len <= PAGE_SIZE) 356 len = 0; 357 else 358 len -= PAGE_SIZE; 359 360 #if 0 361 /* 362 * XXX Zero-ing the entry would not require RMW 363 * 364 * Disabling valid bit while a page is used by a device 365 * causes an uncorrectable DMA error. 366 * Workaround to avoid an uncorrectable DMA error is 367 * eliminating the next line, but the page is mapped 368 * until the next iommu_enter call. 369 */ 370 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] &= ~IOTTE_V; 371 membar_StoreStore(); 372 #endif 373 IOMMUREG_WRITE(is, iommu_flush, va); 374 375 /* Flush cache if necessary. */ 376 slot = IOTSBSLOT(trunc_page(va), is->is_tsbsize); 377 if ((is->is_flags & IOMMU_FLUSH_CACHE) && 378 (len == 0 || (slot % 8) == 7)) 379 IOMMUREG_WRITE(is, iommu_cache_flush, 380 is->is_ptsb + slot * 8); 381 382 va += PAGE_SIZE; 383 } 384 } 385 386 static int 387 iommu_strbuf_flush_done(struct strbuf_ctl *sb) 388 { 389 struct iommu_state *is = sb->sb_is; 390 struct timeval cur, flushtimeout; 391 392 #define BUMPTIME(t, usec) { \ 393 register volatile struct timeval *tp = (t); \ 394 register long us; \ 395 \ 396 tp->tv_usec = us = tp->tv_usec + (usec); \ 397 if (us >= 1000000) { \ 398 tp->tv_usec = us - 1000000; \ 399 tp->tv_sec++; \ 400 } \ 401 } 402 403 if (!sb->sb_flush) 404 return (0); 405 406 /* 407 * Streaming buffer flushes: 408 * 409 * 1 Tell strbuf to flush by storing va to strbuf_pgflush. If 410 * we're not on a cache line boundary (64-bits): 411 * 2 Store 0 in flag 412 * 3 Store pointer to flag in flushsync 413 * 4 wait till flushsync becomes 0x1 414 * 415 * If it takes more than .5 sec, something 416 * went wrong. 417 */ 418 419 *sb->sb_flush = 0; 420 bus_space_write_8(is->is_bustag, sb->sb_sb, 421 STRBUFREG(strbuf_flushsync), sb->sb_flushpa); 422 423 microtime(&flushtimeout); 424 cur = flushtimeout; 425 BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */ 426 427 DPRINTF(IDB_IOMMU, ("%s: flush = %lx at va = %lx pa = %lx now=" 428 "%"PRIx64":%"PRIx32" until = %"PRIx64":%"PRIx32"\n", __func__, 429 (long)*sb->sb_flush, (long)sb->sb_flush, (long)sb->sb_flushpa, 430 cur.tv_sec, cur.tv_usec, 431 flushtimeout.tv_sec, flushtimeout.tv_usec)); 432 433 /* Bypass non-coherent D$ */ 434 while ((!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) && 435 timercmp(&cur, &flushtimeout, <=)) 436 microtime(&cur); 437 438 #ifdef DIAGNOSTIC 439 if (!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) { 440 printf("%s: flush timeout %p, at %p\n", __func__, 441 (void *)(u_long)*sb->sb_flush, 442 (void *)(u_long)sb->sb_flushpa); /* panic? */ 443 #ifdef DDB 444 Debugger(); 445 #endif 446 } 447 #endif 448 DPRINTF(IDB_IOMMU, ("%s: flushed\n", __func__)); 449 return (*sb->sb_flush); 450 } 451 452 /* 453 * IOMMU DVMA operations, common to SBUS and PCI. 454 */ 455 int 456 iommu_dvmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 457 bus_size_t buflen, struct proc *p, int flags) 458 { 459 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 460 struct iommu_state *is = sb->sb_is; 461 int err, needsflush; 462 bus_size_t sgsize; 463 paddr_t curaddr; 464 u_long dvmaddr, sgstart, sgend, bmask; 465 bus_size_t align, boundary, len; 466 vaddr_t vaddr = (vaddr_t)buf; 467 int seg; 468 struct pmap *pmap; 469 int slot; 470 471 if (map->dm_nsegs) { 472 /* Already in use?? */ 473 #ifdef DIAGNOSTIC 474 printf("iommu_dvmamap_load: map still in use\n"); 475 #endif 476 bus_dmamap_unload(t, map); 477 } 478 479 /* 480 * Make sure that on error condition we return "no valid mappings". 481 */ 482 map->dm_nsegs = 0; 483 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz); 484 485 if (buflen > map->_dm_size) { 486 DPRINTF(IDB_BUSDMA, 487 ("iommu_dvmamap_load(): error %d > %d -- " 488 "map size exceeded!\n", (int)buflen, (int)map->_dm_size)); 489 return (EINVAL); 490 } 491 492 sgsize = round_page(buflen + ((int)vaddr & PGOFSET)); 493 494 /* 495 * A boundary presented to bus_dmamem_alloc() takes precedence 496 * over boundary in the map. 497 */ 498 if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0) 499 boundary = map->_dm_boundary; 500 align = max(map->dm_segs[0]._ds_align, PAGE_SIZE); 501 502 /* 503 * If our segment size is larger than the boundary we need to 504 * split the transfer up int little pieces ourselves. 505 */ 506 KASSERT(is->is_dvmamap); 507 mutex_enter(&is->is_lock); 508 err = extent_alloc(is->is_dvmamap, sgsize, align, 509 (sgsize > boundary) ? 0 : boundary, 510 EX_NOWAIT|EX_BOUNDZERO, &dvmaddr); 511 mutex_exit(&is->is_lock); 512 513 #ifdef DEBUG 514 if (err || (dvmaddr == (u_long)-1)) { 515 printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n", 516 (int)sgsize, flags); 517 #ifdef DDB 518 Debugger(); 519 #endif 520 } 521 #endif 522 if (err != 0) 523 return (err); 524 525 if (dvmaddr == (u_long)-1) 526 return (ENOMEM); 527 528 /* Set the active DVMA map */ 529 map->_dm_dvmastart = dvmaddr; 530 map->_dm_dvmasize = sgsize; 531 532 /* 533 * Now split the DVMA range into segments, not crossing 534 * the boundary. 535 */ 536 seg = 0; 537 sgstart = dvmaddr + (vaddr & PGOFSET); 538 sgend = sgstart + buflen - 1; 539 map->dm_segs[seg].ds_addr = sgstart; 540 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: boundary %lx boundary - 1 %lx " 541 "~(boundary - 1) %lx\n", (long)boundary, (long)(boundary - 1), 542 (long)~(boundary - 1))); 543 bmask = ~(boundary - 1); 544 while ((sgstart & bmask) != (sgend & bmask) || 545 sgend - sgstart + 1 > map->dm_maxsegsz) { 546 /* Oops. We crossed a boundary or large seg. Split the xfer. */ 547 len = map->dm_maxsegsz; 548 if ((sgstart & bmask) != (sgend & bmask)) 549 len = min(len, boundary - (sgstart & (boundary - 1))); 550 map->dm_segs[seg].ds_len = len; 551 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 552 "seg %d start %lx size %lx\n", seg, 553 (long)map->dm_segs[seg].ds_addr, 554 (long)map->dm_segs[seg].ds_len)); 555 if (++seg >= map->_dm_segcnt) { 556 /* Too many segments. Fail the operation. */ 557 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 558 "too many segments %d\n", seg)); 559 mutex_enter(&is->is_lock); 560 err = extent_free(is->is_dvmamap, 561 dvmaddr, sgsize, EX_NOWAIT); 562 map->_dm_dvmastart = 0; 563 map->_dm_dvmasize = 0; 564 mutex_exit(&is->is_lock); 565 if (err != 0) 566 printf("warning: %s: %" PRId64 567 " of DVMA space lost\n", __func__, sgsize); 568 return (EFBIG); 569 } 570 sgstart += len; 571 map->dm_segs[seg].ds_addr = sgstart; 572 } 573 map->dm_segs[seg].ds_len = sgend - sgstart + 1; 574 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 575 "seg %d start %lx size %lx\n", seg, 576 (long)map->dm_segs[seg].ds_addr, (long)map->dm_segs[seg].ds_len)); 577 map->dm_nsegs = seg + 1; 578 map->dm_mapsize = buflen; 579 580 if (p != NULL) 581 pmap = p->p_vmspace->vm_map.pmap; 582 else 583 pmap = pmap_kernel(); 584 585 needsflush = 0; 586 for (; buflen > 0; ) { 587 588 /* 589 * Get the physical address for this page. 590 */ 591 if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) { 592 #ifdef DIAGNOSTIC 593 printf("iommu_dvmamap_load: pmap_extract failed %lx\n", vaddr); 594 #endif 595 bus_dmamap_unload(t, map); 596 return (-1); 597 } 598 599 /* 600 * Compute the segment size, and adjust counts. 601 */ 602 sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET); 603 if (buflen < sgsize) 604 sgsize = buflen; 605 606 DPRINTF(IDB_BUSDMA, 607 ("iommu_dvmamap_load: map %p loading va %p " 608 "dva %lx at pa %lx\n", 609 map, (void *)vaddr, (long)dvmaddr, 610 (long)trunc_page(curaddr))); 611 iommu_enter(sb, trunc_page(dvmaddr), trunc_page(curaddr), 612 flags | IOTTE_DEBUG(0x4000)); 613 needsflush = 1; 614 615 vaddr += sgsize; 616 buflen -= sgsize; 617 618 /* Flush cache if necessary. */ 619 slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize); 620 if ((is->is_flags & IOMMU_FLUSH_CACHE) && 621 (buflen <= 0 || (slot % 8) == 7)) 622 IOMMUREG_WRITE(is, iommu_cache_flush, 623 is->is_ptsb + slot * 8); 624 625 dvmaddr += PAGE_SIZE; 626 } 627 if (needsflush) 628 iommu_strbuf_flush_done(sb); 629 #ifdef DIAGNOSTIC 630 for (seg = 0; seg < map->dm_nsegs; seg++) { 631 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 632 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 633 printf("seg %d dvmaddr %lx out of range %x - %x\n", 634 seg, (long)map->dm_segs[seg].ds_addr, 635 is->is_dvmabase, is->is_dvmaend); 636 #ifdef DDB 637 Debugger(); 638 #endif 639 } 640 } 641 #endif 642 return (0); 643 } 644 645 646 void 647 iommu_dvmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) 648 { 649 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 650 struct iommu_state *is = sb->sb_is; 651 int error; 652 bus_size_t sgsize = map->_dm_dvmasize; 653 654 /* Flush the iommu */ 655 #ifdef DEBUG 656 if (!map->_dm_dvmastart) { 657 printf("iommu_dvmamap_unload: No dvmastart is zero\n"); 658 #ifdef DDB 659 Debugger(); 660 #endif 661 } 662 #endif 663 iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize); 664 665 /* Flush the caches */ 666 bus_dmamap_unload(t->_parent, map); 667 668 mutex_enter(&is->is_lock); 669 error = extent_free(is->is_dvmamap, map->_dm_dvmastart, 670 map->_dm_dvmasize, EX_NOWAIT); 671 map->_dm_dvmastart = 0; 672 map->_dm_dvmasize = 0; 673 mutex_exit(&is->is_lock); 674 if (error != 0) 675 printf("warning: %s: %" PRId64 " of DVMA space lost\n", 676 __func__, sgsize); 677 678 /* Clear the map */ 679 } 680 681 682 int 683 iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, 684 bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags) 685 { 686 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 687 struct iommu_state *is = sb->sb_is; 688 struct vm_page *pg; 689 int i, j; 690 int left; 691 int err, needsflush; 692 bus_size_t sgsize; 693 paddr_t pa; 694 bus_size_t boundary, align; 695 u_long dvmaddr, sgstart, sgend, bmask; 696 struct pglist *pglist; 697 const int pagesz = PAGE_SIZE; 698 int slot; 699 #ifdef DEBUG 700 int npg = 0; 701 #endif 702 703 if (map->dm_nsegs) { 704 /* Already in use?? */ 705 #ifdef DIAGNOSTIC 706 printf("iommu_dvmamap_load_raw: map still in use\n"); 707 #endif 708 bus_dmamap_unload(t, map); 709 } 710 711 /* 712 * A boundary presented to bus_dmamem_alloc() takes precedence 713 * over boundary in the map. 714 */ 715 if ((boundary = segs[0]._ds_boundary) == 0) 716 boundary = map->_dm_boundary; 717 718 align = max(segs[0]._ds_align, pagesz); 719 720 /* 721 * Make sure that on error condition we return "no valid mappings". 722 */ 723 map->dm_nsegs = 0; 724 /* Count up the total number of pages we need */ 725 pa = trunc_page(segs[0].ds_addr); 726 sgsize = 0; 727 left = size; 728 for (i = 0; left > 0 && i < nsegs; i++) { 729 if (round_page(pa) != round_page(segs[i].ds_addr)) 730 sgsize = round_page(sgsize) + 731 (segs[i].ds_addr & PGOFSET); 732 sgsize += min(left, segs[i].ds_len); 733 left -= segs[i].ds_len; 734 pa = segs[i].ds_addr + segs[i].ds_len; 735 } 736 sgsize = round_page(sgsize); 737 738 mutex_enter(&is->is_lock); 739 /* 740 * If our segment size is larger than the boundary we need to 741 * split the transfer up into little pieces ourselves. 742 */ 743 err = extent_alloc(is->is_dvmamap, sgsize, align, 744 (sgsize > boundary) ? 0 : boundary, 745 ((flags & BUS_DMA_NOWAIT) == 0 ? EX_WAITOK : EX_NOWAIT) | 746 EX_BOUNDZERO, &dvmaddr); 747 mutex_exit(&is->is_lock); 748 749 if (err != 0) 750 return (err); 751 752 #ifdef DEBUG 753 if (dvmaddr == (u_long)-1) 754 { 755 printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n", 756 (int)sgsize, flags); 757 #ifdef DDB 758 Debugger(); 759 #endif 760 } 761 #endif 762 if (dvmaddr == (u_long)-1) 763 return (ENOMEM); 764 765 /* Set the active DVMA map */ 766 map->_dm_dvmastart = dvmaddr; 767 map->_dm_dvmasize = sgsize; 768 769 bmask = ~(boundary - 1); 770 if ((pglist = segs[0]._ds_mlist) == NULL) { 771 u_long prev_va = 0UL, last_va = dvmaddr; 772 paddr_t prev_pa = 0; 773 int end = 0, offset; 774 bus_size_t len = size; 775 776 /* 777 * This segs is made up of individual physical 778 * segments, probably by _bus_dmamap_load_uio() or 779 * _bus_dmamap_load_mbuf(). Ignore the mlist and 780 * load each one individually. 781 */ 782 j = 0; 783 needsflush = 0; 784 for (i = 0; i < nsegs ; i++) { 785 786 pa = segs[i].ds_addr; 787 offset = (pa & PGOFSET); 788 pa = trunc_page(pa); 789 dvmaddr = trunc_page(dvmaddr); 790 left = min(len, segs[i].ds_len); 791 792 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: converting " 793 "physseg %d start %lx size %lx\n", i, 794 (long)segs[i].ds_addr, (long)segs[i].ds_len)); 795 796 if ((pa == prev_pa) && 797 ((offset != 0) || (end != offset))) { 798 /* We can re-use this mapping */ 799 dvmaddr = prev_va; 800 } 801 802 sgstart = dvmaddr + offset; 803 sgend = sgstart + left - 1; 804 805 /* Are the segments virtually adjacent? */ 806 if ((j > 0) && (end == offset) && 807 ((offset == 0) || (pa == prev_pa)) && 808 (map->dm_segs[j-1].ds_len + left <= 809 map->dm_maxsegsz)) { 810 /* Just append to the previous segment. */ 811 map->dm_segs[--j].ds_len += left; 812 /* Restore sgstart for boundary check */ 813 sgstart = map->dm_segs[j].ds_addr; 814 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 815 "appending seg %d start %lx size %lx\n", j, 816 (long)map->dm_segs[j].ds_addr, 817 (long)map->dm_segs[j].ds_len)); 818 } else { 819 if (j >= map->_dm_segcnt) { 820 iommu_remove(is, map->_dm_dvmastart, 821 last_va - map->_dm_dvmastart); 822 goto fail; 823 } 824 map->dm_segs[j].ds_addr = sgstart; 825 map->dm_segs[j].ds_len = left; 826 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 827 "seg %d start %lx size %lx\n", j, 828 (long)map->dm_segs[j].ds_addr, 829 (long)map->dm_segs[j].ds_len)); 830 } 831 end = (offset + left) & PGOFSET; 832 833 /* Check for boundary issues */ 834 while ((sgstart & bmask) != (sgend & bmask)) { 835 /* Need a new segment. */ 836 map->dm_segs[j].ds_len = 837 boundary - (sgstart & (boundary - 1)); 838 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 839 "seg %d start %lx size %lx\n", j, 840 (long)map->dm_segs[j].ds_addr, 841 (long)map->dm_segs[j].ds_len)); 842 if (++j >= map->_dm_segcnt) { 843 iommu_remove(is, map->_dm_dvmastart, 844 last_va - map->_dm_dvmastart); 845 goto fail; 846 } 847 sgstart += map->dm_segs[j-1].ds_len; 848 map->dm_segs[j].ds_addr = sgstart; 849 map->dm_segs[j].ds_len = sgend - sgstart + 1; 850 } 851 852 if (sgsize == 0) 853 panic("iommu_dmamap_load_raw: size botch"); 854 855 /* Now map a series of pages. */ 856 while (dvmaddr <= sgend) { 857 DPRINTF(IDB_BUSDMA, 858 ("iommu_dvmamap_load_raw: map %p " 859 "loading va %lx at pa %lx\n", 860 map, (long)dvmaddr, 861 (long)(pa))); 862 /* Enter it if we haven't before. */ 863 if (prev_va != dvmaddr) { 864 iommu_enter(sb, prev_va = dvmaddr, 865 prev_pa = pa, 866 flags | IOTTE_DEBUG(++npg << 12)); 867 needsflush = 1; 868 869 /* Flush cache if necessary. */ 870 slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize); 871 if ((is->is_flags & IOMMU_FLUSH_CACHE) && 872 ((dvmaddr + pagesz) > sgend || (slot % 8) == 7)) 873 IOMMUREG_WRITE(is, iommu_cache_flush, 874 is->is_ptsb + slot * 8); 875 } 876 877 dvmaddr += pagesz; 878 pa += pagesz; 879 last_va = dvmaddr; 880 } 881 882 len -= left; 883 ++j; 884 } 885 if (needsflush) 886 iommu_strbuf_flush_done(sb); 887 888 map->dm_mapsize = size; 889 map->dm_nsegs = j; 890 #ifdef DIAGNOSTIC 891 { int seg; 892 for (seg = 0; seg < map->dm_nsegs; seg++) { 893 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 894 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 895 printf("seg %d dvmaddr %lx out of range %x - %x\n", 896 seg, (long)map->dm_segs[seg].ds_addr, 897 is->is_dvmabase, is->is_dvmaend); 898 #ifdef DDB 899 Debugger(); 900 #endif 901 } 902 } 903 } 904 #endif 905 return (0); 906 } 907 908 /* 909 * This was allocated with bus_dmamem_alloc. 910 * The pages are on a `pglist'. 911 */ 912 i = 0; 913 sgstart = dvmaddr; 914 sgend = sgstart + size - 1; 915 map->dm_segs[i].ds_addr = sgstart; 916 while ((sgstart & bmask) != (sgend & bmask)) { 917 /* Oops. We crossed a boundary. Split the xfer. */ 918 map->dm_segs[i].ds_len = boundary - (sgstart & (boundary - 1)); 919 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 920 "seg %d start %lx size %lx\n", i, 921 (long)map->dm_segs[i].ds_addr, 922 (long)map->dm_segs[i].ds_len)); 923 if (++i >= map->_dm_segcnt) { 924 /* Too many segments. Fail the operation. */ 925 goto fail; 926 } 927 sgstart += map->dm_segs[i-1].ds_len; 928 map->dm_segs[i].ds_addr = sgstart; 929 } 930 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 931 "seg %d start %lx size %lx\n", i, 932 (long)map->dm_segs[i].ds_addr, (long)map->dm_segs[i].ds_len)); 933 map->dm_segs[i].ds_len = sgend - sgstart + 1; 934 935 needsflush = 0; 936 TAILQ_FOREACH(pg, pglist, pageq.queue) { 937 if (sgsize == 0) 938 panic("iommu_dmamap_load_raw: size botch"); 939 pa = VM_PAGE_TO_PHYS(pg); 940 941 DPRINTF(IDB_BUSDMA, 942 ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n", 943 map, (long)dvmaddr, (long)(pa))); 944 iommu_enter(sb, dvmaddr, pa, flags | IOTTE_DEBUG(0x8000)); 945 needsflush = 1; 946 947 sgsize -= pagesz; 948 949 /* Flush cache if necessary. */ 950 slot = IOTSBSLOT(trunc_page(dvmaddr), is->is_tsbsize); 951 if ((is->is_flags & IOMMU_FLUSH_CACHE) && 952 (sgsize == 0 || (slot % 8) == 7)) 953 IOMMUREG_WRITE(is, iommu_cache_flush, 954 is->is_ptsb + slot * 8); 955 956 dvmaddr += pagesz; 957 } 958 if (needsflush) 959 iommu_strbuf_flush_done(sb); 960 map->dm_mapsize = size; 961 map->dm_nsegs = i+1; 962 #ifdef DIAGNOSTIC 963 { int seg; 964 for (seg = 0; seg < map->dm_nsegs; seg++) { 965 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 966 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 967 printf("seg %d dvmaddr %lx out of range %x - %x\n", 968 seg, (long)map->dm_segs[seg].ds_addr, 969 is->is_dvmabase, is->is_dvmaend); 970 #ifdef DDB 971 Debugger(); 972 #endif 973 } 974 } 975 } 976 #endif 977 return (0); 978 979 fail: 980 mutex_enter(&is->is_lock); 981 err = extent_free(is->is_dvmamap, map->_dm_dvmastart, sgsize, 982 EX_NOWAIT); 983 map->_dm_dvmastart = 0; 984 map->_dm_dvmasize = 0; 985 mutex_exit(&is->is_lock); 986 if (err != 0) 987 printf("warning: %s: %" PRId64 " of DVMA space lost\n", 988 __func__, sgsize); 989 return (EFBIG); 990 } 991 992 993 /* 994 * Flush an individual dma segment, returns non-zero if the streaming buffers 995 * need flushing afterwards. 996 */ 997 static int 998 iommu_dvmamap_sync_range(struct strbuf_ctl *sb, vaddr_t va, bus_size_t len) 999 { 1000 vaddr_t vaend; 1001 struct iommu_state *is = sb->sb_is; 1002 1003 #ifdef DIAGNOSTIC 1004 if (va < is->is_dvmabase || va > is->is_dvmaend) 1005 panic("invalid va: %llx", (long long)va); 1006 #endif 1007 1008 if ((is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] & IOTTE_STREAM) == 0) { 1009 DPRINTF(IDB_SYNC, 1010 ("iommu_dvmamap_sync_range: attempting to flush " 1011 "non-streaming entry\n")); 1012 return (0); 1013 } 1014 1015 vaend = round_page(va + len) - 1; 1016 va = trunc_page(va); 1017 1018 #ifdef DIAGNOSTIC 1019 if (va < is->is_dvmabase || vaend > is->is_dvmaend) 1020 panic("invalid va range: %llx to %llx (%x to %x)", 1021 (long long)va, (long long)vaend, 1022 is->is_dvmabase, 1023 is->is_dvmaend); 1024 #endif 1025 1026 for ( ; va <= vaend; va += PAGE_SIZE) { 1027 DPRINTF(IDB_SYNC, 1028 ("iommu_dvmamap_sync_range: flushing va %p\n", 1029 (void *)(u_long)va)); 1030 iommu_strbuf_flush(sb, va); 1031 } 1032 1033 return (1); 1034 } 1035 1036 static void 1037 _iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 1038 bus_size_t len, int ops) 1039 { 1040 struct strbuf_ctl *sb = (struct strbuf_ctl *)map->_dm_cookie; 1041 bus_size_t count; 1042 int i, needsflush = 0; 1043 1044 if (!sb->sb_flush) 1045 return; 1046 1047 for (i = 0; i < map->dm_nsegs; i++) { 1048 if (offset < map->dm_segs[i].ds_len) 1049 break; 1050 offset -= map->dm_segs[i].ds_len; 1051 } 1052 1053 if (i == map->dm_nsegs) 1054 panic("%s: segment too short %llu", __func__, 1055 (unsigned long long)offset); 1056 1057 if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_POSTWRITE)) { 1058 /* Nothing to do */; 1059 } 1060 1061 if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)) { 1062 1063 for (; len > 0 && i < map->dm_nsegs; i++) { 1064 count = MIN(map->dm_segs[i].ds_len - offset, len); 1065 if (count > 0 && 1066 iommu_dvmamap_sync_range(sb, 1067 map->dm_segs[i].ds_addr + offset, count)) 1068 needsflush = 1; 1069 offset = 0; 1070 len -= count; 1071 } 1072 #ifdef DIAGNOSTIC 1073 if (i == map->dm_nsegs && len > 0) 1074 panic("%s: leftover %llu", __func__, 1075 (unsigned long long)len); 1076 #endif 1077 1078 if (needsflush) 1079 iommu_strbuf_flush_done(sb); 1080 } 1081 } 1082 1083 void 1084 iommu_dvmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 1085 bus_size_t len, int ops) 1086 { 1087 1088 /* If len is 0, then there is nothing to do */ 1089 if (len == 0) 1090 return; 1091 1092 if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) { 1093 /* Flush the CPU then the IOMMU */ 1094 bus_dmamap_sync(t->_parent, map, offset, len, ops); 1095 _iommu_dvmamap_sync(t, map, offset, len, ops); 1096 } 1097 if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)) { 1098 /* Flush the IOMMU then the CPU */ 1099 _iommu_dvmamap_sync(t, map, offset, len, ops); 1100 bus_dmamap_sync(t->_parent, map, offset, len, ops); 1101 } 1102 } 1103 1104 int 1105 iommu_dvmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, 1106 bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, 1107 int flags) 1108 { 1109 1110 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx " 1111 "segp %p flags %d\n", (unsigned long long)size, 1112 (unsigned long long)alignment, (unsigned long long)boundary, 1113 segs, flags)); 1114 return (bus_dmamem_alloc(t->_parent, size, alignment, boundary, 1115 segs, nsegs, rsegs, flags|BUS_DMA_DVMA)); 1116 } 1117 1118 void 1119 iommu_dvmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs) 1120 { 1121 1122 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n", 1123 segs, nsegs)); 1124 bus_dmamem_free(t->_parent, segs, nsegs); 1125 } 1126 1127 /* 1128 * Map the DVMA mappings into the kernel pmap. 1129 * Check the flags to see whether we're streaming or coherent. 1130 */ 1131 int 1132 iommu_dvmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, 1133 size_t size, void **kvap, int flags) 1134 { 1135 struct vm_page *pg; 1136 vaddr_t va; 1137 bus_addr_t addr; 1138 struct pglist *pglist; 1139 int cbit; 1140 const uvm_flag_t kmflags = 1141 (flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0; 1142 1143 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n", 1144 segs, nsegs, size)); 1145 1146 /* 1147 * Allocate some space in the kernel map, and then map these pages 1148 * into this space. 1149 */ 1150 size = round_page(size); 1151 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags); 1152 if (va == 0) 1153 return (ENOMEM); 1154 1155 *kvap = (void *)va; 1156 1157 /* 1158 * digest flags: 1159 */ 1160 cbit = 0; 1161 if (flags & BUS_DMA_COHERENT) /* Disable vcache */ 1162 cbit |= PMAP_NVC; 1163 if (flags & BUS_DMA_NOCACHE) /* side effects */ 1164 cbit |= PMAP_NC; 1165 1166 /* 1167 * Now take this and map it into the CPU. 1168 */ 1169 pglist = segs[0]._ds_mlist; 1170 TAILQ_FOREACH(pg, pglist, pageq.queue) { 1171 #ifdef DIAGNOSTIC 1172 if (size == 0) 1173 panic("iommu_dvmamem_map: size botch"); 1174 #endif 1175 addr = VM_PAGE_TO_PHYS(pg); 1176 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: " 1177 "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit)); 1178 pmap_kenter_pa(va, addr | cbit, 1179 VM_PROT_READ | VM_PROT_WRITE, 0); 1180 va += PAGE_SIZE; 1181 size -= PAGE_SIZE; 1182 } 1183 pmap_update(pmap_kernel()); 1184 return (0); 1185 } 1186 1187 /* 1188 * Unmap DVMA mappings from kernel 1189 */ 1190 void 1191 iommu_dvmamem_unmap(bus_dma_tag_t t, void *kva, size_t size) 1192 { 1193 1194 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n", 1195 kva, size)); 1196 1197 #ifdef DIAGNOSTIC 1198 if ((u_long)kva & PGOFSET) 1199 panic("iommu_dvmamem_unmap"); 1200 #endif 1201 1202 size = round_page(size); 1203 pmap_kremove((vaddr_t)kva, size); 1204 pmap_update(pmap_kernel()); 1205 uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY); 1206 } 1207