1 /* $NetBSD: iommu.c,v 1.65 2003/05/17 01:49:59 nakayama Exp $ */ 2 3 /* 4 * Copyright (c) 2001, 2002 Eduardo Horvath 5 * Copyright (c) 1999, 2000 Matthew R. Green 6 * All rights reserved. 7 * 8 * Redistribution and use in source and binary forms, with or without 9 * modification, are permitted provided that the following conditions 10 * are met: 11 * 1. Redistributions of source code must retain the above copyright 12 * notice, this list of conditions and the following disclaimer. 13 * 2. Redistributions in binary form must reproduce the above copyright 14 * notice, this list of conditions and the following disclaimer in the 15 * documentation and/or other materials provided with the distribution. 16 * 3. The name of the author may not be used to endorse or promote products 17 * derived from this software without specific prior written permission. 18 * 19 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 20 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 21 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 22 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 23 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 24 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 25 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 26 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 27 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 28 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 29 * SUCH DAMAGE. 30 */ 31 32 /* 33 * UltraSPARC IOMMU support; used by both the sbus and pci code. 34 */ 35 #include "opt_ddb.h" 36 37 #include <sys/param.h> 38 #include <sys/extent.h> 39 #include <sys/malloc.h> 40 #include <sys/systm.h> 41 #include <sys/device.h> 42 #include <sys/proc.h> 43 44 #include <uvm/uvm_extern.h> 45 46 #include <machine/bus.h> 47 #include <sparc64/sparc64/cache.h> 48 #include <sparc64/dev/iommureg.h> 49 #include <sparc64/dev/iommuvar.h> 50 51 #include <machine/autoconf.h> 52 #include <machine/cpu.h> 53 54 #ifdef DEBUG 55 #define IDB_BUSDMA 0x1 56 #define IDB_IOMMU 0x2 57 #define IDB_INFO 0x4 58 #define IDB_SYNC 0x8 59 int iommudebug = 0x0; 60 #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0) 61 #else 62 #define DPRINTF(l, s) 63 #endif 64 65 #define iommu_strbuf_flush(i, v) do { \ 66 if ((i)->sb_flush) \ 67 bus_space_write_8((i)->sb_is->is_bustag, (i)->sb_sb, \ 68 STRBUFREG(strbuf_pgflush), (v)); \ 69 } while (0) 70 71 static int iommu_strbuf_flush_done __P((struct strbuf_ctl *)); 72 73 /* 74 * initialise the UltraSPARC IOMMU (SBUS or PCI): 75 * - allocate and setup the iotsb. 76 * - enable the IOMMU 77 * - initialise the streaming buffers (if they exist) 78 * - create a private DVMA map. 79 */ 80 void 81 iommu_init(name, is, tsbsize, iovabase) 82 char *name; 83 struct iommu_state *is; 84 int tsbsize; 85 u_int32_t iovabase; 86 { 87 psize_t size; 88 vaddr_t va; 89 paddr_t pa; 90 struct vm_page *pg; 91 struct pglist pglist; 92 93 /* 94 * Setup the iommu. 95 * 96 * The sun4u iommu is part of the SBUS or PCI controller so we will 97 * deal with it here.. 98 * 99 * For sysio and psycho/psycho+ the IOMMU address space always ends at 100 * 0xffffe000, but the starting address depends on the size of the 101 * map. The map size is 1024 * 2 ^ is->is_tsbsize entries, where each 102 * entry is 8 bytes. The start of the map can be calculated by 103 * (0xffffe000 << (8 + is->is_tsbsize)). 104 * 105 * But sabre and hummingbird use a different scheme that seems to 106 * be hard-wired, so we read the start and size from the PROM and 107 * just use those values. 108 */ 109 is->is_cr = (tsbsize << 16) | IOMMUCR_EN; 110 is->is_tsbsize = tsbsize; 111 if (iovabase == -1) { 112 is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); 113 is->is_dvmaend = IOTSB_VEND; 114 } else { 115 is->is_dvmabase = iovabase; 116 is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize); 117 } 118 119 /* 120 * Allocate memory for I/O pagetables. They need to be physically 121 * contiguous. 122 */ 123 124 size = PAGE_SIZE << is->is_tsbsize; 125 if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1, 126 (paddr_t)PAGE_SIZE, (paddr_t)0, &pglist, 1, 0) != 0) 127 panic("iommu_init: no memory"); 128 129 va = uvm_km_valloc(kernel_map, size); 130 if (va == 0) 131 panic("iommu_init: no memory"); 132 is->is_tsb = (int64_t *)va; 133 134 is->is_ptsb = VM_PAGE_TO_PHYS(TAILQ_FIRST(&pglist)); 135 136 /* Map the pages */ 137 TAILQ_FOREACH(pg, &pglist, pageq) { 138 pa = VM_PAGE_TO_PHYS(pg); 139 pmap_kenter_pa(va, pa | PMAP_NVC, VM_PROT_READ | VM_PROT_WRITE); 140 va += PAGE_SIZE; 141 } 142 pmap_update(pmap_kernel()); 143 memset(is->is_tsb, 0, size); 144 145 #ifdef DEBUG 146 if (iommudebug & IDB_INFO) 147 { 148 /* Probe the iommu */ 149 150 printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n", 151 (u_long)bus_space_read_8(is->is_bustag, is->is_iommu, 152 offsetof (struct iommureg, iommu_cr)), 153 (u_long)bus_space_read_8(is->is_bustag, is->is_iommu, 154 offsetof (struct iommureg, iommu_tsb)), 155 (u_long)bus_space_read_8(is->is_bustag, is->is_iommu, 156 offsetof (struct iommureg, iommu_flush))); 157 printf("iommu cr=%llx tsb=%llx\n", 158 (unsigned long long)bus_space_read_8(is->is_bustag, 159 is->is_iommu, 160 offsetof (struct iommureg, iommu_cr)), 161 (unsigned long long)bus_space_read_8(is->is_bustag, 162 is->is_iommu, 163 offsetof (struct iommureg, iommu_tsb))); 164 printf("TSB base %p phys %llx\n", (void *)is->is_tsb, 165 (unsigned long long)is->is_ptsb); 166 delay(1000000); /* 1 s */ 167 } 168 #endif 169 170 /* 171 * now actually start up the IOMMU 172 */ 173 iommu_reset(is); 174 175 /* 176 * Now all the hardware's working we need to allocate a dvma map. 177 */ 178 printf("DVMA map: %x to %x\n", 179 (unsigned int)is->is_dvmabase, 180 (unsigned int)is->is_dvmaend); 181 printf("IOTSB: %llx to %llx\n", 182 (unsigned long long)is->is_ptsb, 183 (unsigned long long)(is->is_ptsb + size)); 184 is->is_dvmamap = extent_create(name, 185 is->is_dvmabase, is->is_dvmaend - PAGE_SIZE, 186 M_DEVBUF, 0, 0, EX_NOWAIT); 187 } 188 189 /* 190 * Streaming buffers don't exist on the UltraSPARC IIi; we should have 191 * detected that already and disabled them. If not, we will notice that 192 * they aren't there when the STRBUF_EN bit does not remain. 193 */ 194 void 195 iommu_reset(is) 196 struct iommu_state *is; 197 { 198 int i; 199 struct strbuf_ctl *sb; 200 201 /* Need to do 64-bit stores */ 202 bus_space_write_8(is->is_bustag, is->is_iommu, IOMMUREG(iommu_tsb), 203 is->is_ptsb); 204 205 /* Enable IOMMU in diagnostic mode */ 206 bus_space_write_8(is->is_bustag, is->is_iommu, IOMMUREG(iommu_cr), 207 is->is_cr|IOMMUCR_DE); 208 209 for (i = 0; i < 2; i++) { 210 if ((sb = is->is_sb[i])) { 211 212 /* Enable diagnostics mode? */ 213 bus_space_write_8(is->is_bustag, is->is_sb[i]->sb_sb, 214 STRBUFREG(strbuf_ctl), STRBUF_EN); 215 216 /* No streaming buffers? Disable them */ 217 if (bus_space_read_8(is->is_bustag, 218 is->is_sb[i]->sb_sb, 219 STRBUFREG(strbuf_ctl)) == 0) { 220 is->is_sb[i]->sb_flush = NULL; 221 } else { 222 223 /* 224 * locate the pa of the flush buffer. 225 */ 226 (void)pmap_extract(pmap_kernel(), 227 (vaddr_t)is->is_sb[i]->sb_flush, 228 &is->is_sb[i]->sb_flushpa); 229 } 230 } 231 } 232 } 233 234 /* 235 * Here are the iommu control routines. 236 */ 237 void 238 iommu_enter(sb, va, pa, flags) 239 struct strbuf_ctl *sb; 240 vaddr_t va; 241 int64_t pa; 242 int flags; 243 { 244 struct iommu_state *is = sb->sb_is; 245 int strbuf = (flags & BUS_DMA_STREAMING); 246 int64_t tte; 247 248 #ifdef DIAGNOSTIC 249 if (va < is->is_dvmabase || va > is->is_dvmaend) 250 panic("iommu_enter: va %#lx not in DVMA space", va); 251 #endif 252 253 /* Is the streamcache flush really needed? */ 254 if (sb->sb_flush) { 255 iommu_strbuf_flush(sb, va); 256 iommu_strbuf_flush_done(sb); 257 } else 258 /* If we can't flush the strbuf don't enable it. */ 259 strbuf = 0; 260 261 tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE), 262 !(flags & BUS_DMA_NOCACHE), (strbuf)); 263 #ifdef DEBUG 264 tte |= (flags & 0xff000LL)<<(4*8); 265 #endif 266 267 DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n", 268 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va)); 269 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte; 270 bus_space_write_8(is->is_bustag, is->is_iommu, 271 IOMMUREG(iommu_flush), va); 272 DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n", 273 va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize), 274 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 275 (u_long)tte)); 276 } 277 278 /* 279 * Find the value of a DVMA address (debug routine). 280 */ 281 paddr_t 282 iommu_extract(is, dva) 283 struct iommu_state *is; 284 vaddr_t dva; 285 { 286 int64_t tte = 0; 287 288 if (dva >= is->is_dvmabase && dva < is->is_dvmaend) 289 tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)]; 290 291 if ((tte & IOTTE_V) == 0) 292 return ((paddr_t)-1L); 293 return (tte & IOTTE_PAMASK); 294 } 295 296 /* 297 * iommu_remove: removes mappings created by iommu_enter 298 * 299 * Only demap from IOMMU if flag is set. 300 * 301 * XXX: this function needs better internal error checking. 302 */ 303 void 304 iommu_remove(is, va, len) 305 struct iommu_state *is; 306 vaddr_t va; 307 size_t len; 308 { 309 310 #ifdef DIAGNOSTIC 311 if (va < is->is_dvmabase || va > is->is_dvmaend) 312 panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va); 313 if ((long)(va + len) < (long)va) 314 panic("iommu_remove: va 0x%lx + len 0x%lx wraps", 315 (long) va, (long) len); 316 if (len & ~0xfffffff) 317 panic("iommu_remove: rediculous len 0x%lx", (u_long)len); 318 #endif 319 320 va = trunc_page(va); 321 DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n", 322 va, (u_long)IOTSBSLOT(va, is->is_tsbsize), 323 &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)])); 324 while (len > 0) { 325 DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d " 326 "for va %p size %lx\n", 327 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va, 328 (u_long)len)); 329 if (len <= PAGE_SIZE) 330 len = 0; 331 else 332 len -= PAGE_SIZE; 333 334 /* XXX Zero-ing the entry would not require RMW */ 335 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] &= ~IOTTE_V; 336 bus_space_write_8(is->is_bustag, is->is_iommu, 337 IOMMUREG(iommu_flush), va); 338 va += PAGE_SIZE; 339 } 340 } 341 342 static int 343 iommu_strbuf_flush_done(sb) 344 struct strbuf_ctl *sb; 345 { 346 struct iommu_state *is = sb->sb_is; 347 struct timeval cur, flushtimeout; 348 349 #define BUMPTIME(t, usec) { \ 350 register volatile struct timeval *tp = (t); \ 351 register long us; \ 352 \ 353 tp->tv_usec = us = tp->tv_usec + (usec); \ 354 if (us >= 1000000) { \ 355 tp->tv_usec = us - 1000000; \ 356 tp->tv_sec++; \ 357 } \ 358 } 359 360 if (!sb->sb_flush) 361 return (0); 362 363 /* 364 * Streaming buffer flushes: 365 * 366 * 1 Tell strbuf to flush by storing va to strbuf_pgflush. If 367 * we're not on a cache line boundary (64-bits): 368 * 2 Store 0 in flag 369 * 3 Store pointer to flag in flushsync 370 * 4 wait till flushsync becomes 0x1 371 * 372 * If it takes more than .5 sec, something 373 * went wrong. 374 */ 375 376 *sb->sb_flush = 0; 377 bus_space_write_8(is->is_bustag, sb->sb_sb, 378 STRBUFREG(strbuf_flushsync), sb->sb_flushpa); 379 380 microtime(&flushtimeout); 381 cur = flushtimeout; 382 BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */ 383 384 DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flush = %lx " 385 "at va = %lx pa = %lx now=%lx:%lx until = %lx:%lx\n", 386 (long)*sb->sb_flush, (long)sb->sb_flush, (long)sb->sb_flushpa, 387 cur.tv_sec, cur.tv_usec, 388 flushtimeout.tv_sec, flushtimeout.tv_usec)); 389 390 /* Bypass non-coherent D$ */ 391 while ((!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) && 392 timercmp(&cur, &flushtimeout, <=)) 393 microtime(&cur); 394 395 #ifdef DIAGNOSTIC 396 if (!ldxa(sb->sb_flushpa, ASI_PHYS_CACHED)) { 397 printf("iommu_strbuf_flush_done: flush timeout %p, at %p\n", 398 (void *)(u_long)*sb->sb_flush, 399 (void *)(u_long)sb->sb_flushpa); /* panic? */ 400 #ifdef DDB 401 Debugger(); 402 #endif 403 } 404 #endif 405 DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flushed\n")); 406 return (*sb->sb_flush); 407 } 408 409 /* 410 * IOMMU DVMA operations, common to SBUS and PCI. 411 */ 412 int 413 iommu_dvmamap_load(t, sb, map, buf, buflen, p, flags) 414 bus_dma_tag_t t; 415 struct strbuf_ctl *sb; 416 bus_dmamap_t map; 417 void *buf; 418 bus_size_t buflen; 419 struct proc *p; 420 int flags; 421 { 422 struct iommu_state *is = sb->sb_is; 423 int s; 424 int err; 425 bus_size_t sgsize; 426 paddr_t curaddr; 427 u_long dvmaddr, sgstart, sgend; 428 bus_size_t align, boundary; 429 vaddr_t vaddr = (vaddr_t)buf; 430 int seg; 431 struct pmap *pmap; 432 433 if (map->dm_nsegs) { 434 /* Already in use?? */ 435 #ifdef DIAGNOSTIC 436 printf("iommu_dvmamap_load: map still in use\n"); 437 #endif 438 bus_dmamap_unload(t, map); 439 } 440 441 /* 442 * Make sure that on error condition we return "no valid mappings". 443 */ 444 map->dm_nsegs = 0; 445 if (buflen > map->_dm_size) { 446 DPRINTF(IDB_BUSDMA, 447 ("iommu_dvmamap_load(): error %d > %d -- " 448 "map size exceeded!\n", (int)buflen, (int)map->_dm_size)); 449 return (EINVAL); 450 } 451 452 sgsize = round_page(buflen + ((int)vaddr & PGOFSET)); 453 454 /* 455 * A boundary presented to bus_dmamem_alloc() takes precedence 456 * over boundary in the map. 457 */ 458 if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0) 459 boundary = map->_dm_boundary; 460 align = max(map->dm_segs[0]._ds_align, PAGE_SIZE); 461 462 /* 463 * If our segment size is larger than the boundary we need to 464 * split the transfer up int little pieces ourselves. 465 */ 466 s = splhigh(); 467 err = extent_alloc(is->is_dvmamap, sgsize, align, 468 (sgsize > boundary) ? 0 : boundary, 469 EX_NOWAIT|EX_BOUNDZERO, &dvmaddr); 470 splx(s); 471 472 #ifdef DEBUG 473 if (err || (dvmaddr == (u_long)-1)) 474 { 475 printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n", 476 (int)sgsize, flags); 477 #ifdef DDB 478 Debugger(); 479 #endif 480 } 481 #endif 482 if (err != 0) 483 return (err); 484 485 if (dvmaddr == (u_long)-1) 486 return (ENOMEM); 487 488 /* Set the active DVMA map */ 489 map->_dm_dvmastart = dvmaddr; 490 map->_dm_dvmasize = sgsize; 491 492 /* 493 * Now split the DVMA range into segments, not crossing 494 * the boundary. 495 */ 496 seg = 0; 497 sgstart = dvmaddr + (vaddr & PGOFSET); 498 sgend = sgstart + buflen - 1; 499 map->dm_segs[seg].ds_addr = sgstart; 500 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: boundary %lx boundary-1 %lx " 501 "~(boundary-1) %lx\n", (long)boundary, (long)(boundary-1), (long)~(boundary-1))); 502 while ((sgstart & ~(boundary - 1)) != (sgend & ~(boundary - 1))) { 503 /* Oops. We crossed a boundary. Split the xfer. */ 504 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 505 "seg %d start %lx size %lx\n", seg, 506 (long)map->dm_segs[seg].ds_addr, 507 (long)map->dm_segs[seg].ds_len)); 508 map->dm_segs[seg].ds_len = 509 boundary - (sgstart & (boundary - 1)); 510 if (++seg >= map->_dm_segcnt) { 511 /* Too many segments. Fail the operation. */ 512 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 513 "too many segments %d\n", seg)); 514 s = splhigh(); 515 /* How can this fail? And if it does what can we do? */ 516 err = extent_free(is->is_dvmamap, 517 dvmaddr, sgsize, EX_NOWAIT); 518 map->_dm_dvmastart = 0; 519 map->_dm_dvmasize = 0; 520 splx(s); 521 return (E2BIG); 522 } 523 sgstart = roundup(sgstart, boundary); 524 map->dm_segs[seg].ds_addr = sgstart; 525 } 526 map->dm_segs[seg].ds_len = sgend - sgstart + 1; 527 DPRINTF(IDB_INFO, ("iommu_dvmamap_load: " 528 "seg %d start %lx size %lx\n", seg, 529 (long)map->dm_segs[seg].ds_addr, (long)map->dm_segs[seg].ds_len)); 530 map->dm_nsegs = seg+1; 531 map->dm_mapsize = buflen; 532 533 if (p != NULL) 534 pmap = p->p_vmspace->vm_map.pmap; 535 else 536 pmap = pmap_kernel(); 537 538 for (; buflen > 0; ) { 539 540 /* 541 * Get the physical address for this page. 542 */ 543 if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) { 544 bus_dmamap_unload(t, map); 545 return (-1); 546 } 547 548 /* 549 * Compute the segment size, and adjust counts. 550 */ 551 sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET); 552 if (buflen < sgsize) 553 sgsize = buflen; 554 555 DPRINTF(IDB_BUSDMA, 556 ("iommu_dvmamap_load: map %p loading va %p " 557 "dva %lx at pa %lx\n", 558 map, (void *)vaddr, (long)dvmaddr, 559 (long)(curaddr & ~(PAGE_SIZE-1)))); 560 iommu_enter(sb, trunc_page(dvmaddr), trunc_page(curaddr), 561 flags|0x4000); 562 563 dvmaddr += PAGE_SIZE; 564 vaddr += sgsize; 565 buflen -= sgsize; 566 } 567 #ifdef DIAGNOSTIC 568 for (seg = 0; seg < map->dm_nsegs; seg++) { 569 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 570 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 571 printf("seg %d dvmaddr %lx out of range %x - %x\n", 572 seg, (long)map->dm_segs[seg].ds_addr, 573 is->is_dvmabase, is->is_dvmaend); 574 #ifdef DDB 575 Debugger(); 576 #endif 577 } 578 } 579 #endif 580 return (0); 581 } 582 583 584 void 585 iommu_dvmamap_unload(t, sb, map) 586 bus_dma_tag_t t; 587 struct strbuf_ctl *sb; 588 bus_dmamap_t map; 589 { 590 struct iommu_state *is = sb->sb_is; 591 int error, s; 592 bus_size_t sgsize; 593 594 /* Flush the iommu */ 595 #ifdef DEBUG 596 if (!map->_dm_dvmastart) { 597 printf("iommu_dvmamap_unload: No dvmastart is zero\n"); 598 #ifdef DDB 599 Debugger(); 600 #endif 601 } 602 #endif 603 iommu_remove(is, map->_dm_dvmastart, map->_dm_dvmasize); 604 605 /* Flush the caches */ 606 bus_dmamap_unload(t->_parent, map); 607 608 /* Mark the mappings as invalid. */ 609 map->dm_mapsize = 0; 610 map->dm_nsegs = 0; 611 612 s = splhigh(); 613 error = extent_free(is->is_dvmamap, map->_dm_dvmastart, 614 map->_dm_dvmasize, EX_NOWAIT); 615 map->_dm_dvmastart = 0; 616 map->_dm_dvmasize = 0; 617 splx(s); 618 if (error != 0) 619 printf("warning: %qd of DVMA space lost\n", (long long)sgsize); 620 621 /* Clear the map */ 622 } 623 624 625 int 626 iommu_dvmamap_load_raw(t, sb, map, segs, nsegs, flags, size) 627 bus_dma_tag_t t; 628 struct strbuf_ctl *sb; 629 bus_dmamap_t map; 630 bus_dma_segment_t *segs; 631 int nsegs; 632 int flags; 633 bus_size_t size; 634 { 635 struct iommu_state *is = sb->sb_is; 636 struct vm_page *pg; 637 int i, j, s; 638 int left; 639 int err; 640 bus_size_t sgsize; 641 paddr_t pa; 642 bus_size_t boundary, align; 643 u_long dvmaddr, sgstart, sgend; 644 struct pglist *pglist; 645 int pagesz = PAGE_SIZE; 646 int npg = 0; /* DEBUG */ 647 648 if (map->dm_nsegs) { 649 /* Already in use?? */ 650 #ifdef DIAGNOSTIC 651 printf("iommu_dvmamap_load_raw: map still in use\n"); 652 #endif 653 bus_dmamap_unload(t, map); 654 } 655 656 /* 657 * A boundary presented to bus_dmamem_alloc() takes precedence 658 * over boundary in the map. 659 */ 660 if ((boundary = segs[0]._ds_boundary) == 0) 661 boundary = map->_dm_boundary; 662 663 align = max(segs[0]._ds_align, pagesz); 664 665 /* 666 * Make sure that on error condition we return "no valid mappings". 667 */ 668 map->dm_nsegs = 0; 669 /* Count up the total number of pages we need */ 670 pa = segs[0].ds_addr; 671 sgsize = 0; 672 left = size; 673 for (i = 0; left && i < nsegs; i++) { 674 if (round_page(pa) != round_page(segs[i].ds_addr)) 675 sgsize = round_page(sgsize); 676 sgsize += min(left, segs[i].ds_len); 677 left -= segs[i].ds_len; 678 pa = segs[i].ds_addr + segs[i].ds_len; 679 } 680 sgsize = round_page(sgsize); 681 682 s = splhigh(); 683 /* 684 * If our segment size is larger than the boundary we need to 685 * split the transfer up into little pieces ourselves. 686 */ 687 err = extent_alloc(is->is_dvmamap, sgsize, align, 688 (sgsize > boundary) ? 0 : boundary, 689 ((flags & BUS_DMA_NOWAIT) == 0 ? EX_WAITOK : EX_NOWAIT) | 690 EX_BOUNDZERO, &dvmaddr); 691 splx(s); 692 693 if (err != 0) 694 return (err); 695 696 #ifdef DEBUG 697 if (dvmaddr == (u_long)-1) 698 { 699 printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n", 700 (int)sgsize, flags); 701 #ifdef DDB 702 Debugger(); 703 #endif 704 } 705 #endif 706 if (dvmaddr == (u_long)-1) 707 return (ENOMEM); 708 709 /* Set the active DVMA map */ 710 map->_dm_dvmastart = dvmaddr; 711 map->_dm_dvmasize = sgsize; 712 713 if ((pglist = segs[0]._ds_mlist) == NULL) { 714 u_long prev_va = NULL; 715 paddr_t prev_pa = 0; 716 int end = 0, offset; 717 718 /* 719 * This segs is made up of individual physical 720 * segments, probably by _bus_dmamap_load_uio() or 721 * _bus_dmamap_load_mbuf(). Ignore the mlist and 722 * load each one individually. 723 */ 724 map->dm_mapsize = size; 725 726 j = 0; 727 for (i = 0; i < nsegs ; i++) { 728 729 pa = segs[i].ds_addr; 730 offset = (pa & PGOFSET); 731 pa = trunc_page(pa); 732 dvmaddr = trunc_page(dvmaddr); 733 left = min(size, segs[i].ds_len); 734 735 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: converting " 736 "physseg %d start %lx size %lx\n", i, 737 (long)segs[i].ds_addr, (long)segs[i].ds_len)); 738 739 if ((pa == prev_pa) && 740 ((offset != 0) || (end != offset))) { 741 /* We can re-use this mapping */ 742 dvmaddr = prev_va; 743 } 744 745 sgstart = dvmaddr + offset; 746 sgend = sgstart + left - 1; 747 748 /* Are the segments virtually adjacent? */ 749 if ((j > 0) && (end == offset) && 750 ((offset == 0) || (pa == prev_pa))) { 751 /* Just append to the previous segment. */ 752 map->dm_segs[--j].ds_len += left; 753 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 754 "appending seg %d start %lx size %lx\n", j, 755 (long)map->dm_segs[j].ds_addr, 756 (long)map->dm_segs[j].ds_len)); 757 } else { 758 if (j >= map->_dm_segcnt) { 759 iommu_dvmamap_unload(t, sb, map); 760 return (E2BIG); 761 } 762 map->dm_segs[j].ds_addr = sgstart; 763 map->dm_segs[j].ds_len = left; 764 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 765 "seg %d start %lx size %lx\n", j, 766 (long)map->dm_segs[j].ds_addr, 767 (long)map->dm_segs[j].ds_len)); 768 } 769 end = (offset + left) & PGOFSET; 770 771 /* Check for boundary issues */ 772 while ((sgstart & ~(boundary - 1)) != 773 (sgend & ~(boundary - 1))) { 774 /* Need a new segment. */ 775 map->dm_segs[j].ds_len = 776 boundary - (sgstart & (boundary - 1)); 777 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 778 "seg %d start %lx size %lx\n", j, 779 (long)map->dm_segs[j].ds_addr, 780 (long)map->dm_segs[j].ds_len)); 781 if (++j >= map->_dm_segcnt) { 782 iommu_dvmamap_unload(t, sb, map); 783 return (E2BIG); 784 } 785 sgstart = roundup(sgstart, boundary); 786 map->dm_segs[j].ds_addr = sgstart; 787 map->dm_segs[j].ds_len = sgend - sgstart + 1; 788 } 789 790 if (sgsize == 0) 791 panic("iommu_dmamap_load_raw: size botch"); 792 793 /* Now map a series of pages. */ 794 while (dvmaddr <= sgend) { 795 DPRINTF(IDB_BUSDMA, 796 ("iommu_dvmamap_load_raw: map %p " 797 "loading va %lx at pa %lx\n", 798 map, (long)dvmaddr, 799 (long)(pa))); 800 /* Enter it if we haven't before. */ 801 if (prev_va != dvmaddr) 802 iommu_enter(sb, prev_va = dvmaddr, 803 prev_pa = pa, 804 flags | (++npg << 12)); 805 dvmaddr += pagesz; 806 pa += pagesz; 807 } 808 809 size -= left; 810 ++j; 811 } 812 813 map->dm_nsegs = j; 814 #ifdef DIAGNOSTIC 815 { int seg; 816 for (seg = 0; seg < map->dm_nsegs; seg++) { 817 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 818 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 819 printf("seg %d dvmaddr %lx out of range %x - %x\n", 820 seg, (long)map->dm_segs[seg].ds_addr, 821 is->is_dvmabase, is->is_dvmaend); 822 #ifdef DDB 823 Debugger(); 824 #endif 825 } 826 } 827 } 828 #endif 829 return (0); 830 } 831 832 /* 833 * This was allocated with bus_dmamem_alloc. 834 * The pages are on a `pglist'. 835 */ 836 map->dm_mapsize = size; 837 i = 0; 838 sgstart = dvmaddr; 839 sgend = sgstart + size - 1; 840 map->dm_segs[i].ds_addr = sgstart; 841 while ((sgstart & ~(boundary - 1)) != (sgend & ~(boundary - 1))) { 842 /* Oops. We crossed a boundary. Split the xfer. */ 843 map->dm_segs[i].ds_len = boundary - (sgstart & (boundary - 1)); 844 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 845 "seg %d start %lx size %lx\n", i, 846 (long)map->dm_segs[i].ds_addr, 847 (long)map->dm_segs[i].ds_len)); 848 if (++i >= map->_dm_segcnt) { 849 /* Too many segments. Fail the operation. */ 850 s = splhigh(); 851 /* How can this fail? And if it does what can we do? */ 852 err = extent_free(is->is_dvmamap, 853 dvmaddr, sgsize, EX_NOWAIT); 854 map->_dm_dvmastart = 0; 855 map->_dm_dvmasize = 0; 856 splx(s); 857 return (E2BIG); 858 } 859 sgstart = roundup(sgstart, boundary); 860 map->dm_segs[i].ds_addr = sgstart; 861 } 862 DPRINTF(IDB_INFO, ("iommu_dvmamap_load_raw: " 863 "seg %d start %lx size %lx\n", i, 864 (long)map->dm_segs[i].ds_addr, (long)map->dm_segs[i].ds_len)); 865 map->dm_segs[i].ds_len = sgend - sgstart + 1; 866 867 TAILQ_FOREACH(pg, pglist, pageq) { 868 if (sgsize == 0) 869 panic("iommu_dmamap_load_raw: size botch"); 870 pa = VM_PAGE_TO_PHYS(pg); 871 872 DPRINTF(IDB_BUSDMA, 873 ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n", 874 map, (long)dvmaddr, (long)(pa))); 875 iommu_enter(sb, dvmaddr, pa, flags|0x8000); 876 877 dvmaddr += pagesz; 878 sgsize -= pagesz; 879 } 880 map->dm_mapsize = size; 881 map->dm_nsegs = i+1; 882 #ifdef DIAGNOSTIC 883 { int seg; 884 for (seg = 0; seg < map->dm_nsegs; seg++) { 885 if (map->dm_segs[seg].ds_addr < is->is_dvmabase || 886 map->dm_segs[seg].ds_addr > is->is_dvmaend) { 887 printf("seg %d dvmaddr %lx out of range %x - %x\n", 888 seg, (long)map->dm_segs[seg].ds_addr, 889 is->is_dvmabase, is->is_dvmaend); 890 #ifdef DDB 891 Debugger(); 892 #endif 893 } 894 } 895 } 896 #endif 897 return (0); 898 } 899 900 void 901 iommu_dvmamap_sync(t, sb, map, offset, len, ops) 902 bus_dma_tag_t t; 903 struct strbuf_ctl *sb; 904 bus_dmamap_t map; 905 bus_addr_t offset; 906 bus_size_t len; 907 int ops; 908 { 909 struct iommu_state *is = sb->sb_is; 910 vaddr_t va = map->dm_segs[0].ds_addr + offset; 911 int64_t tte; 912 vaddr_t vaend; 913 914 if (!sb->sb_flush) 915 return; 916 917 /* 918 * We only support one DMA segment; supporting more makes this code 919 * too unwieldy. 920 */ 921 if (map->dm_nsegs > 1) 922 panic("iommu_dvmamap_sync: %d segments", map->dm_nsegs); 923 924 DPRINTF(IDB_SYNC, 925 ("iommu_dvmamap_sync: syncing va %p len %lu " 926 "ops 0x%x\n", (void *)(u_long)va, (u_long)len, ops)); 927 928 if (ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_POSTWRITE)) { 929 /* Nothing to do */; 930 } 931 932 if (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE)) { 933 #ifdef DIAGNOSTIC 934 if (va < is->is_dvmabase || va >= is->is_dvmaend) 935 panic("iommu_dvmamap_sync: invalid dva %lx", va); 936 #endif 937 tte = is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]; 938 939 /* if we have a streaming buffer, flush it here first */ 940 if ((tte & IOTTE_STREAM) && sb->sb_flush) { 941 vaend = (va + len + PGOFSET) & ~PGOFSET; 942 943 for (va &= ~PGOFSET; va <= vaend; va += PAGE_SIZE) { 944 DPRINTF(IDB_BUSDMA, 945 ("iommu_dvmamap_sync: flushing va %p\n", 946 (void *)(u_long)va)); 947 iommu_strbuf_flush(sb, va); 948 } 949 950 iommu_strbuf_flush_done(sb); 951 } 952 } 953 } 954 955 int 956 iommu_dvmamem_alloc(t, sb, size, alignment, boundary, segs, nsegs, rsegs, flags) 957 bus_dma_tag_t t; 958 struct strbuf_ctl *sb; 959 bus_size_t size, alignment, boundary; 960 bus_dma_segment_t *segs; 961 int nsegs; 962 int *rsegs; 963 int flags; 964 { 965 966 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx " 967 "segp %p flags %d\n", (unsigned long long)size, 968 (unsigned long long)alignment, (unsigned long long)boundary, 969 segs, flags)); 970 return (bus_dmamem_alloc(t->_parent, size, alignment, boundary, 971 segs, nsegs, rsegs, flags|BUS_DMA_DVMA)); 972 } 973 974 void 975 iommu_dvmamem_free(t, sb, segs, nsegs) 976 bus_dma_tag_t t; 977 struct strbuf_ctl *sb; 978 bus_dma_segment_t *segs; 979 int nsegs; 980 { 981 982 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n", 983 segs, nsegs)); 984 bus_dmamem_free(t->_parent, segs, nsegs); 985 } 986 987 /* 988 * Map the DVMA mappings into the kernel pmap. 989 * Check the flags to see whether we're streaming or coherent. 990 */ 991 int 992 iommu_dvmamem_map(t, sb, segs, nsegs, size, kvap, flags) 993 bus_dma_tag_t t; 994 struct strbuf_ctl *sb; 995 bus_dma_segment_t *segs; 996 int nsegs; 997 size_t size; 998 caddr_t *kvap; 999 int flags; 1000 { 1001 struct vm_page *pg; 1002 vaddr_t va; 1003 bus_addr_t addr; 1004 struct pglist *pglist; 1005 int cbit; 1006 1007 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n", 1008 segs, nsegs, size)); 1009 1010 /* 1011 * Allocate some space in the kernel map, and then map these pages 1012 * into this space. 1013 */ 1014 size = round_page(size); 1015 va = uvm_km_valloc(kernel_map, size); 1016 if (va == 0) 1017 return (ENOMEM); 1018 1019 *kvap = (caddr_t)va; 1020 1021 /* 1022 * digest flags: 1023 */ 1024 cbit = 0; 1025 if (flags & BUS_DMA_COHERENT) /* Disable vcache */ 1026 cbit |= PMAP_NVC; 1027 if (flags & BUS_DMA_NOCACHE) /* sideffects */ 1028 cbit |= PMAP_NC; 1029 1030 /* 1031 * Now take this and map it into the CPU. 1032 */ 1033 pglist = segs[0]._ds_mlist; 1034 TAILQ_FOREACH(pg, pglist, pageq) { 1035 #ifdef DIAGNOSTIC 1036 if (size == 0) 1037 panic("iommu_dvmamem_map: size botch"); 1038 #endif 1039 addr = VM_PAGE_TO_PHYS(pg); 1040 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: " 1041 "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit)); 1042 pmap_kenter_pa(va, addr | cbit, VM_PROT_READ | VM_PROT_WRITE); 1043 va += PAGE_SIZE; 1044 size -= PAGE_SIZE; 1045 } 1046 pmap_update(pmap_kernel()); 1047 return (0); 1048 } 1049 1050 /* 1051 * Unmap DVMA mappings from kernel 1052 */ 1053 void 1054 iommu_dvmamem_unmap(t, sb, kva, size) 1055 bus_dma_tag_t t; 1056 struct strbuf_ctl *sb; 1057 caddr_t kva; 1058 size_t size; 1059 { 1060 1061 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n", 1062 kva, size)); 1063 1064 #ifdef DIAGNOSTIC 1065 if ((u_long)kva & PGOFSET) 1066 panic("iommu_dvmamem_unmap"); 1067 #endif 1068 1069 size = round_page(size); 1070 pmap_kremove((vaddr_t)kva, size); 1071 pmap_update(pmap_kernel()); 1072 uvm_km_free(kernel_map, (vaddr_t)kva, size); 1073 } 1074