1 /* $OpenBSD: iommu.c,v 1.57 2009/04/14 16:01:04 oga Exp $ */ 2 /* $NetBSD: iommu.c,v 1.47 2002/02/08 20:03:45 eeh Exp $ */ 3 4 /* 5 * Copyright (c) 2003 Henric Jungheim 6 * Copyright (c) 2001, 2002 Eduardo Horvath 7 * Copyright (c) 1999, 2000 Matthew R. Green 8 * All rights reserved. 9 * 10 * Redistribution and use in source and binary forms, with or without 11 * modification, are permitted provided that the following conditions 12 * are met: 13 * 1. Redistributions of source code must retain the above copyright 14 * notice, this list of conditions and the following disclaimer. 15 * 2. Redistributions in binary form must reproduce the above copyright 16 * notice, this list of conditions and the following disclaimer in the 17 * documentation and/or other materials provided with the distribution. 18 * 3. The name of the author may not be used to endorse or promote products 19 * derived from this software without specific prior written permission. 20 * 21 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 22 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 23 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 24 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 25 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 26 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 27 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 28 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 29 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 30 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 31 * SUCH DAMAGE. 32 */ 33 34 /* 35 * UltraSPARC IOMMU support; used by both the sbus and pci code. 36 */ 37 #include <sys/param.h> 38 #include <sys/extent.h> 39 #include <sys/malloc.h> 40 #include <sys/systm.h> 41 #include <sys/device.h> 42 #include <sys/mbuf.h> 43 44 #include <uvm/uvm_extern.h> 45 46 #include <machine/bus.h> 47 #include <sparc64/sparc64/cache.h> 48 #include <sparc64/dev/iommureg.h> 49 #include <sparc64/dev/iommuvar.h> 50 51 #include <machine/autoconf.h> 52 #include <machine/cpu.h> 53 54 #ifdef DDB 55 #include <machine/db_machdep.h> 56 #include <ddb/db_sym.h> 57 #include <ddb/db_extern.h> 58 #endif 59 60 #ifdef DEBUG 61 #define IDB_BUSDMA 0x1 62 #define IDB_IOMMU 0x2 63 #define IDB_INFO 0x4 64 #define IDB_SYNC 0x8 65 #define IDB_XXX 0x10 66 #define IDB_PRINT_MAP 0x20 67 #define IDB_BREAK 0x40 68 int iommudebug = IDB_INFO; 69 #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0) 70 #else 71 #define DPRINTF(l, s) 72 #endif 73 74 void iommu_enter(struct iommu_state *, struct strbuf_ctl *, vaddr_t, paddr_t, 75 int); 76 void iommu_remove(struct iommu_state *, struct strbuf_ctl *, vaddr_t); 77 int iommu_dvmamap_sync_range(struct strbuf_ctl*, vaddr_t, bus_size_t); 78 int iommu_strbuf_flush_done(struct iommu_map_state *); 79 int iommu_dvmamap_load_seg(bus_dma_tag_t, struct iommu_state *, 80 bus_dmamap_t, bus_dma_segment_t *, int, int, bus_size_t, bus_size_t); 81 int iommu_dvmamap_load_mlist(bus_dma_tag_t, struct iommu_state *, 82 bus_dmamap_t, struct pglist *, int, bus_size_t, bus_size_t); 83 int iommu_dvmamap_validate_map(bus_dma_tag_t, struct iommu_state *, 84 bus_dmamap_t); 85 void iommu_dvmamap_print_map(bus_dma_tag_t, struct iommu_state *, 86 bus_dmamap_t); 87 int iommu_dvmamap_append_range(bus_dma_tag_t, bus_dmamap_t, paddr_t, 88 bus_size_t, int, bus_size_t); 89 int64_t iommu_tsb_entry(struct iommu_state *, vaddr_t); 90 void strbuf_reset(struct strbuf_ctl *); 91 int iommu_iomap_insert_page(struct iommu_map_state *, paddr_t); 92 vaddr_t iommu_iomap_translate(struct iommu_map_state *, paddr_t); 93 void iommu_iomap_load_map(struct iommu_state *, struct iommu_map_state *, 94 vaddr_t, int); 95 void iommu_iomap_unload_map(struct iommu_state *, struct iommu_map_state *); 96 struct iommu_map_state *iommu_iomap_create(int); 97 void iommu_iomap_destroy(struct iommu_map_state *); 98 void iommu_iomap_clear_pages(struct iommu_map_state *); 99 void _iommu_dvmamap_sync(bus_dma_tag_t, bus_dma_tag_t, bus_dmamap_t, 100 bus_addr_t, bus_size_t, int); 101 102 /* 103 * Initiate an STC entry flush. 104 */ 105 static inline void 106 iommu_strbuf_flush(struct strbuf_ctl *sb, vaddr_t va) 107 { 108 #ifdef DEBUG 109 if (sb->sb_flush == NULL) { 110 printf("iommu_strbuf_flush: attempting to flush w/o STC\n"); 111 return; 112 } 113 #endif 114 115 bus_space_write_8(sb->sb_bustag, sb->sb_sb, 116 STRBUFREG(strbuf_pgflush), va); 117 } 118 119 /* 120 * initialise the UltraSPARC IOMMU (SBus or PCI): 121 * - allocate and setup the iotsb. 122 * - enable the IOMMU 123 * - initialise the streaming buffers (if they exist) 124 * - create a private DVMA map. 125 */ 126 void 127 iommu_init(char *name, struct iommu_state *is, int tsbsize, u_int32_t iovabase) 128 { 129 psize_t size; 130 vaddr_t va; 131 paddr_t pa; 132 struct vm_page *m; 133 struct pglist mlist; 134 135 /* 136 * Setup the iommu. 137 * 138 * The sun4u iommu is part of the SBus or PCI controller so we will 139 * deal with it here.. 140 * 141 * For sysio and psycho/psycho+ the IOMMU address space always ends at 142 * 0xffffe000, but the starting address depends on the size of the 143 * map. The map size is 1024 * 2 ^ is->is_tsbsize entries, where each 144 * entry is 8 bytes. The start of the map can be calculated by 145 * (0xffffe000 << (8 + is->is_tsbsize)). 146 * 147 * But sabre and hummingbird use a different scheme that seems to 148 * be hard-wired, so we read the start and size from the PROM and 149 * just use those values. 150 */ 151 is->is_cr = IOMMUCR_EN; 152 is->is_tsbsize = tsbsize; 153 if (iovabase == (u_int32_t)-1) { 154 is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); 155 is->is_dvmaend = IOTSB_VEND; 156 } else { 157 is->is_dvmabase = iovabase; 158 is->is_dvmaend = iovabase + IOTSB_VSIZE(tsbsize) - 1; 159 } 160 161 /* 162 * Allocate memory for I/O pagetables. They need to be physically 163 * contiguous. 164 */ 165 166 size = PAGE_SIZE << is->is_tsbsize; 167 TAILQ_INIT(&mlist); 168 if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1, 169 (paddr_t)PAGE_SIZE, (paddr_t)0, &mlist, 1, UVM_PLA_NOWAIT) != 0) 170 panic("iommu_init: no memory"); 171 172 va = uvm_km_valloc(kernel_map, size); 173 if (va == 0) 174 panic("iommu_init: no memory"); 175 is->is_tsb = (int64_t *)va; 176 177 m = TAILQ_FIRST(&mlist); 178 is->is_ptsb = VM_PAGE_TO_PHYS(m); 179 180 /* Map the pages */ 181 for (; m != NULL; m = TAILQ_NEXT(m,pageq)) { 182 pa = VM_PAGE_TO_PHYS(m); 183 pmap_enter(pmap_kernel(), va, pa | PMAP_NVC, 184 VM_PROT_READ|VM_PROT_WRITE, 185 VM_PROT_READ|VM_PROT_WRITE|PMAP_WIRED); 186 va += PAGE_SIZE; 187 } 188 pmap_update(pmap_kernel()); 189 memset(is->is_tsb, 0, size); 190 191 #ifdef DEBUG 192 if (iommudebug & IDB_INFO) { 193 /* Probe the iommu */ 194 /* The address or contents of the regs...? */ 195 printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n", 196 (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) + 197 IOMMUREG(iommu_cr), 198 (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) + 199 IOMMUREG(iommu_tsb), 200 (u_long)bus_space_vaddr(is->is_bustag, is->is_iommu) + 201 IOMMUREG(iommu_flush)); 202 printf("iommu cr=%llx tsb=%llx\n", 203 IOMMUREG_READ(is, iommu_cr), 204 IOMMUREG_READ(is, iommu_tsb)); 205 printf("TSB base %p phys %llx\n", 206 (void *)is->is_tsb, (unsigned long long)is->is_ptsb); 207 delay(1000000); /* 1 s */ 208 } 209 #endif 210 211 /* 212 * Now all the hardware's working we need to allocate a dvma map. 213 */ 214 printf("dvma map %x-%x", is->is_dvmabase, is->is_dvmaend); 215 #ifdef DEBUG 216 printf(", iotdb %llx-%llx", 217 (unsigned long long)is->is_ptsb, 218 (unsigned long long)(is->is_ptsb + size)); 219 #endif 220 is->is_dvmamap = extent_create(name, 221 is->is_dvmabase, (u_long)is->is_dvmaend + 1, 222 M_DEVBUF, 0, 0, EX_NOWAIT); 223 mtx_init(&is->is_mtx, IPL_HIGH); 224 225 /* 226 * Set the TSB size. The relevant bits were moved to the TSB 227 * base register in the PCIe host bridges. 228 */ 229 if (strncmp(name, "pyro", 4) == 0) 230 is->is_ptsb |= is->is_tsbsize; 231 else 232 is->is_cr |= (is->is_tsbsize << 16); 233 234 /* 235 * Now actually start up the IOMMU. 236 */ 237 iommu_reset(is); 238 printf("\n"); 239 } 240 241 /* 242 * Streaming buffers don't exist on the UltraSPARC IIi/e; we should have 243 * detected that already and disabled them. If not, we will notice that 244 * they aren't there when the STRBUF_EN bit does not remain. 245 */ 246 void 247 iommu_reset(struct iommu_state *is) 248 { 249 int i; 250 251 IOMMUREG_WRITE(is, iommu_tsb, is->is_ptsb); 252 253 /* Enable IOMMU */ 254 IOMMUREG_WRITE(is, iommu_cr, is->is_cr); 255 256 for (i = 0; i < 2; ++i) { 257 struct strbuf_ctl *sb = is->is_sb[i]; 258 259 if (sb == NULL) 260 continue; 261 262 sb->sb_iommu = is; 263 strbuf_reset(sb); 264 265 if (sb->sb_flush) 266 printf(", STC%d enabled", i); 267 } 268 269 if (is->is_flags & IOMMU_FLUSH_CACHE) 270 IOMMUREG_WRITE(is, iommu_cache_invalidate, -1ULL); 271 } 272 273 /* 274 * Initialize one STC. 275 */ 276 void 277 strbuf_reset(struct strbuf_ctl *sb) 278 { 279 if(sb->sb_flush == NULL) 280 return; 281 282 bus_space_write_8(sb->sb_bustag, sb->sb_sb, 283 STRBUFREG(strbuf_ctl), STRBUF_EN); 284 285 membar(Lookaside); 286 287 /* No streaming buffers? Disable them */ 288 if (bus_space_read_8(sb->sb_bustag, sb->sb_sb, 289 STRBUFREG(strbuf_ctl)) == 0) { 290 sb->sb_flush = NULL; 291 } else { 292 /* 293 * locate the pa of the flush buffer 294 */ 295 if (pmap_extract(pmap_kernel(), 296 (vaddr_t)sb->sb_flush, &sb->sb_flushpa) == FALSE) 297 sb->sb_flush = NULL; 298 mtx_init(&sb->sb_mtx, IPL_HIGH); 299 } 300 } 301 302 /* 303 * Add an entry to the IOMMU table. 304 * 305 * The entry is marked streaming if an STC was detected and 306 * the BUS_DMA_STREAMING flag is set. 307 */ 308 void 309 iommu_enter(struct iommu_state *is, struct strbuf_ctl *sb, vaddr_t va, 310 paddr_t pa, int flags) 311 { 312 int64_t tte; 313 volatile int64_t *tte_ptr = &is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)]; 314 315 #ifdef DIAGNOSTIC 316 if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend) 317 panic("iommu_enter: va %#lx not in DVMA space", va); 318 319 tte = *tte_ptr; 320 321 if (tte & IOTTE_V) { 322 printf("Overwriting valid tte entry (dva %lx pa %lx " 323 "&tte %p tte %llx)\n", va, pa, tte_ptr, tte); 324 extent_print(is->is_dvmamap); 325 panic("IOMMU overwrite"); 326 } 327 #endif 328 329 tte = MAKEIOTTE(pa, !(flags & BUS_DMA_NOWRITE), 330 !(flags & BUS_DMA_NOCACHE), (flags & BUS_DMA_STREAMING)); 331 332 DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n", 333 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va)); 334 335 *tte_ptr = tte; 336 337 /* 338 * Why bother to flush this va? It should only be relevant for 339 * V ==> V or V ==> non-V transitions. The former is illegal and 340 * the latter is never done here. It is true that this provides 341 * some protection against a misbehaving master using an address 342 * after it should. The IOMMU documentations specifically warns 343 * that the consequences of a simultaneous IOMMU flush and DVMA 344 * access to the same address are undefined. (By that argument, 345 * the STC should probably be flushed as well.) Note that if 346 * a bus master keeps using a memory region after it has been 347 * unmapped, the specific behavior of the IOMMU is likely to 348 * be the least of our worries. 349 */ 350 IOMMUREG_WRITE(is, iommu_flush, va); 351 352 DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n", 353 va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize), 354 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 355 (u_long)tte)); 356 } 357 358 /* 359 * Remove an entry from the IOMMU table. 360 * 361 * The entry is flushed from the STC if an STC is detected and the TSB 362 * entry has the IOTTE_STREAM flags set. It should be impossible for 363 * the TSB entry to have this flag set without the BUS_DMA_STREAMING 364 * flag, but better to be safe. (The IOMMU will be ignored as long 365 * as an STC entry exists.) 366 */ 367 void 368 iommu_remove(struct iommu_state *is, struct strbuf_ctl *sb, vaddr_t va) 369 { 370 int64_t *tte_ptr = &is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)]; 371 int64_t tte; 372 373 #ifdef DIAGNOSTIC 374 if (va < is->is_dvmabase || (va + PAGE_MASK) > is->is_dvmaend) 375 panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va); 376 if (va != trunc_page(va)) { 377 printf("iommu_remove: unaligned va: %lx\n", va); 378 va = trunc_page(va); 379 } 380 #endif 381 tte = *tte_ptr; 382 383 DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%llx]@%p\n", 384 va, tte, tte_ptr)); 385 386 #ifdef DIAGNOSTIC 387 if ((tte & IOTTE_V) == 0) { 388 printf("Removing invalid tte entry (dva %lx &tte %p " 389 "tte %llx)\n", va, tte_ptr, tte); 390 extent_print(is->is_dvmamap); 391 panic("IOMMU remove overwrite"); 392 } 393 #endif 394 395 *tte_ptr = tte & ~IOTTE_V; 396 397 /* 398 * IO operations are strongly ordered WRT each other. It is 399 * unclear how they relate to normal memory accesses. 400 */ 401 membar(StoreStore); 402 403 IOMMUREG_WRITE(is, iommu_flush, va); 404 405 if (sb && (tte & IOTTE_STREAM)) 406 iommu_strbuf_flush(sb, va); 407 408 /* Should we sync the iommu and stc here? */ 409 } 410 411 /* 412 * Find the physical address of a DVMA address (debug routine). 413 */ 414 paddr_t 415 iommu_extract(struct iommu_state *is, vaddr_t dva) 416 { 417 int64_t tte = 0; 418 419 if (dva >= is->is_dvmabase && dva <= is->is_dvmaend) 420 tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)]; 421 422 return (tte & IOTTE_PAMASK); 423 } 424 425 /* 426 * Lookup a TSB entry for a given DVMA (debug routine). 427 */ 428 int64_t 429 iommu_lookup_tte(struct iommu_state *is, vaddr_t dva) 430 { 431 int64_t tte = 0; 432 433 if (dva >= is->is_dvmabase && dva <= is->is_dvmaend) 434 tte = is->is_tsb[IOTSBSLOT(dva, is->is_tsbsize)]; 435 436 return (tte); 437 } 438 439 /* 440 * Lookup a TSB entry at a given physical address (debug routine). 441 */ 442 int64_t 443 iommu_fetch_tte(struct iommu_state *is, paddr_t pa) 444 { 445 int64_t tte = 0; 446 447 if (pa >= is->is_ptsb && pa < is->is_ptsb + 448 (PAGE_SIZE << is->is_tsbsize)) 449 tte = ldxa(pa, ASI_PHYS_CACHED); 450 451 return (tte); 452 } 453 454 /* 455 * Fetch a TSB entry with some sanity checking. 456 */ 457 int64_t 458 iommu_tsb_entry(struct iommu_state *is, vaddr_t dva) 459 { 460 int64_t tte; 461 462 if (dva < is->is_dvmabase || dva > is->is_dvmaend) 463 panic("invalid dva: %llx", (long long)dva); 464 465 tte = is->is_tsb[IOTSBSLOT(dva,is->is_tsbsize)]; 466 467 if ((tte & IOTTE_V) == 0) 468 panic("iommu_tsb_entry: invalid entry %lx", dva); 469 470 return (tte); 471 } 472 473 /* 474 * Initiate and then block until an STC flush synchronization has completed. 475 */ 476 int 477 iommu_strbuf_flush_done(struct iommu_map_state *ims) 478 { 479 struct strbuf_ctl *sb = ims->ims_sb; 480 struct strbuf_flush *sf = &ims->ims_flush; 481 struct timeval cur, flushtimeout; 482 struct timeval to = { 0, 500000 }; 483 u_int64_t flush; 484 int timeout_started = 0; 485 486 #ifdef DIAGNOSTIC 487 if (sb == NULL) { 488 panic("iommu_strbuf_flush_done: invalid flush buffer"); 489 } 490 #endif 491 492 mtx_enter(&sb->sb_mtx); 493 494 /* 495 * Streaming buffer flushes: 496 * 497 * 1 Tell strbuf to flush by storing va to strbuf_pgflush. 498 * 2 Store 0 in flag 499 * 3 Store pointer to flag in flushsync 500 * 4 wait till flushsync becomes 0x1 501 * 502 * If it takes more than .5 sec, something went very, very wrong. 503 */ 504 505 /* 506 * If we're reading from ASI_PHYS_CACHED, then we'll write to 507 * it too. No need to tempt fate or learn about Si bugs or such. 508 * FreeBSD just uses normal "volatile" reads/writes... 509 */ 510 511 stxa(sf->sbf_flushpa, ASI_PHYS_CACHED, 0); 512 513 /* 514 * Insure any previous strbuf operations are complete and that 515 * memory is initialized before the IOMMU uses it. 516 * Is this Needed? How are IO and memory operations ordered? 517 */ 518 membar(StoreStore); 519 520 bus_space_write_8(sb->sb_bustag, sb->sb_sb, 521 STRBUFREG(strbuf_flushsync), sf->sbf_flushpa); 522 523 DPRINTF(IDB_IOMMU, 524 ("iommu_strbuf_flush_done: flush = %llx pa = %lx\n", 525 ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED), sf->sbf_flushpa)); 526 527 membar(StoreLoad | Lookaside); 528 529 for(;;) { 530 int i; 531 532 /* 533 * Try to shave a few instruction cycles off the average 534 * latency by only checking the elapsed time every few 535 * fetches. 536 */ 537 for (i = 0; i < 1000; ++i) { 538 membar(LoadLoad); 539 /* Bypass non-coherent D$ */ 540 /* non-coherent...? Huh? */ 541 flush = ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED); 542 543 if (flush) { 544 DPRINTF(IDB_IOMMU, 545 ("iommu_strbuf_flush_done: flushed\n")); 546 mtx_leave(&sb->sb_mtx); 547 return (0); 548 } 549 } 550 551 microtime(&cur); 552 553 if (timeout_started) { 554 if (timercmp(&cur, &flushtimeout, >)) 555 panic("STC timeout at %lx (%lld)", 556 sf->sbf_flushpa, flush); 557 } else { 558 timeradd(&cur, &to, &flushtimeout); 559 560 timeout_started = 1; 561 562 DPRINTF(IDB_IOMMU, 563 ("iommu_strbuf_flush_done: flush = %llx pa = %lx " 564 "now=%lx:%lx until = %lx:%lx\n", 565 ldxa(sf->sbf_flushpa, ASI_PHYS_CACHED), 566 sf->sbf_flushpa, cur.tv_sec, cur.tv_usec, 567 flushtimeout.tv_sec, flushtimeout.tv_usec)); 568 } 569 } 570 } 571 572 /* 573 * IOMMU DVMA operations, common to SBus and PCI. 574 */ 575 576 #define BUS_DMA_FIND_PARENT(t, fn) \ 577 if (t->_parent == NULL) \ 578 panic("null bus_dma parent (" #fn ")"); \ 579 for (t = t->_parent; t->fn == NULL; t = t->_parent) \ 580 if (t->_parent == NULL) \ 581 panic("no bus_dma " #fn " located"); 582 583 int 584 iommu_dvmamap_create(bus_dma_tag_t t, bus_dma_tag_t t0, struct strbuf_ctl *sb, 585 bus_size_t size, int nsegments, bus_size_t maxsegsz, bus_size_t boundary, 586 int flags, bus_dmamap_t *dmamap) 587 { 588 int ret; 589 bus_dmamap_t map; 590 struct iommu_map_state *ims; 591 592 BUS_DMA_FIND_PARENT(t, _dmamap_create); 593 ret = (*t->_dmamap_create)(t, t0, size, nsegments, maxsegsz, boundary, 594 flags, &map); 595 596 if (ret) 597 return (ret); 598 599 ims = iommu_iomap_create(atop(round_page(size))); 600 601 if (ims == NULL) { 602 bus_dmamap_destroy(t0, map); 603 return (ENOMEM); 604 } 605 606 ims->ims_sb = sb; 607 map->_dm_cookie = ims; 608 609 #ifdef DIAGNOSTIC 610 if (ims->ims_sb == NULL) 611 panic("iommu_dvmamap_create: null sb"); 612 if (ims->ims_sb->sb_iommu == NULL) 613 panic("iommu_dvmamap_create: null iommu"); 614 #endif 615 *dmamap = map; 616 617 return (0); 618 } 619 620 void 621 iommu_dvmamap_destroy(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map) 622 { 623 /* 624 * The specification (man page) requires a loaded 625 * map to be unloaded before it is destroyed. 626 */ 627 if (map->dm_nsegs) 628 bus_dmamap_unload(t0, map); 629 630 if (map->_dm_cookie) 631 iommu_iomap_destroy(map->_dm_cookie); 632 map->_dm_cookie = NULL; 633 634 BUS_DMA_FIND_PARENT(t, _dmamap_destroy); 635 (*t->_dmamap_destroy)(t, t0, map); 636 } 637 638 /* 639 * Load a contiguous kva buffer into a dmamap. The physical pages are 640 * not assumed to be contiguous. Two passes are made through the buffer 641 * and both call pmap_extract() for the same va->pa translations. It 642 * is possible to run out of pa->dvma mappings; the code should be smart 643 * enough to resize the iomap (when the "flags" permit allocation). It 644 * is trivial to compute the number of entries required (round the length 645 * up to the page size and then divide by the page size)... 646 */ 647 int 648 iommu_dvmamap_load(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, 649 void *buf, bus_size_t buflen, struct proc *p, int flags) 650 { 651 int err = 0; 652 bus_size_t sgsize; 653 u_long dvmaddr, sgstart, sgend; 654 bus_size_t align, boundary; 655 struct iommu_state *is; 656 struct iommu_map_state *ims = map->_dm_cookie; 657 pmap_t pmap; 658 659 #ifdef DIAGNOSTIC 660 if (ims == NULL) 661 panic("iommu_dvmamap_load: null map state"); 662 #endif 663 #ifdef DEBUG 664 if (ims->ims_sb == NULL) 665 panic("iommu_dvmamap_load: null sb"); 666 if (ims->ims_sb->sb_iommu == NULL) 667 panic("iommu_dvmamap_load: null iommu"); 668 #endif /* DEBUG */ 669 is = ims->ims_sb->sb_iommu; 670 671 if (map->dm_nsegs) { 672 /* 673 * Is it still in use? _bus_dmamap_load should have taken care 674 * of this. 675 */ 676 #ifdef DIAGNOSTIC 677 panic("iommu_dvmamap_load: map still in use"); 678 #endif 679 bus_dmamap_unload(t0, map); 680 } 681 682 /* 683 * Make sure that on error condition we return "no valid mappings". 684 */ 685 map->dm_nsegs = 0; 686 687 if (buflen < 1 || buflen > map->_dm_size) { 688 DPRINTF(IDB_BUSDMA, 689 ("iommu_dvmamap_load(): error %d > %d -- " 690 "map size exceeded!\n", (int)buflen, (int)map->_dm_size)); 691 return (EINVAL); 692 } 693 694 /* 695 * A boundary presented to bus_dmamem_alloc() takes precedence 696 * over boundary in the map. 697 */ 698 if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0) 699 boundary = map->_dm_boundary; 700 align = MAX(map->dm_segs[0]._ds_align, PAGE_SIZE); 701 702 pmap = p ? p->p_vmspace->vm_map.pmap : pmap_kernel(); 703 704 /* Count up the total number of pages we need */ 705 iommu_iomap_clear_pages(ims); 706 { /* Scope */ 707 bus_addr_t a, aend; 708 bus_addr_t addr = (vaddr_t)buf; 709 int seg_len = buflen; 710 711 aend = round_page(addr + seg_len); 712 for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { 713 paddr_t pa; 714 715 if (pmap_extract(pmap, a, &pa) == FALSE) { 716 printf("iomap pmap error addr 0x%llx\n", a); 717 iommu_iomap_clear_pages(ims); 718 return (EFBIG); 719 } 720 721 err = iommu_iomap_insert_page(ims, pa); 722 if (err) { 723 printf("iomap insert error: %d for " 724 "va 0x%llx pa 0x%lx " 725 "(buf %p len %lld/%llx)\n", 726 err, a, pa, buf, buflen, buflen); 727 iommu_dvmamap_print_map(t, is, map); 728 iommu_iomap_clear_pages(ims); 729 return (EFBIG); 730 } 731 } 732 } 733 sgsize = ims->ims_map.ipm_pagecnt * PAGE_SIZE; 734 735 mtx_enter(&is->is_mtx); 736 if (flags & BUS_DMA_24BIT) { 737 sgstart = MAX(is->is_dvmamap->ex_start, 0xff000000); 738 sgend = MIN(is->is_dvmamap->ex_end, 0xffffffff); 739 } else { 740 sgstart = is->is_dvmamap->ex_start; 741 sgend = is->is_dvmamap->ex_end; 742 } 743 744 /* 745 * If our segment size is larger than the boundary we need to 746 * split the transfer up into little pieces ourselves. 747 */ 748 err = extent_alloc_subregion(is->is_dvmamap, sgstart, sgend, 749 sgsize, align, 0, (sgsize > boundary) ? 0 : boundary, 750 EX_NOWAIT | EX_BOUNDZERO, (u_long *)&dvmaddr); 751 mtx_leave(&is->is_mtx); 752 753 #ifdef DEBUG 754 if (err || (dvmaddr == (bus_addr_t)-1)) { 755 printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n", 756 (int)sgsize, flags); 757 #ifdef DDB 758 if (iommudebug & IDB_BREAK) 759 Debugger(); 760 #endif 761 } 762 #endif 763 if (err != 0) 764 return (err); 765 766 /* Set the active DVMA map */ 767 map->_dm_dvmastart = dvmaddr; 768 map->_dm_dvmasize = sgsize; 769 770 map->dm_mapsize = buflen; 771 772 #ifdef DEBUG 773 iommu_dvmamap_validate_map(t, is, map); 774 #endif 775 776 iommu_iomap_load_map(is, ims, dvmaddr, flags); 777 778 { /* Scope */ 779 bus_addr_t a, aend; 780 bus_addr_t addr = (vaddr_t)buf; 781 int seg_len = buflen; 782 783 aend = round_page(addr + seg_len); 784 for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { 785 bus_addr_t pgstart; 786 bus_addr_t pgend; 787 paddr_t pa; 788 int pglen; 789 790 /* Yuck... Redoing the same pmap_extract... */ 791 if (pmap_extract(pmap, a, &pa) == FALSE) { 792 printf("iomap pmap error addr 0x%llx\n", a); 793 err = EFBIG; 794 break; 795 } 796 797 pgstart = pa | (MAX(a, addr) & PAGE_MASK); 798 pgend = pa | (MIN(a + PAGE_SIZE - 1, 799 addr + seg_len - 1) & PAGE_MASK); 800 pglen = pgend - pgstart + 1; 801 802 if (pglen < 1) 803 continue; 804 805 err = iommu_dvmamap_append_range(t, map, pgstart, 806 pglen, flags, boundary); 807 if (err == EFBIG) 808 break; 809 else if (err) { 810 printf("iomap load seg page: %d for " 811 "va 0x%llx pa %lx (%llx - %llx) " 812 "for %d/0x%x\n", 813 err, a, pa, pgstart, pgend, pglen, pglen); 814 break; 815 } 816 } 817 } 818 #ifdef DEBUG 819 iommu_dvmamap_validate_map(t, is, map); 820 821 if (err) 822 printf("**** iommu_dvmamap_load failed with error %d\n", 823 err); 824 825 if (err || (iommudebug & IDB_PRINT_MAP)) { 826 iommu_dvmamap_print_map(t, is, map); 827 #ifdef DDB 828 if (iommudebug & IDB_BREAK) 829 Debugger(); 830 #endif 831 } 832 #endif 833 if (err) { 834 /* XXX keep enough state and just call unload here? */ 835 iommu_iomap_unload_map(is, ims); 836 iommu_iomap_clear_pages(ims); 837 map->dm_mapsize = 0; 838 map->dm_nsegs = 0; 839 mtx_enter(&is->is_mtx); 840 err = extent_free(is->is_dvmamap, dvmaddr, sgsize, EX_NOWAIT); 841 map->_dm_dvmastart = 0; 842 map->_dm_dvmasize = 0; 843 mtx_leave(&is->is_mtx); 844 } 845 846 return (err); 847 } 848 849 /* 850 * Load a dvmamap from an array of segs or an mlist (if the first 851 * "segs" entry's mlist is non-null). It calls iommu_dvmamap_load_segs() 852 * or iommu_dvmamap_load_mlist() for part of the 2nd pass through the 853 * mapping. This is ugly. A better solution would probably be to have 854 * function pointers for implementing the traversal. That way, there 855 * could be one core load routine for each of the three required algorithms 856 * (buffer, seg, and mlist). That would also mean that the traversal 857 * algorithm would then only need one implementation for each algorithm 858 * instead of two (one for populating the iomap and one for populating 859 * the dvma map). 860 */ 861 int 862 iommu_dvmamap_load_raw(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, 863 bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags) 864 { 865 int i; 866 int left; 867 int err = 0; 868 bus_size_t sgsize; 869 bus_size_t boundary, align; 870 u_long dvmaddr, sgstart, sgend; 871 struct iommu_state *is; 872 struct iommu_map_state *ims = map->_dm_cookie; 873 874 #ifdef DIAGNOSTIC 875 if (ims == NULL) 876 panic("iommu_dvmamap_load_raw: null map state"); 877 #endif 878 #ifdef DEBUG 879 if (ims->ims_sb == NULL) 880 panic("iommu_dvmamap_load_raw: null sb"); 881 if (ims->ims_sb->sb_iommu == NULL) 882 panic("iommu_dvmamap_load_raw: null iommu"); 883 #endif /* DEBUG */ 884 is = ims->ims_sb->sb_iommu; 885 886 if (map->dm_nsegs) { 887 /* Already in use?? */ 888 #ifdef DIAGNOSTIC 889 panic("iommu_dvmamap_load_raw: map still in use"); 890 #endif 891 bus_dmamap_unload(t0, map); 892 } 893 894 /* 895 * A boundary presented to bus_dmamem_alloc() takes precedence 896 * over boundary in the map. 897 */ 898 if ((boundary = segs[0]._ds_boundary) == 0) 899 boundary = map->_dm_boundary; 900 901 align = MAX(segs[0]._ds_align, PAGE_SIZE); 902 903 /* 904 * Make sure that on error condition we return "no valid mappings". 905 */ 906 map->dm_nsegs = 0; 907 908 iommu_iomap_clear_pages(ims); 909 if (segs[0]._ds_mlist) { 910 struct pglist *mlist = segs[0]._ds_mlist; 911 struct vm_page *m; 912 for (m = TAILQ_FIRST(mlist); m != NULL; 913 m = TAILQ_NEXT(m,pageq)) { 914 err = iommu_iomap_insert_page(ims, VM_PAGE_TO_PHYS(m)); 915 916 if(err) { 917 printf("iomap insert error: %d for " 918 "pa 0x%lx\n", err, VM_PAGE_TO_PHYS(m)); 919 iommu_dvmamap_print_map(t, is, map); 920 iommu_iomap_clear_pages(ims); 921 return (EFBIG); 922 } 923 } 924 } else { 925 /* Count up the total number of pages we need */ 926 for (i = 0, left = size; left > 0 && i < nsegs; i++) { 927 bus_addr_t a, aend; 928 bus_size_t len = segs[i].ds_len; 929 bus_addr_t addr = segs[i].ds_addr; 930 int seg_len = MIN(left, len); 931 932 if (len < 1) 933 continue; 934 935 aend = round_page(addr + seg_len); 936 for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { 937 938 err = iommu_iomap_insert_page(ims, a); 939 if (err) { 940 printf("iomap insert error: %d for " 941 "pa 0x%llx\n", err, a); 942 iommu_dvmamap_print_map(t, is, map); 943 iommu_iomap_clear_pages(ims); 944 return (EFBIG); 945 } 946 } 947 948 left -= seg_len; 949 } 950 } 951 sgsize = ims->ims_map.ipm_pagecnt * PAGE_SIZE; 952 953 mtx_enter(&is->is_mtx); 954 if (flags & BUS_DMA_24BIT) { 955 sgstart = MAX(is->is_dvmamap->ex_start, 0xff000000); 956 sgend = MIN(is->is_dvmamap->ex_end, 0xffffffff); 957 } else { 958 sgstart = is->is_dvmamap->ex_start; 959 sgend = is->is_dvmamap->ex_end; 960 } 961 962 /* 963 * If our segment size is larger than the boundary we need to 964 * split the transfer up into little pieces ourselves. 965 */ 966 err = extent_alloc_subregion(is->is_dvmamap, sgstart, sgend, 967 sgsize, align, 0, (sgsize > boundary) ? 0 : boundary, 968 EX_NOWAIT | EX_BOUNDZERO, (u_long *)&dvmaddr); 969 mtx_leave(&is->is_mtx); 970 971 if (err != 0) 972 return (err); 973 974 #ifdef DEBUG 975 if (dvmaddr == (bus_addr_t)-1) { 976 printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) " 977 "failed!\n", (int)sgsize, flags); 978 #ifdef DDB 979 if (iommudebug & IDB_BREAK) 980 Debugger(); 981 #else 982 panic(""); 983 #endif 984 } 985 #endif 986 987 /* Set the active DVMA map */ 988 map->_dm_dvmastart = dvmaddr; 989 map->_dm_dvmasize = sgsize; 990 991 map->dm_mapsize = size; 992 993 #ifdef DEBUG 994 iommu_dvmamap_validate_map(t, is, map); 995 #endif 996 997 iommu_iomap_load_map(is, ims, dvmaddr, flags); 998 999 if (segs[0]._ds_mlist) 1000 err = iommu_dvmamap_load_mlist(t, is, map, segs[0]._ds_mlist, 1001 flags, size, boundary); 1002 else 1003 err = iommu_dvmamap_load_seg(t, is, map, segs, nsegs, 1004 flags, size, boundary); 1005 1006 #ifdef DEBUG 1007 /* The map should be valid even if the load failed */ 1008 if (iommu_dvmamap_validate_map(t, is, map)) { 1009 printf("load size %lld/0x%llx\n", size, size); 1010 if (segs[0]._ds_mlist) 1011 printf("mlist %p\n", segs[0]._ds_mlist); 1012 else { 1013 long tot_len = 0; 1014 long clip_len = 0; 1015 printf("segs %p nsegs %d\n", segs, nsegs); 1016 1017 left = size; 1018 for(i = 0; i < nsegs; i++) { 1019 bus_size_t len = segs[i].ds_len; 1020 bus_addr_t addr = segs[i].ds_addr; 1021 int seg_len = MIN(left, len); 1022 1023 printf("addr %llx len %lld/0x%llx seg_len " 1024 "%d/0x%x left %d/0x%x\n", addr, len, len, 1025 seg_len, seg_len, left, left); 1026 1027 left -= seg_len; 1028 1029 clip_len += seg_len; 1030 tot_len += segs[i].ds_len; 1031 } 1032 printf("total length %ld/0x%lx total seg. " 1033 "length %ld/0x%lx\n", tot_len, tot_len, clip_len, 1034 clip_len); 1035 } 1036 1037 if (err == 0) 1038 err = 1; 1039 } 1040 1041 if (err) 1042 printf("**** iommu_dvmamap_load_raw failed with error %d\n", 1043 err); 1044 1045 if (err || (iommudebug & IDB_PRINT_MAP)) { 1046 iommu_dvmamap_print_map(t, is, map); 1047 #ifdef DDB 1048 if (iommudebug & IDB_BREAK) 1049 Debugger(); 1050 #endif 1051 } 1052 #endif 1053 if (err) { 1054 /* XXX keep enough state and just call unload here? */ 1055 iommu_iomap_unload_map(is, ims); 1056 iommu_iomap_clear_pages(ims); 1057 map->dm_mapsize = 0; 1058 map->dm_nsegs = 0; 1059 mtx_enter(&is->is_mtx); 1060 err = extent_free(is->is_dvmamap, dvmaddr, sgsize, EX_NOWAIT); 1061 map->_dm_dvmastart = 0; 1062 map->_dm_dvmasize = 0; 1063 mtx_leave(&is->is_mtx); 1064 } 1065 1066 return (err); 1067 } 1068 1069 /* 1070 * Insert a range of addresses into a loaded map respecting the specified 1071 * boundary and alignment restrictions. The range is specified by its 1072 * physical address and length. The range cannot cross a page boundary. 1073 * This code (along with most of the rest of the function in this file) 1074 * assumes that the IOMMU page size is equal to PAGE_SIZE. 1075 */ 1076 int 1077 iommu_dvmamap_append_range(bus_dma_tag_t t, bus_dmamap_t map, paddr_t pa, 1078 bus_size_t length, int flags, bus_size_t boundary) 1079 { 1080 struct iommu_map_state *ims = map->_dm_cookie; 1081 bus_addr_t sgstart, sgend, bd_mask; 1082 bus_dma_segment_t *seg = NULL; 1083 int i = map->dm_nsegs; 1084 1085 #ifdef DEBUG 1086 if (ims == NULL) 1087 panic("iommu_dvmamap_append_range: null map state"); 1088 #endif 1089 1090 sgstart = iommu_iomap_translate(ims, pa); 1091 sgend = sgstart + length - 1; 1092 1093 #ifdef DIAGNOSTIC 1094 if (sgstart == NULL || sgstart > sgend) { 1095 printf("append range invalid mapping for %lx " 1096 "(0x%llx - 0x%llx)\n", pa, sgstart, sgend); 1097 map->dm_nsegs = 0; 1098 return (EINVAL); 1099 } 1100 #endif 1101 1102 #ifdef DEBUG 1103 if (trunc_page(sgstart) != trunc_page(sgend)) { 1104 printf("append range crossing page boundary! " 1105 "pa %lx length %lld/0x%llx sgstart %llx sgend %llx\n", 1106 pa, length, length, sgstart, sgend); 1107 } 1108 #endif 1109 1110 /* 1111 * We will attempt to merge this range with the previous entry 1112 * (if there is one). 1113 */ 1114 if (i > 0) { 1115 seg = &map->dm_segs[i - 1]; 1116 if (sgstart == seg->ds_addr + seg->ds_len) { 1117 length += seg->ds_len; 1118 sgstart = seg->ds_addr; 1119 sgend = sgstart + length - 1; 1120 } else 1121 seg = NULL; 1122 } 1123 1124 if (seg == NULL) { 1125 seg = &map->dm_segs[i]; 1126 if (++i > map->_dm_segcnt) { 1127 map->dm_nsegs = 0; 1128 return (EFBIG); 1129 } 1130 } 1131 1132 /* 1133 * At this point, "i" is the index of the *next* bus_dma_segment_t 1134 * (the segment count, aka map->dm_nsegs) and "seg" points to the 1135 * *current* entry. "length", "sgstart", and "sgend" reflect what 1136 * we intend to put in "*seg". No assumptions should be made about 1137 * the contents of "*seg". Only "boundary" issue can change this 1138 * and "boundary" is often zero, so explicitly test for that case 1139 * (the test is strictly an optimization). 1140 */ 1141 if (boundary != 0) { 1142 bd_mask = ~(boundary - 1); 1143 1144 while ((sgstart & bd_mask) != (sgend & bd_mask)) { 1145 /* 1146 * We are crossing a boundary so fill in the current 1147 * segment with as much as possible, then grab a new 1148 * one. 1149 */ 1150 1151 seg->ds_addr = sgstart; 1152 seg->ds_len = boundary - (sgstart & bd_mask); 1153 1154 sgstart += seg->ds_len; /* sgend stays the same */ 1155 length -= seg->ds_len; 1156 1157 seg = &map->dm_segs[i]; 1158 if (++i > map->_dm_segcnt) { 1159 map->dm_nsegs = 0; 1160 return (EFBIG); 1161 } 1162 } 1163 } 1164 1165 seg->ds_addr = sgstart; 1166 seg->ds_len = length; 1167 map->dm_nsegs = i; 1168 1169 return (0); 1170 } 1171 1172 /* 1173 * Populate the iomap from a bus_dma_segment_t array. See note for 1174 * iommu_dvmamap_load() * regarding page entry exhaustion of the iomap. 1175 * This is less of a problem for load_seg, as the number of pages 1176 * is usually similar to the number of segments (nsegs). 1177 */ 1178 int 1179 iommu_dvmamap_load_seg(bus_dma_tag_t t, struct iommu_state *is, 1180 bus_dmamap_t map, bus_dma_segment_t *segs, int nsegs, int flags, 1181 bus_size_t size, bus_size_t boundary) 1182 { 1183 int i; 1184 int left; 1185 int seg; 1186 1187 /* 1188 * This segs is made up of individual physical 1189 * segments, probably by _bus_dmamap_load_uio() or 1190 * _bus_dmamap_load_mbuf(). Ignore the mlist and 1191 * load each one individually. 1192 */ 1193 1194 /* 1195 * Keep in mind that each segment could span 1196 * multiple pages and that these are not always 1197 * adjacent. The code is no longer adding dvma 1198 * aliases to the IOMMU. The STC will not cross 1199 * page boundaries anyway and a IOMMU table walk 1200 * vs. what may be a streamed PCI DMA to a ring 1201 * descriptor is probably a wash. It eases TLB 1202 * pressure and in the worst possible case, it is 1203 * only as bad a non-IOMMUed architecture. More 1204 * importantly, the code is not quite as hairy. 1205 * (It's bad enough as it is.) 1206 */ 1207 left = size; 1208 seg = 0; 1209 for (i = 0; left > 0 && i < nsegs; i++) { 1210 bus_addr_t a, aend; 1211 bus_size_t len = segs[i].ds_len; 1212 bus_addr_t addr = segs[i].ds_addr; 1213 int seg_len = MIN(left, len); 1214 1215 if (len < 1) 1216 continue; 1217 1218 aend = round_page(addr + seg_len); 1219 for (a = trunc_page(addr); a < aend; a += PAGE_SIZE) { 1220 bus_addr_t pgstart; 1221 bus_addr_t pgend; 1222 int pglen; 1223 int err; 1224 1225 pgstart = MAX(a, addr); 1226 pgend = MIN(a + PAGE_SIZE - 1, addr + seg_len - 1); 1227 pglen = pgend - pgstart + 1; 1228 1229 if (pglen < 1) 1230 continue; 1231 1232 err = iommu_dvmamap_append_range(t, map, pgstart, 1233 pglen, flags, boundary); 1234 if (err == EFBIG) 1235 return (err); 1236 if (err) { 1237 printf("iomap load seg page: %d for " 1238 "pa 0x%llx (%llx - %llx for %d/%x\n", 1239 err, a, pgstart, pgend, pglen, pglen); 1240 return (err); 1241 } 1242 1243 } 1244 1245 left -= seg_len; 1246 } 1247 return (0); 1248 } 1249 1250 /* 1251 * Populate the iomap from an mlist. See note for iommu_dvmamap_load() 1252 * regarding page entry exhaustion of the iomap. 1253 */ 1254 int 1255 iommu_dvmamap_load_mlist(bus_dma_tag_t t, struct iommu_state *is, 1256 bus_dmamap_t map, struct pglist *mlist, int flags, 1257 bus_size_t size, bus_size_t boundary) 1258 { 1259 struct vm_page *m; 1260 paddr_t pa; 1261 int err; 1262 1263 /* 1264 * This was allocated with bus_dmamem_alloc. 1265 * The pages are on an `mlist'. 1266 */ 1267 for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq)) { 1268 pa = VM_PAGE_TO_PHYS(m); 1269 1270 err = iommu_dvmamap_append_range(t, map, pa, PAGE_SIZE, 1271 flags, boundary); 1272 if (err == EFBIG) 1273 return (err); 1274 if (err) { 1275 printf("iomap load seg page: %d for pa 0x%lx " 1276 "(%lx - %lx for %d/%x\n", err, pa, pa, 1277 pa + PAGE_SIZE, PAGE_SIZE, PAGE_SIZE); 1278 return (err); 1279 } 1280 } 1281 1282 return (0); 1283 } 1284 1285 /* 1286 * Unload a dvmamap. 1287 */ 1288 void 1289 iommu_dvmamap_unload(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map) 1290 { 1291 struct iommu_state *is; 1292 struct iommu_map_state *ims = map->_dm_cookie; 1293 bus_addr_t dvmaddr = map->_dm_dvmastart; 1294 bus_size_t sgsize = map->_dm_dvmasize; 1295 int error; 1296 1297 #ifdef DEBUG 1298 if (ims == NULL) 1299 panic("iommu_dvmamap_unload: null map state"); 1300 if (ims->ims_sb == NULL) 1301 panic("iommu_dvmamap_unload: null sb"); 1302 if (ims->ims_sb->sb_iommu == NULL) 1303 panic("iommu_dvmamap_unload: null iommu"); 1304 #endif /* DEBUG */ 1305 1306 is = ims->ims_sb->sb_iommu; 1307 1308 /* Flush the iommu */ 1309 #ifdef DEBUG 1310 if (dvmaddr == 0) { 1311 printf("iommu_dvmamap_unload: No dvmastart\n"); 1312 #ifdef DDB 1313 if (iommudebug & IDB_BREAK) 1314 Debugger(); 1315 #endif 1316 return; 1317 } 1318 1319 iommu_dvmamap_validate_map(t, is, map); 1320 1321 if (iommudebug & IDB_PRINT_MAP) 1322 iommu_dvmamap_print_map(t, is, map); 1323 #endif /* DEBUG */ 1324 1325 /* Remove the IOMMU entries */ 1326 iommu_iomap_unload_map(is, ims); 1327 1328 /* Clear the iomap */ 1329 iommu_iomap_clear_pages(ims); 1330 1331 bus_dmamap_unload(t->_parent, map); 1332 1333 /* Mark the mappings as invalid. */ 1334 map->dm_mapsize = 0; 1335 map->dm_nsegs = 0; 1336 1337 mtx_enter(&is->is_mtx); 1338 error = extent_free(is->is_dvmamap, dvmaddr, 1339 sgsize, EX_NOWAIT); 1340 map->_dm_dvmastart = 0; 1341 map->_dm_dvmasize = 0; 1342 mtx_leave(&is->is_mtx); 1343 if (error != 0) 1344 printf("warning: %qd of DVMA space lost\n", sgsize); 1345 } 1346 1347 #ifdef DEBUG 1348 /* 1349 * Perform internal consistency checking on a dvmamap. 1350 */ 1351 int 1352 iommu_dvmamap_validate_map(bus_dma_tag_t t, struct iommu_state *is, 1353 bus_dmamap_t map) 1354 { 1355 int err = 0; 1356 int seg; 1357 1358 if (trunc_page(map->_dm_dvmastart) != map->_dm_dvmastart) { 1359 printf("**** dvmastart address not page aligned: %llx", 1360 map->_dm_dvmastart); 1361 err = 1; 1362 } 1363 if (trunc_page(map->_dm_dvmasize) != map->_dm_dvmasize) { 1364 printf("**** dvmasize not a multiple of page size: %llx", 1365 map->_dm_dvmasize); 1366 err = 1; 1367 } 1368 if (map->_dm_dvmastart < is->is_dvmabase || 1369 (round_page(map->_dm_dvmastart + map->_dm_dvmasize) - 1) > 1370 is->is_dvmaend) { 1371 printf("dvmaddr %llx len %llx out of range %x - %x\n", 1372 map->_dm_dvmastart, map->_dm_dvmasize, 1373 is->is_dvmabase, is->is_dvmaend); 1374 err = 1; 1375 } 1376 for (seg = 0; seg < map->dm_nsegs; seg++) { 1377 if (map->dm_segs[seg].ds_addr == 0 || 1378 map->dm_segs[seg].ds_len == 0) { 1379 printf("seg %d null segment dvmaddr %llx len %llx for " 1380 "range %llx len %llx\n", 1381 seg, 1382 map->dm_segs[seg].ds_addr, 1383 map->dm_segs[seg].ds_len, 1384 map->_dm_dvmastart, map->_dm_dvmasize); 1385 err = 1; 1386 } else if (map->dm_segs[seg].ds_addr < map->_dm_dvmastart || 1387 round_page(map->dm_segs[seg].ds_addr + 1388 map->dm_segs[seg].ds_len) > 1389 map->_dm_dvmastart + map->_dm_dvmasize) { 1390 printf("seg %d dvmaddr %llx len %llx out of " 1391 "range %llx len %llx\n", 1392 seg, 1393 map->dm_segs[seg].ds_addr, 1394 map->dm_segs[seg].ds_len, 1395 map->_dm_dvmastart, map->_dm_dvmasize); 1396 err = 1; 1397 } 1398 } 1399 1400 if (err) { 1401 iommu_dvmamap_print_map(t, is, map); 1402 #if defined(DDB) && defined(DEBUG) 1403 if (iommudebug & IDB_BREAK) 1404 Debugger(); 1405 #endif 1406 } 1407 1408 return (err); 1409 } 1410 #endif /* DEBUG */ 1411 1412 void 1413 iommu_dvmamap_print_map(bus_dma_tag_t t, struct iommu_state *is, 1414 bus_dmamap_t map) 1415 { 1416 int seg, i; 1417 long full_len, source_len; 1418 struct mbuf *m; 1419 1420 printf("DVMA %x for %x, mapping %p: dvstart %llx dvsize %llx " 1421 "size %lld/%llx maxsegsz %llx boundary %llx segcnt %d " 1422 "flags %x type %d source %p " 1423 "cookie %p mapsize %llx nsegs %d\n", 1424 is ? is->is_dvmabase : 0, is ? is->is_dvmaend : 0, map, 1425 map->_dm_dvmastart, map->_dm_dvmasize, 1426 map->_dm_size, map->_dm_size, map->_dm_maxsegsz, map->_dm_boundary, 1427 map->_dm_segcnt, map->_dm_flags, map->_dm_type, 1428 map->_dm_source, map->_dm_cookie, map->dm_mapsize, 1429 map->dm_nsegs); 1430 1431 full_len = 0; 1432 for (seg = 0; seg < map->dm_nsegs; seg++) { 1433 printf("seg %d dvmaddr %llx pa %lx len %llx (tte %llx)\n", 1434 seg, map->dm_segs[seg].ds_addr, 1435 is ? iommu_extract(is, map->dm_segs[seg].ds_addr) : 0, 1436 map->dm_segs[seg].ds_len, 1437 is ? iommu_lookup_tte(is, map->dm_segs[seg].ds_addr) : 0); 1438 full_len += map->dm_segs[seg].ds_len; 1439 } 1440 printf("total length = %ld/0x%lx\n", full_len, full_len); 1441 1442 if (map->_dm_source) switch (map->_dm_type) { 1443 case _DM_TYPE_MBUF: 1444 m = map->_dm_source; 1445 if (m->m_flags & M_PKTHDR) 1446 printf("source PKTHDR mbuf (%p) hdr len = %d/0x%x:\n", 1447 m, m->m_pkthdr.len, m->m_pkthdr.len); 1448 else 1449 printf("source mbuf (%p):\n", m); 1450 1451 source_len = 0; 1452 for ( ; m; m = m->m_next) { 1453 vaddr_t vaddr = mtod(m, vaddr_t); 1454 long len = m->m_len; 1455 paddr_t pa; 1456 1457 if (pmap_extract(pmap_kernel(), vaddr, &pa)) 1458 printf("kva %lx pa %lx len %ld/0x%lx\n", 1459 vaddr, pa, len, len); 1460 else 1461 printf("kva %lx pa <invalid> len %ld/0x%lx\n", 1462 vaddr, len, len); 1463 1464 source_len += len; 1465 } 1466 1467 if (full_len != source_len) 1468 printf("mbuf length %ld/0x%lx is %s than mapping " 1469 "length %ld/0x%lx\n", source_len, source_len, 1470 (source_len > full_len) ? "greater" : "less", 1471 full_len, full_len); 1472 else 1473 printf("mbuf length %ld/0x%lx\n", source_len, 1474 source_len); 1475 break; 1476 case _DM_TYPE_LOAD: 1477 case _DM_TYPE_SEGS: 1478 case _DM_TYPE_UIO: 1479 default: 1480 break; 1481 } 1482 1483 if (map->_dm_cookie) { 1484 struct iommu_map_state *ims = map->_dm_cookie; 1485 struct iommu_page_map *ipm = &ims->ims_map; 1486 1487 printf("page map (%p) of size %d with %d entries\n", 1488 ipm, ipm->ipm_maxpage, ipm->ipm_pagecnt); 1489 for (i = 0; i < ipm->ipm_pagecnt; ++i) { 1490 struct iommu_page_entry *e = &ipm->ipm_map[i]; 1491 printf("%d: vmaddr 0x%lx pa 0x%lx\n", i, 1492 e->ipe_va, e->ipe_pa); 1493 } 1494 } else 1495 printf("iommu map state (cookie) is NULL\n"); 1496 } 1497 1498 void 1499 _iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, 1500 bus_addr_t offset, bus_size_t len, int ops) 1501 { 1502 struct iommu_state *is; 1503 struct iommu_map_state *ims = map->_dm_cookie; 1504 struct strbuf_ctl *sb; 1505 bus_size_t count; 1506 int i, needsflush = 0; 1507 1508 sb = ims->ims_sb; 1509 is = sb->sb_iommu; 1510 1511 for (i = 0; i < map->dm_nsegs; i++) { 1512 if (offset < map->dm_segs[i].ds_len) 1513 break; 1514 offset -= map->dm_segs[i].ds_len; 1515 } 1516 1517 if (i == map->dm_nsegs) 1518 panic("iommu_dvmamap_sync: too short %llu", offset); 1519 1520 for (; len > 0 && i < map->dm_nsegs; i++) { 1521 count = MIN(map->dm_segs[i].ds_len - offset, len); 1522 if (count > 0 && iommu_dvmamap_sync_range(sb, 1523 map->dm_segs[i].ds_addr + offset, count)) 1524 needsflush = 1; 1525 len -= count; 1526 } 1527 1528 #ifdef DIAGNOSTIC 1529 if (i == map->dm_nsegs && len > 0) 1530 panic("iommu_dvmamap_sync: leftover %llu", len); 1531 #endif 1532 1533 if (needsflush) 1534 iommu_strbuf_flush_done(ims); 1535 } 1536 1537 void 1538 iommu_dvmamap_sync(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dmamap_t map, 1539 bus_addr_t offset, bus_size_t len, int ops) 1540 { 1541 struct iommu_map_state *ims = map->_dm_cookie; 1542 1543 #ifdef DIAGNOSTIC 1544 if (ims == NULL) 1545 panic("iommu_dvmamap_sync: null map state"); 1546 if (ims->ims_sb == NULL) 1547 panic("iommu_dvmamap_sync: null sb"); 1548 if (ims->ims_sb->sb_iommu == NULL) 1549 panic("iommu_dvmamap_sync: null iommu"); 1550 #endif 1551 if (len == 0) 1552 return; 1553 1554 if (ops & BUS_DMASYNC_PREWRITE) 1555 membar(MemIssue); 1556 1557 if ((ims->ims_flags & IOMMU_MAP_STREAM) && 1558 (ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_PREWRITE))) 1559 _iommu_dvmamap_sync(t, t0, map, offset, len, ops); 1560 1561 if (ops & BUS_DMASYNC_POSTREAD) 1562 membar(MemIssue); 1563 } 1564 1565 /* 1566 * Flush an individual dma segment, returns non-zero if the streaming buffers 1567 * need flushing afterwards. 1568 */ 1569 int 1570 iommu_dvmamap_sync_range(struct strbuf_ctl *sb, vaddr_t va, bus_size_t len) 1571 { 1572 vaddr_t vaend; 1573 #ifdef DIAGNOSTIC 1574 struct iommu_state *is = sb->sb_iommu; 1575 1576 if (va < is->is_dvmabase || va > is->is_dvmaend) 1577 panic("invalid va: %llx", (long long)va); 1578 1579 if ((is->is_tsb[IOTSBSLOT(va, is->is_tsbsize)] & IOTTE_STREAM) == 0) { 1580 printf("iommu_dvmamap_sync_range: attempting to flush " 1581 "non-streaming entry\n"); 1582 return (0); 1583 } 1584 #endif 1585 1586 vaend = (va + len + PAGE_MASK) & ~PAGE_MASK; 1587 va &= ~PAGE_MASK; 1588 1589 #ifdef DIAGNOSTIC 1590 if (va < is->is_dvmabase || (vaend - 1) > is->is_dvmaend) 1591 panic("invalid va range: %llx to %llx (%x to %x)", 1592 (long long)va, (long long)vaend, 1593 is->is_dvmabase, 1594 is->is_dvmaend); 1595 #endif 1596 1597 for ( ; va <= vaend; va += PAGE_SIZE) { 1598 DPRINTF(IDB_BUSDMA, 1599 ("iommu_dvmamap_sync_range: flushing va %p\n", 1600 (void *)(u_long)va)); 1601 iommu_strbuf_flush(sb, va); 1602 } 1603 1604 return (1); 1605 } 1606 1607 int 1608 iommu_dvmamem_alloc(bus_dma_tag_t t, bus_dma_tag_t t0, bus_size_t size, 1609 bus_size_t alignment, bus_size_t boundary, bus_dma_segment_t *segs, 1610 int nsegs, int *rsegs, int flags) 1611 { 1612 1613 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx " 1614 "bound %llx segp %p flags %d\n", (unsigned long long)size, 1615 (unsigned long long)alignment, (unsigned long long)boundary, 1616 segs, flags)); 1617 BUS_DMA_FIND_PARENT(t, _dmamem_alloc); 1618 return ((*t->_dmamem_alloc)(t, t0, size, alignment, boundary, 1619 segs, nsegs, rsegs, flags | BUS_DMA_DVMA)); 1620 } 1621 1622 void 1623 iommu_dvmamem_free(bus_dma_tag_t t, bus_dma_tag_t t0, bus_dma_segment_t *segs, 1624 int nsegs) 1625 { 1626 1627 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n", 1628 segs, nsegs)); 1629 BUS_DMA_FIND_PARENT(t, _dmamem_free); 1630 (*t->_dmamem_free)(t, t0, segs, nsegs); 1631 } 1632 1633 /* 1634 * Create a new iomap. 1635 */ 1636 struct iommu_map_state * 1637 iommu_iomap_create(int n) 1638 { 1639 struct iommu_map_state *ims; 1640 struct strbuf_flush *sbf; 1641 vaddr_t va; 1642 1643 /* Safety for heavily fragmented data, such as mbufs */ 1644 n += 4; 1645 if (n < 16) 1646 n = 16; 1647 1648 ims = malloc(sizeof(*ims) + (n - 1) * sizeof(ims->ims_map.ipm_map[0]), 1649 M_DEVBUF, M_NOWAIT | M_ZERO); 1650 if (ims == NULL) 1651 return (NULL); 1652 1653 /* Initialize the map. */ 1654 ims->ims_map.ipm_maxpage = n; 1655 SPLAY_INIT(&ims->ims_map.ipm_tree); 1656 1657 /* Initialize the flush area. */ 1658 sbf = &ims->ims_flush; 1659 va = (vaddr_t)&sbf->sbf_area[0x40]; 1660 va &= ~0x3f; 1661 pmap_extract(pmap_kernel(), va, &sbf->sbf_flushpa); 1662 sbf->sbf_flush = (void *)va; 1663 1664 return (ims); 1665 } 1666 1667 /* 1668 * Destroy an iomap. 1669 */ 1670 void 1671 iommu_iomap_destroy(struct iommu_map_state *ims) 1672 { 1673 #ifdef DIAGNOSTIC 1674 if (ims->ims_map.ipm_pagecnt > 0) 1675 printf("iommu_iomap_destroy: %d page entries in use\n", 1676 ims->ims_map.ipm_pagecnt); 1677 #endif 1678 1679 free(ims, M_DEVBUF); 1680 } 1681 1682 /* 1683 * Utility function used by splay tree to order page entries by pa. 1684 */ 1685 static inline int 1686 iomap_compare(struct iommu_page_entry *a, struct iommu_page_entry *b) 1687 { 1688 return ((a->ipe_pa > b->ipe_pa) ? 1 : 1689 (a->ipe_pa < b->ipe_pa) ? -1 : 0); 1690 } 1691 1692 SPLAY_PROTOTYPE(iommu_page_tree, iommu_page_entry, ipe_node, iomap_compare); 1693 1694 SPLAY_GENERATE(iommu_page_tree, iommu_page_entry, ipe_node, iomap_compare); 1695 1696 /* 1697 * Insert a pa entry in the iomap. 1698 */ 1699 int 1700 iommu_iomap_insert_page(struct iommu_map_state *ims, paddr_t pa) 1701 { 1702 struct iommu_page_map *ipm = &ims->ims_map; 1703 struct iommu_page_entry *e; 1704 1705 if (ipm->ipm_pagecnt >= ipm->ipm_maxpage) { 1706 struct iommu_page_entry ipe; 1707 1708 ipe.ipe_pa = pa; 1709 if (SPLAY_FIND(iommu_page_tree, &ipm->ipm_tree, &ipe)) 1710 return (0); 1711 1712 return (ENOMEM); 1713 } 1714 1715 e = &ipm->ipm_map[ipm->ipm_pagecnt]; 1716 1717 e->ipe_pa = pa; 1718 e->ipe_va = NULL; 1719 1720 e = SPLAY_INSERT(iommu_page_tree, &ipm->ipm_tree, e); 1721 1722 /* Duplicates are okay, but only count them once. */ 1723 if (e) 1724 return (0); 1725 1726 ++ipm->ipm_pagecnt; 1727 1728 return (0); 1729 } 1730 1731 /* 1732 * Locate the iomap by filling in the pa->va mapping and inserting it 1733 * into the IOMMU tables. 1734 */ 1735 void 1736 iommu_iomap_load_map(struct iommu_state *is, struct iommu_map_state *ims, 1737 vaddr_t vmaddr, int flags) 1738 { 1739 struct iommu_page_map *ipm = &ims->ims_map; 1740 struct iommu_page_entry *e; 1741 struct strbuf_ctl *sb = ims->ims_sb; 1742 int i, slot; 1743 1744 if (sb->sb_flush == NULL) 1745 flags &= ~BUS_DMA_STREAMING; 1746 1747 if (flags & BUS_DMA_STREAMING) 1748 ims->ims_flags |= IOMMU_MAP_STREAM; 1749 else 1750 ims->ims_flags &= ~IOMMU_MAP_STREAM; 1751 1752 for (i = 0, e = ipm->ipm_map; i < ipm->ipm_pagecnt; ++i, ++e) { 1753 e->ipe_va = vmaddr; 1754 iommu_enter(is, sb, e->ipe_va, e->ipe_pa, flags); 1755 1756 /* Flush cache if necessary. */ 1757 slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize); 1758 if (is->is_flags & IOMMU_FLUSH_CACHE && 1759 (i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7)) 1760 IOMMUREG_WRITE(is, iommu_cache_flush, 1761 is->is_ptsb + slot * 8); 1762 1763 vmaddr += PAGE_SIZE; 1764 } 1765 } 1766 1767 /* 1768 * Remove the iomap from the IOMMU. 1769 */ 1770 void 1771 iommu_iomap_unload_map(struct iommu_state *is, struct iommu_map_state *ims) 1772 { 1773 struct iommu_page_map *ipm = &ims->ims_map; 1774 struct iommu_page_entry *e; 1775 struct strbuf_ctl *sb = ims->ims_sb; 1776 int i, slot; 1777 1778 for (i = 0, e = ipm->ipm_map; i < ipm->ipm_pagecnt; ++i, ++e) { 1779 iommu_remove(is, sb, e->ipe_va); 1780 1781 /* Flush cache if necessary. */ 1782 slot = IOTSBSLOT(e->ipe_va, is->is_tsbsize); 1783 if (is->is_flags & IOMMU_FLUSH_CACHE && 1784 (i == (ipm->ipm_pagecnt - 1) || (slot % 8) == 7)) 1785 IOMMUREG_WRITE(is, iommu_cache_flush, 1786 is->is_ptsb + slot * 8); 1787 } 1788 } 1789 1790 /* 1791 * Translate a physical address (pa) into a DVMA address. 1792 */ 1793 vaddr_t 1794 iommu_iomap_translate(struct iommu_map_state *ims, paddr_t pa) 1795 { 1796 struct iommu_page_map *ipm = &ims->ims_map; 1797 struct iommu_page_entry *e; 1798 struct iommu_page_entry pe; 1799 paddr_t offset = pa & PAGE_MASK; 1800 1801 pe.ipe_pa = trunc_page(pa); 1802 1803 e = SPLAY_FIND(iommu_page_tree, &ipm->ipm_tree, &pe); 1804 1805 if (e == NULL) 1806 return (NULL); 1807 1808 return (e->ipe_va | offset); 1809 } 1810 1811 /* 1812 * Clear the iomap table and tree. 1813 */ 1814 void 1815 iommu_iomap_clear_pages(struct iommu_map_state *ims) 1816 { 1817 ims->ims_map.ipm_pagecnt = 0; 1818 SPLAY_INIT(&ims->ims_map.ipm_tree); 1819 } 1820 1821