1 /* $NetBSD: iommu.c,v 1.37 2001/08/06 22:02:58 eeh Exp $ */ 2 3 /* 4 * Copyright (c) 1999, 2000 Matthew R. Green 5 * All rights reserved. 6 * 7 * Redistribution and use in source and binary forms, with or without 8 * modification, are permitted provided that the following conditions 9 * are met: 10 * 1. Redistributions of source code must retain the above copyright 11 * notice, this list of conditions and the following disclaimer. 12 * 2. Redistributions in binary form must reproduce the above copyright 13 * notice, this list of conditions and the following disclaimer in the 14 * documentation and/or other materials provided with the distribution. 15 * 3. The name of the author may not be used to endorse or promote products 16 * derived from this software without specific prior written permission. 17 * 18 * THIS SOFTWARE IS PROVIDED BY THE AUTHOR ``AS IS'' AND ANY EXPRESS OR 19 * IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE IMPLIED WARRANTIES 20 * OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE ARE DISCLAIMED. 21 * IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR ANY DIRECT, INDIRECT, 22 * INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL DAMAGES (INCLUDING, 23 * BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS OR SERVICES; 24 * LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) HOWEVER CAUSED 25 * AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT LIABILITY, 26 * OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 27 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 28 * SUCH DAMAGE. 29 */ 30 31 /*- 32 * Copyright (c) 1998 The NetBSD Foundation, Inc. 33 * All rights reserved. 34 * 35 * This code is derived from software contributed to The NetBSD Foundation 36 * by Paul Kranenburg. 37 * 38 * Redistribution and use in source and binary forms, with or without 39 * modification, are permitted provided that the following conditions 40 * are met: 41 * 1. Redistributions of source code must retain the above copyright 42 * notice, this list of conditions and the following disclaimer. 43 * 2. Redistributions in binary form must reproduce the above copyright 44 * notice, this list of conditions and the following disclaimer in the 45 * documentation and/or other materials provided with the distribution. 46 * 3. All advertising materials mentioning features or use of this software 47 * must display the following acknowledgement: 48 * This product includes software developed by the NetBSD 49 * Foundation, Inc. and its contributors. 50 * 4. Neither the name of The NetBSD Foundation nor the names of its 51 * contributors may be used to endorse or promote products derived 52 * from this software without specific prior written permission. 53 * 54 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 55 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 56 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 57 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 58 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 59 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 60 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 61 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 62 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 63 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 64 * POSSIBILITY OF SUCH DAMAGE. 65 */ 66 67 /* 68 * Copyright (c) 1992, 1993 69 * The Regents of the University of California. All rights reserved. 70 * 71 * This software was developed by the Computer Systems Engineering group 72 * at Lawrence Berkeley Laboratory under DARPA contract BG 91-66 and 73 * contributed to Berkeley. 74 * 75 * All advertising materials mentioning features or use of this software 76 * must display the following acknowledgement: 77 * This product includes software developed by the University of 78 * California, Lawrence Berkeley Laboratory. 79 * 80 * Redistribution and use in source and binary forms, with or without 81 * modification, are permitted provided that the following conditions 82 * are met: 83 * 1. Redistributions of source code must retain the above copyright 84 * notice, this list of conditions and the following disclaimer. 85 * 2. Redistributions in binary form must reproduce the above copyright 86 * notice, this list of conditions and the following disclaimer in the 87 * documentation and/or other materials provided with the distribution. 88 * 3. All advertising materials mentioning features or use of this software 89 * must display the following acknowledgement: 90 * This product includes software developed by the University of 91 * California, Berkeley and its contributors. 92 * 4. Neither the name of the University nor the names of its contributors 93 * may be used to endorse or promote products derived from this software 94 * without specific prior written permission. 95 * 96 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND 97 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE 98 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE 99 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE 100 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL 101 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS 102 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION) 103 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT 104 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY 105 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF 106 * SUCH DAMAGE. 107 * 108 * from: NetBSD: sbus.c,v 1.13 1999/05/23 07:24:02 mrg Exp 109 * from: @(#)sbus.c 8.1 (Berkeley) 6/11/93 110 */ 111 112 /* 113 * UltraSPARC IOMMU support; used by both the sbus and pci code. 114 */ 115 #include "opt_ddb.h" 116 117 #include <sys/param.h> 118 #include <sys/extent.h> 119 #include <sys/malloc.h> 120 #include <sys/systm.h> 121 #include <sys/device.h> 122 123 #include <uvm/uvm_extern.h> 124 125 #include <machine/bus.h> 126 #include <sparc64/sparc64/cache.h> 127 #include <sparc64/dev/iommureg.h> 128 #include <sparc64/dev/iommuvar.h> 129 130 #include <machine/autoconf.h> 131 #include <machine/cpu.h> 132 133 #ifdef DEBUG 134 #define IDB_BUSDMA 0x1 135 #define IDB_IOMMU 0x2 136 #define IDB_INFO 0x4 137 #define IDB_SYNC 0x8 138 int iommudebug = 0x0; 139 #define DPRINTF(l, s) do { if (iommudebug & l) printf s; } while (0) 140 #else 141 #define DPRINTF(l, s) 142 #endif 143 144 #define iommu_strbuf_flush(i,v) bus_space_write_8((i)->is_bustag, \ 145 (bus_space_handle_t)(u_long)&(i)->is_sb->strbuf_pgflush, 0, (v)) 146 static int iommu_strbuf_flush_done __P((struct iommu_state *)); 147 148 /* 149 * initialise the UltraSPARC IOMMU (SBUS or PCI): 150 * - allocate and setup the iotsb. 151 * - enable the IOMMU 152 * - initialise the streaming buffers (if they exist) 153 * - create a private DVMA map. 154 */ 155 void 156 iommu_init(name, is, tsbsize, iovabase) 157 char *name; 158 struct iommu_state *is; 159 int tsbsize; 160 u_int32_t iovabase; 161 { 162 psize_t size; 163 vaddr_t va; 164 paddr_t pa; 165 struct vm_page *m; 166 struct pglist mlist; 167 168 /* 169 * Setup the iommu. 170 * 171 * The sun4u iommu is part of the SBUS or PCI controller so we 172 * will deal with it here.. 173 * 174 * The IOMMU address space always ends at 0xffffe000, but the starting 175 * address depends on the size of the map. The map size is 1024 * 2 ^ 176 * is->is_tsbsize entries, where each entry is 8 bytes. The start of 177 * the map can be calculated by (0xffffe000 << (8 + is->is_tsbsize)). 178 */ 179 is->is_cr = (tsbsize << 16) | IOMMUCR_EN; 180 is->is_tsbsize = tsbsize; 181 is->is_dvmabase = iovabase; 182 if (iovabase == -1) is->is_dvmabase = IOTSB_VSTART(is->is_tsbsize); 183 184 /* 185 * Allocate memory for I/O pagetables. They need to be physically 186 * contiguous. 187 */ 188 189 size = NBPG<<(is->is_tsbsize); 190 TAILQ_INIT(&mlist); 191 if (uvm_pglistalloc((psize_t)size, (paddr_t)0, (paddr_t)-1, 192 (paddr_t)NBPG, (paddr_t)0, &mlist, 1, 0) != 0) 193 panic("iommu_init: no memory"); 194 195 va = uvm_km_valloc(kernel_map, size); 196 if (va == 0) 197 panic("iommu_init: no memory"); 198 is->is_tsb = (int64_t *)va; 199 200 m = TAILQ_FIRST(&mlist); 201 is->is_ptsb = VM_PAGE_TO_PHYS(m); 202 203 /* Map the pages */ 204 for (; m != NULL; m = TAILQ_NEXT(m,pageq)) { 205 pa = VM_PAGE_TO_PHYS(m); 206 pmap_enter(pmap_kernel(), va, pa | PMAP_NVC, 207 VM_PROT_READ|VM_PROT_WRITE, 208 VM_PROT_READ|VM_PROT_WRITE|PMAP_WIRED); 209 va += NBPG; 210 } 211 pmap_update(); 212 bzero(is->is_tsb, size); 213 214 #ifdef DEBUG 215 if (iommudebug & IDB_INFO) 216 { 217 /* Probe the iommu */ 218 struct iommureg *regs = is->is_iommu; 219 220 printf("iommu regs at: cr=%lx tsb=%lx flush=%lx\n", 221 (u_long)®s->iommu_cr, 222 (u_long)®s->iommu_tsb, 223 (u_long)®s->iommu_flush); 224 printf("iommu cr=%llx tsb=%llx\n", (unsigned long long)regs->iommu_cr, (unsigned long long)regs->iommu_tsb); 225 printf("TSB base %p phys %llx\n", (void *)is->is_tsb, (unsigned long long)is->is_ptsb); 226 delay(1000000); /* 1 s */ 227 } 228 #endif 229 230 /* 231 * Initialize streaming buffer, if it is there. 232 */ 233 if (is->is_sb) 234 (void)pmap_extract(pmap_kernel(), (vaddr_t)&is->is_flush, 235 (paddr_t *)&is->is_flushpa); 236 237 /* 238 * now actually start up the IOMMU 239 */ 240 iommu_reset(is); 241 242 /* 243 * Now all the hardware's working we need to allocate a dvma map. 244 */ 245 printf("DVMA map: %x to %x\n", 246 (unsigned int)is->is_dvmabase, 247 (unsigned int)(is->is_dvmabase+(size<<10))); 248 is->is_dvmamap = extent_create(name, 249 is->is_dvmabase, (u_long)IOTSB_VEND, 250 M_DEVBUF, 0, 0, EX_NOWAIT); 251 } 252 253 /* 254 * Streaming buffers don't exist on the UltraSPARC IIi; we should have 255 * detected that already and disabled them. If not, we will notice that 256 * they aren't there when the STRBUF_EN bit does not remain. 257 */ 258 void 259 iommu_reset(is) 260 struct iommu_state *is; 261 { 262 263 /* Need to do 64-bit stores */ 264 bus_space_write_8(is->is_bustag, 265 (bus_space_handle_t)(u_long)&is->is_iommu->iommu_tsb, 266 0, is->is_ptsb); 267 /* Enable IOMMU in diagnostic mode */ 268 bus_space_write_8(is->is_bustag, 269 (bus_space_handle_t)(u_long)&is->is_iommu->iommu_cr, 0, 270 is->is_cr|IOMMUCR_DE); 271 272 273 if (!is->is_sb) 274 return; 275 276 /* Enable diagnostics mode? */ 277 bus_space_write_8(is->is_bustag, 278 (bus_space_handle_t)(u_long)&is->is_sb->strbuf_ctl, 279 0, STRBUF_EN); 280 281 /* No streaming buffers? Disable them */ 282 if (bus_space_read_8(is->is_bustag, 283 (bus_space_handle_t)(u_long)&is->is_sb->strbuf_ctl, 284 0) == 0) 285 is->is_sb = 0; 286 } 287 288 /* 289 * Here are the iommu control routines. 290 */ 291 void 292 iommu_enter(is, va, pa, flags) 293 struct iommu_state *is; 294 vaddr_t va; 295 int64_t pa; 296 int flags; 297 { 298 int64_t tte; 299 300 #ifdef DIAGNOSTIC 301 if (va < is->is_dvmabase) 302 panic("iommu_enter: va %#lx not in DVMA space", va); 303 #endif 304 305 tte = MAKEIOTTE(pa, !(flags&BUS_DMA_NOWRITE), !(flags&BUS_DMA_NOCACHE), 306 (flags&BUS_DMA_STREAMING)); 307 308 /* Is the streamcache flush really needed? */ 309 if (is->is_sb) { 310 iommu_strbuf_flush(is, va); 311 iommu_strbuf_flush_done(is); 312 } 313 DPRINTF(IDB_IOMMU, ("Clearing TSB slot %d for va %p\n", 314 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va)); 315 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = tte; 316 bus_space_write_8(is->is_bustag, (bus_space_handle_t)(u_long) 317 &is->is_iommu->iommu_flush, 0, va); 318 DPRINTF(IDB_IOMMU, ("iommu_enter: va %lx pa %lx TSB[%lx]@%p=%lx\n", 319 va, (long)pa, (u_long)IOTSBSLOT(va,is->is_tsbsize), 320 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 321 (u_long)tte)); 322 } 323 324 /* 325 * iommu_remove: removes mappings created by iommu_enter 326 * 327 * Only demap from IOMMU if flag is set. 328 * 329 * XXX: this function needs better internal error checking. 330 */ 331 void 332 iommu_remove(is, va, len) 333 struct iommu_state *is; 334 vaddr_t va; 335 size_t len; 336 { 337 338 #ifdef DIAGNOSTIC 339 if (va < is->is_dvmabase) 340 panic("iommu_remove: va 0x%lx not in DVMA space", (u_long)va); 341 if ((long)(va + len) < (long)va) 342 panic("iommu_remove: va 0x%lx + len 0x%lx wraps", 343 (long) va, (long) len); 344 if (len & ~0xfffffff) 345 panic("iommu_remove: rediculous len 0x%lx", (u_long)len); 346 #endif 347 348 va = trunc_page(va); 349 DPRINTF(IDB_IOMMU, ("iommu_remove: va %lx TSB[%lx]@%p\n", 350 va, (u_long)IOTSBSLOT(va,is->is_tsbsize), 351 &is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)])); 352 while (len > 0) { 353 DPRINTF(IDB_IOMMU, ("iommu_remove: clearing TSB slot %d for va %p size %lx\n", 354 (int)IOTSBSLOT(va,is->is_tsbsize), (void *)(u_long)va, (u_long)len)); 355 if (is->is_sb) { 356 DPRINTF(IDB_IOMMU, ("iommu_remove: flushing va %p TSB[%lx]@%p=%lx, %lu bytes left\n", 357 (void *)(u_long)va, (long)IOTSBSLOT(va,is->is_tsbsize), 358 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 359 (long)(is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)]), 360 (u_long)len)); 361 iommu_strbuf_flush(is, va); 362 if (len <= NBPG) 363 iommu_strbuf_flush_done(is); 364 DPRINTF(IDB_IOMMU, ("iommu_remove: flushed va %p TSB[%lx]@%p=%lx, %lu bytes left\n", 365 (void *)(u_long)va, (long)IOTSBSLOT(va,is->is_tsbsize), 366 (void *)(u_long)&is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)], 367 (long)(is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)]), 368 (u_long)len)); 369 } 370 371 if (len <= NBPG) 372 len = 0; 373 else 374 len -= NBPG; 375 376 is->is_tsb[IOTSBSLOT(va,is->is_tsbsize)] = 0; 377 bus_space_write_8(is->is_bustag, (bus_space_handle_t)(u_long) 378 &is->is_iommu->iommu_flush, 0, va); 379 va += NBPG; 380 } 381 } 382 383 static int 384 iommu_strbuf_flush_done(is) 385 struct iommu_state *is; 386 { 387 struct timeval cur, flushtimeout; 388 389 #define BUMPTIME(t, usec) { \ 390 register volatile struct timeval *tp = (t); \ 391 register long us; \ 392 \ 393 tp->tv_usec = us = tp->tv_usec + (usec); \ 394 if (us >= 1000000) { \ 395 tp->tv_usec = us - 1000000; \ 396 tp->tv_sec++; \ 397 } \ 398 } 399 400 if (!is->is_sb) 401 return (0); 402 403 /* 404 * Streaming buffer flushes: 405 * 406 * 1 Tell strbuf to flush by storing va to strbuf_pgflush. If 407 * we're not on a cache line boundary (64-bits): 408 * 2 Store 0 in flag 409 * 3 Store pointer to flag in flushsync 410 * 4 wait till flushsync becomes 0x1 411 * 412 * If it takes more than .5 sec, something 413 * went wrong. 414 */ 415 416 is->is_flush = 0; 417 membar_sync(); /* #StoreStore is prolly enuf. */ 418 bus_space_write_8(is->is_bustag, (bus_space_handle_t)(u_long) 419 &is->is_sb->strbuf_flushsync, 0, is->is_flushpa); 420 membar_sync(); /* Prolly not needed at all. */ 421 422 microtime(&flushtimeout); 423 cur = flushtimeout; 424 BUMPTIME(&flushtimeout, 500000); /* 1/2 sec */ 425 426 DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flush = %lx at va = %lx pa = %lx now=%lx:%lx until = %lx:%lx\n", 427 (long)is->is_flush, (long)&is->is_flush, 428 (long)is->is_flushpa, cur.tv_sec, cur.tv_usec, 429 flushtimeout.tv_sec, flushtimeout.tv_usec)); 430 /* Bypass non-coherent D$ */ 431 while (!ldxa(is->is_flushpa, ASI_PHYS_CACHED) && 432 ((cur.tv_sec <= flushtimeout.tv_sec) && 433 (cur.tv_usec <= flushtimeout.tv_usec))) 434 microtime(&cur); 435 436 #ifdef DIAGNOSTIC 437 if (!ldxa(is->is_flushpa, ASI_PHYS_CACHED)) { 438 printf("iommu_strbuf_flush_done: flush timeout %p at %p\n", 439 (void *)(u_long)is->is_flush, 440 (void *)(u_long)is->is_flushpa); /* panic? */ 441 #ifdef DDB 442 Debugger(); 443 #endif 444 } 445 #endif 446 DPRINTF(IDB_IOMMU, ("iommu_strbuf_flush_done: flushed\n")); 447 return (is->is_flush); 448 } 449 450 /* 451 * IOMMU DVMA operations, common to SBUS and PCI. 452 */ 453 int 454 iommu_dvmamap_load(t, is, map, buf, buflen, p, flags) 455 bus_dma_tag_t t; 456 struct iommu_state *is; 457 bus_dmamap_t map; 458 void *buf; 459 bus_size_t buflen; 460 struct proc *p; 461 int flags; 462 { 463 int s; 464 int err; 465 bus_size_t sgsize; 466 paddr_t curaddr; 467 u_long dvmaddr; 468 bus_size_t align, boundary; 469 vaddr_t vaddr = (vaddr_t)buf; 470 pmap_t pmap; 471 472 if (map->dm_nsegs) { 473 /* Already in use?? */ 474 #ifdef DIAGNOSTIC 475 printf("iommu_dvmamap_load: map still in use\n"); 476 #endif 477 bus_dmamap_unload(t, map); 478 } 479 /* 480 * Make sure that on error condition we return "no valid mappings". 481 */ 482 map->dm_nsegs = 0; 483 484 if (buflen > map->_dm_size) { 485 DPRINTF(IDB_BUSDMA, 486 ("iommu_dvmamap_load(): error %d > %d -- " 487 "map size exceeded!\n", (int)buflen, (int)map->_dm_size)); 488 return (EINVAL); 489 } 490 491 sgsize = round_page(buflen + ((int)vaddr & PGOFSET)); 492 493 /* 494 * A boundary presented to bus_dmamem_alloc() takes precedence 495 * over boundary in the map. 496 */ 497 if ((boundary = (map->dm_segs[0]._ds_boundary)) == 0) 498 boundary = map->_dm_boundary; 499 align = max(map->dm_segs[0]._ds_align, NBPG); 500 s = splhigh(); 501 err = extent_alloc(is->is_dvmamap, sgsize, align, 502 boundary, EX_NOWAIT|EX_BOUNDZERO, (u_long *)&dvmaddr); 503 splx(s); 504 505 #ifdef DEBUG 506 if (err || (dvmaddr == (bus_addr_t)-1)) 507 { 508 printf("iommu_dvmamap_load(): extent_alloc(%d, %x) failed!\n", 509 (int)sgsize, flags); 510 Debugger(); 511 } 512 #endif 513 if (err != 0) 514 return (err); 515 516 if (dvmaddr == (bus_addr_t)-1) 517 return (ENOMEM); 518 519 /* 520 * We always use just one segment. 521 */ 522 map->dm_mapsize = buflen; 523 map->dm_nsegs = 1; 524 map->dm_segs[0].ds_addr = dvmaddr + (vaddr & PGOFSET); 525 map->dm_segs[0].ds_len = buflen; 526 527 if (p != NULL) 528 pmap = p->p_vmspace->vm_map.pmap; 529 else 530 pmap = pmap_kernel(); 531 532 dvmaddr = trunc_page(map->dm_segs[0].ds_addr); 533 for (; buflen > 0; ) { 534 /* 535 * Get the physical address for this page. 536 */ 537 if (pmap_extract(pmap, (vaddr_t)vaddr, &curaddr) == FALSE) { 538 bus_dmamap_unload(t, map); 539 return (-1); 540 } 541 542 /* 543 * Compute the segment size, and adjust counts. 544 */ 545 sgsize = NBPG - ((u_long)vaddr & PGOFSET); 546 if (buflen < sgsize) 547 sgsize = buflen; 548 549 DPRINTF(IDB_BUSDMA, 550 ("iommu_dvmamap_load: map %p loading va %p " 551 "dva %lx at pa %lx\n", 552 map, (void *)vaddr, (long)dvmaddr, 553 (long)(curaddr&~(NBPG-1)))); 554 iommu_enter(is, trunc_page(dvmaddr), trunc_page(curaddr), 555 flags); 556 557 dvmaddr += PAGE_SIZE; 558 vaddr += sgsize; 559 buflen -= sgsize; 560 } 561 return (0); 562 } 563 564 565 void 566 iommu_dvmamap_unload(t, is, map) 567 bus_dma_tag_t t; 568 struct iommu_state *is; 569 bus_dmamap_t map; 570 { 571 vaddr_t addr, offset; 572 size_t len; 573 int error, s, i; 574 bus_addr_t dvmaddr; 575 bus_size_t sgsize; 576 paddr_t pa; 577 578 dvmaddr = (map->dm_segs[0].ds_addr & ~PGOFSET); 579 pa = 0; 580 sgsize = 0; 581 for (i = 0; i<map->dm_nsegs; i++) { 582 583 addr = trunc_page(map->dm_segs[i].ds_addr); 584 offset = map->dm_segs[i].ds_addr & PGOFSET; 585 len = map->dm_segs[i].ds_len; 586 if (len == 0 || addr == 0) 587 printf("iommu_dvmamap_unload: map = %p, i = %d, len = %d, addr = %lx\n", 588 map, (int)i, (int)len, (unsigned long)addr); 589 590 DPRINTF(IDB_BUSDMA, 591 ("iommu_dvmamap_unload: map %p removing va %lx size %lx\n", 592 map, (long)addr, (long)len)); 593 iommu_remove(is, addr, len); 594 595 if (trunc_page(pa) == addr) 596 sgsize += trunc_page(len + offset); 597 else 598 sgsize += round_page(len + offset); 599 pa = addr + offset + len; 600 601 } 602 /* Flush the caches */ 603 bus_dmamap_unload(t->_parent, map); 604 605 /* Mark the mappings as invalid. */ 606 map->dm_mapsize = 0; 607 map->dm_nsegs = 0; 608 609 s = splhigh(); 610 error = extent_free(is->is_dvmamap, dvmaddr, sgsize, EX_NOWAIT); 611 splx(s); 612 if (error != 0) 613 printf("warning: %qd of DVMA space lost\n", (long long)sgsize); 614 } 615 616 617 int 618 iommu_dvmamap_load_raw(t, is, map, segs, nsegs, flags, size) 619 bus_dma_tag_t t; 620 struct iommu_state *is; 621 bus_dmamap_t map; 622 bus_dma_segment_t *segs; 623 int nsegs; 624 int flags; 625 bus_size_t size; 626 { 627 struct vm_page *m; 628 int i, s; 629 int left; 630 int err; 631 bus_size_t sgsize; 632 paddr_t pa; 633 bus_size_t boundary, align; 634 u_long dvmaddr; 635 struct pglist *mlist; 636 int pagesz = PAGE_SIZE; 637 638 if (map->dm_nsegs) { 639 /* Already in use?? */ 640 #ifdef DIAGNOSTIC 641 printf("iommu_dvmamap_load_raw: map still in use\n"); 642 #endif 643 bus_dmamap_unload(t, map); 644 } 645 /* 646 * Make sure that on error condition we return "no valid mappings". 647 */ 648 map->dm_nsegs = 0; 649 /* Count up the total number of pages we need */ 650 pa = segs[0].ds_addr; 651 sgsize = 0; 652 for (i=0; i<nsegs; i++) { 653 if (round_page(pa) != round_page(segs[i].ds_addr)) 654 sgsize = round_page(sgsize); 655 sgsize += segs[i].ds_len; 656 pa = segs[i].ds_addr + segs[i].ds_len; 657 } 658 sgsize = round_page(sgsize); 659 660 /* 661 * A boundary presented to bus_dmamem_alloc() takes precedence 662 * over boundary in the map. 663 */ 664 if ((boundary = segs[0]._ds_boundary) == 0) 665 boundary = map->_dm_boundary; 666 667 align = max(segs[0]._ds_align, NBPG); 668 s = splhigh(); 669 err = extent_alloc(is->is_dvmamap, sgsize, align, boundary, 670 ((flags & BUS_DMA_NOWAIT) == 0 ? EX_WAITOK : EX_NOWAIT)|EX_BOUNDZERO, 671 (u_long *)&dvmaddr); 672 splx(s); 673 674 if (err != 0) 675 return (err); 676 677 #ifdef DEBUG 678 if (dvmaddr == (bus_addr_t)-1) 679 { 680 printf("iommu_dvmamap_load_raw(): extent_alloc(%d, %x) failed!\n", 681 (int)sgsize, flags); 682 Debugger(); 683 } 684 #endif 685 if (dvmaddr == (bus_addr_t)-1) 686 return (ENOMEM); 687 688 if ((mlist = segs[0]._ds_mlist) == NULL) { 689 u_long prev_va = NULL; 690 /* 691 * This segs is made up of individual physical pages, 692 * probably by _bus_dmamap_load_uio() or 693 * _bus_dmamap_load_mbuf(). Ignore the mlist and 694 * load each segment individually. 695 */ 696 697 /* We'll never end up with less segments than we got as input. 698 this gives us a chance to fail quickly */ 699 if (nsegs > map->_dm_segcnt) 700 return (E2BIG); 701 702 i = 0; 703 dvmaddr += (segs[i].ds_addr & PGOFSET); 704 map->dm_segs[i].ds_addr = dvmaddr; 705 map->dm_segs[i].ds_len = left = segs[i].ds_len; 706 pa = segs[i].ds_addr; 707 708 while (left > 0) { 709 int incr; 710 711 if (sgsize == 0) 712 panic("iommu_dmamap_load_raw: size botch"); 713 DPRINTF(IDB_BUSDMA, 714 ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n", 715 map, (long)dvmaddr, (long)(pa))); 716 /* Enter it if we haven't before. */ 717 if (prev_va != trunc_page(dvmaddr)) 718 iommu_enter(is, prev_va = trunc_page(dvmaddr), 719 trunc_page(pa), flags); 720 incr = min(pagesz, left); 721 dvmaddr += incr; 722 pa += incr; 723 left -= incr; 724 725 /* Next segment */ 726 if (left <= 0 && ++i < nsegs) { 727 u_long offset; 728 729 /* 730 * If the two segs are on different physical pages 731 * move to a new virtual page. 732 */ 733 offset = (segs[i].ds_addr & PGOFSET); 734 if (trunc_page(pa) != trunc_page(segs[i].ds_addr)) 735 dvmaddr += NBPG; 736 737 pa = segs[i].ds_addr; 738 dvmaddr = trunc_page(dvmaddr) + offset; 739 740 map->dm_segs[i].ds_len = left = segs[i].ds_len; 741 map->dm_segs[i].ds_addr = dvmaddr; 742 } 743 } 744 map->dm_nsegs = i; 745 746 /* bail out if we created more segments than the dmamap is 747 allowed to carry */ 748 if (i > map->_dm_segcnt) { 749 iommu_dvmamap_unload(t, is, map); 750 return (E2BIG); 751 } 752 753 return (0); 754 } 755 /* 756 * This was allocated with bus_dmamem_alloc. We only 757 * have one segment, and the pages are on an `mlist'. 758 */ 759 map->dm_mapsize = size; 760 i = 0; 761 map->dm_segs[0].ds_addr = dvmaddr; 762 map->dm_segs[0].ds_len = size; 763 764 for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq)) { 765 if (sgsize == 0) 766 panic("iommu_dmamap_load_raw: size botch"); 767 pa = VM_PAGE_TO_PHYS(m); 768 769 DPRINTF(IDB_BUSDMA, 770 ("iommu_dvmamap_load_raw: map %p loading va %lx at pa %lx\n", 771 map, (long)dvmaddr, (long)(pa))); 772 iommu_enter(is, dvmaddr, pa, flags); 773 774 dvmaddr += pagesz; 775 sgsize -= pagesz; 776 } 777 map->dm_nsegs = i; 778 return (0); 779 } 780 781 void 782 iommu_dvmamap_sync(t, is, map, offset, len, ops) 783 bus_dma_tag_t t; 784 struct iommu_state *is; 785 bus_dmamap_t map; 786 bus_addr_t offset; 787 bus_size_t len; 788 int ops; 789 { 790 vaddr_t va = map->dm_segs[0].ds_addr + offset; 791 792 /* 793 * We only support one DMA segment; supporting more makes this code 794 * too unweildy. 795 */ 796 797 if (ops & BUS_DMASYNC_PREREAD) { 798 DPRINTF(IDB_SYNC, 799 ("iommu_dvmamap_sync: syncing va %p len %lu " 800 "BUS_DMASYNC_PREREAD\n", (void *)(u_long)va, (u_long)len)); 801 802 /* Nothing to do */; 803 } 804 if (ops & BUS_DMASYNC_POSTREAD) { 805 DPRINTF(IDB_SYNC, 806 ("iommu_dvmamap_sync: syncing va %p len %lu " 807 "BUS_DMASYNC_POSTREAD\n", (void *)(u_long)va, (u_long)len)); 808 /* if we have a streaming buffer, flush it here first */ 809 if (is->is_sb) 810 while (len > 0) { 811 DPRINTF(IDB_BUSDMA, 812 ("iommu_dvmamap_sync: flushing va %p, %lu " 813 "bytes left\n", (void *)(u_long)va, (u_long)len)); 814 iommu_strbuf_flush(is, va); 815 if (len <= NBPG) { 816 iommu_strbuf_flush_done(is); 817 len = 0; 818 } else 819 len -= NBPG; 820 va += NBPG; 821 } 822 } 823 if (ops & BUS_DMASYNC_PREWRITE) { 824 DPRINTF(IDB_SYNC, 825 ("iommu_dvmamap_sync: syncing va %p len %lu " 826 "BUS_DMASYNC_PREWRITE\n", (void *)(u_long)va, (u_long)len)); 827 /* if we have a streaming buffer, flush it here first */ 828 if (is->is_sb) 829 while (len > 0) { 830 DPRINTF(IDB_BUSDMA, 831 ("iommu_dvmamap_sync: flushing va %p, %lu " 832 "bytes left\n", (void *)(u_long)va, (u_long)len)); 833 iommu_strbuf_flush(is, va); 834 if (len <= NBPG) { 835 iommu_strbuf_flush_done(is); 836 len = 0; 837 } else 838 len -= NBPG; 839 va += NBPG; 840 } 841 } 842 if (ops & BUS_DMASYNC_POSTWRITE) { 843 DPRINTF(IDB_SYNC, 844 ("iommu_dvmamap_sync: syncing va %p len %lu " 845 "BUS_DMASYNC_POSTWRITE\n", (void *)(u_long)va, (u_long)len)); 846 /* Nothing to do */; 847 } 848 } 849 850 int 851 iommu_dvmamem_alloc(t, is, size, alignment, boundary, segs, nsegs, rsegs, flags) 852 bus_dma_tag_t t; 853 struct iommu_state *is; 854 bus_size_t size, alignment, boundary; 855 bus_dma_segment_t *segs; 856 int nsegs; 857 int *rsegs; 858 int flags; 859 { 860 861 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_alloc: sz %llx align %llx bound %llx " 862 "segp %p flags %d\n", (unsigned long long)size, 863 (unsigned long long)alignment, (unsigned long long)boundary, 864 segs, flags)); 865 return (bus_dmamem_alloc(t->_parent, size, alignment, boundary, 866 segs, nsegs, rsegs, flags|BUS_DMA_DVMA)); 867 } 868 869 void 870 iommu_dvmamem_free(t, is, segs, nsegs) 871 bus_dma_tag_t t; 872 struct iommu_state *is; 873 bus_dma_segment_t *segs; 874 int nsegs; 875 { 876 877 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_free: segp %p nsegs %d\n", 878 segs, nsegs)); 879 bus_dmamem_free(t->_parent, segs, nsegs); 880 } 881 882 /* 883 * Map the DVMA mappings into the kernel pmap. 884 * Check the flags to see whether we're streaming or coherent. 885 */ 886 int 887 iommu_dvmamem_map(t, is, segs, nsegs, size, kvap, flags) 888 bus_dma_tag_t t; 889 struct iommu_state *is; 890 bus_dma_segment_t *segs; 891 int nsegs; 892 size_t size; 893 caddr_t *kvap; 894 int flags; 895 { 896 struct vm_page *m; 897 vaddr_t va; 898 bus_addr_t addr; 899 struct pglist *mlist; 900 int cbit; 901 902 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: segp %p nsegs %d size %lx\n", 903 segs, nsegs, size)); 904 905 /* 906 * Allocate some space in the kernel map, and then map these pages 907 * into this space. 908 */ 909 size = round_page(size); 910 va = uvm_km_valloc(kernel_map, size); 911 if (va == 0) 912 return (ENOMEM); 913 914 *kvap = (caddr_t)va; 915 916 /* 917 * digest flags: 918 */ 919 cbit = 0; 920 if (flags & BUS_DMA_COHERENT) /* Disable vcache */ 921 cbit |= PMAP_NVC; 922 if (flags & BUS_DMA_NOCACHE) /* sideffects */ 923 cbit |= PMAP_NC; 924 925 /* 926 * Now take this and map it into the CPU. 927 */ 928 mlist = segs[0]._ds_mlist; 929 for (m = mlist->tqh_first; m != NULL; m = m->pageq.tqe_next) { 930 #ifdef DIAGNOSTIC 931 if (size == 0) 932 panic("iommu_dvmamem_map: size botch"); 933 #endif 934 addr = VM_PAGE_TO_PHYS(m); 935 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_map: " 936 "mapping va %lx at %llx\n", va, (unsigned long long)addr | cbit)); 937 pmap_enter(pmap_kernel(), va, addr | cbit, 938 VM_PROT_READ | VM_PROT_WRITE, 939 VM_PROT_READ | VM_PROT_WRITE | PMAP_WIRED); 940 va += PAGE_SIZE; 941 size -= PAGE_SIZE; 942 } 943 pmap_update(); 944 945 return (0); 946 } 947 948 /* 949 * Unmap DVMA mappings from kernel 950 */ 951 void 952 iommu_dvmamem_unmap(t, is, kva, size) 953 bus_dma_tag_t t; 954 struct iommu_state *is; 955 caddr_t kva; 956 size_t size; 957 { 958 959 DPRINTF(IDB_BUSDMA, ("iommu_dvmamem_unmap: kvm %p size %lx\n", 960 kva, size)); 961 962 #ifdef DIAGNOSTIC 963 if ((u_long)kva & PGOFSET) 964 panic("iommu_dvmamem_unmap"); 965 #endif 966 967 size = round_page(size); 968 pmap_remove(pmap_kernel(), (vaddr_t)kva, size); 969 pmap_update(); 970 #if 0 971 /* 972 * XXX ? is this necessary? i think so and i think other 973 * implementations are missing it. 974 */ 975 uvm_km_free(kernel_map, (vaddr_t)kva, size); 976 #endif 977 } 978