1 /* $NetBSD: bus_dma.c,v 1.59 2005/07/30 17:25:51 mhitch Exp $ */ 2 3 /*- 4 * Copyright (c) 1997, 1998 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 3. All advertising materials mentioning features or use of this software 20 * must display the following acknowledgement: 21 * This product includes software developed by the NetBSD 22 * Foundation, Inc. and its contributors. 23 * 4. Neither the name of The NetBSD Foundation nor the names of its 24 * contributors may be used to endorse or promote products derived 25 * from this software without specific prior written permission. 26 * 27 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 28 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 29 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 30 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 31 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 32 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 33 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 34 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 35 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 36 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 37 * POSSIBILITY OF SUCH DAMAGE. 38 */ 39 40 #include <sys/cdefs.h> /* RCS ID & Copyright macro defns */ 41 42 __KERNEL_RCSID(0, "$NetBSD: bus_dma.c,v 1.59 2005/07/30 17:25:51 mhitch Exp $"); 43 44 #include <sys/param.h> 45 #include <sys/systm.h> 46 #include <sys/kernel.h> 47 #include <sys/device.h> 48 #include <sys/malloc.h> 49 #include <sys/proc.h> 50 #include <sys/mbuf.h> 51 52 #include <uvm/uvm_extern.h> 53 54 #define _ALPHA_BUS_DMA_PRIVATE 55 #include <machine/bus.h> 56 #include <machine/intr.h> 57 58 int _bus_dmamap_load_buffer_direct(bus_dma_tag_t, 59 bus_dmamap_t, void *, bus_size_t, struct proc *, int, 60 paddr_t *, int *, int); 61 62 extern paddr_t avail_start, avail_end; /* from pmap.c */ 63 64 /* 65 * Common function for DMA map creation. May be called by bus-specific 66 * DMA map creation functions. 67 */ 68 int 69 _bus_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, 70 bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp) 71 { 72 struct alpha_bus_dmamap *map; 73 void *mapstore; 74 size_t mapsize; 75 76 /* 77 * Allocate and initialize the DMA map. The end of the map 78 * is a variable-sized array of segments, so we allocate enough 79 * room for them in one shot. 80 * 81 * Note we don't preserve the WAITOK or NOWAIT flags. Preservation 82 * of ALLOCNOW notifies others that we've reserved these resources, 83 * and they are not to be freed. 84 * 85 * The bus_dmamap_t includes one bus_dma_segment_t, hence 86 * the (nsegments - 1). 87 */ 88 mapsize = sizeof(struct alpha_bus_dmamap) + 89 (sizeof(bus_dma_segment_t) * (nsegments - 1)); 90 if ((mapstore = malloc(mapsize, M_DMAMAP, 91 (flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK)) == NULL) 92 return (ENOMEM); 93 94 memset(mapstore, 0, mapsize); 95 map = (struct alpha_bus_dmamap *)mapstore; 96 map->_dm_size = size; 97 map->_dm_segcnt = nsegments; 98 map->_dm_maxmaxsegsz = maxsegsz; 99 if (t->_boundary != 0 && t->_boundary < boundary) 100 map->_dm_boundary = t->_boundary; 101 else 102 map->_dm_boundary = boundary; 103 map->_dm_flags = flags & ~(BUS_DMA_WAITOK|BUS_DMA_NOWAIT); 104 map->dm_maxsegsz = maxsegsz; 105 map->dm_mapsize = 0; /* no valid mappings */ 106 map->dm_nsegs = 0; 107 map->_dm_window = NULL; 108 109 *dmamp = map; 110 return (0); 111 } 112 113 /* 114 * Common function for DMA map destruction. May be called by bus-specific 115 * DMA map destruction functions. 116 */ 117 void 118 _bus_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map) 119 { 120 121 free(map, M_DMAMAP); 122 } 123 124 /* 125 * Utility function to load a linear buffer. lastaddrp holds state 126 * between invocations (for multiple-buffer loads). segp contains 127 * the starting segment on entrance, and the ending segment on exit. 128 * first indicates if this is the first invocation of this function. 129 */ 130 int 131 _bus_dmamap_load_buffer_direct(bus_dma_tag_t t, bus_dmamap_t map, 132 void *buf, size_t buflen, struct proc *p, int flags, paddr_t *lastaddrp, 133 int *segp, int first) 134 { 135 bus_size_t sgsize; 136 bus_addr_t curaddr, lastaddr, baddr, bmask; 137 vaddr_t vaddr = (vaddr_t)buf; 138 int seg; 139 140 lastaddr = *lastaddrp; 141 bmask = ~(map->_dm_boundary - 1); 142 143 for (seg = *segp; buflen > 0 ; ) { 144 /* 145 * Get the physical address for this segment. 146 */ 147 if (p != NULL) 148 (void) pmap_extract(p->p_vmspace->vm_map.pmap, 149 vaddr, &curaddr); 150 else 151 curaddr = vtophys(vaddr); 152 153 /* 154 * If we're beyond the current DMA window, indicate 155 * that and try to fall back into SGMAPs. 156 */ 157 if (t->_wsize != 0 && curaddr >= t->_wsize) 158 return (EINVAL); 159 160 curaddr |= t->_wbase; 161 162 /* 163 * Compute the segment size, and adjust counts. 164 */ 165 sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET); 166 if (buflen < sgsize) 167 sgsize = buflen; 168 if (map->dm_maxsegsz < sgsize) 169 sgsize = map->dm_maxsegsz; 170 171 /* 172 * Make sure we don't cross any boundaries. 173 */ 174 if (map->_dm_boundary > 0) { 175 baddr = (curaddr + map->_dm_boundary) & bmask; 176 if (sgsize > (baddr - curaddr)) 177 sgsize = (baddr - curaddr); 178 } 179 180 /* 181 * Insert chunk into a segment, coalescing with 182 * the previous segment if possible. 183 */ 184 if (first) { 185 map->dm_segs[seg].ds_addr = curaddr; 186 map->dm_segs[seg].ds_len = sgsize; 187 first = 0; 188 } else { 189 if ((map->_dm_flags & DMAMAP_NO_COALESCE) == 0 && 190 curaddr == lastaddr && 191 (map->dm_segs[seg].ds_len + sgsize) <= 192 map->dm_maxsegsz && 193 (map->_dm_boundary == 0 || 194 (map->dm_segs[seg].ds_addr & bmask) == 195 (curaddr & bmask))) 196 map->dm_segs[seg].ds_len += sgsize; 197 else { 198 if (++seg >= map->_dm_segcnt) 199 break; 200 map->dm_segs[seg].ds_addr = curaddr; 201 map->dm_segs[seg].ds_len = sgsize; 202 } 203 } 204 205 lastaddr = curaddr + sgsize; 206 vaddr += sgsize; 207 buflen -= sgsize; 208 } 209 210 *segp = seg; 211 *lastaddrp = lastaddr; 212 213 /* 214 * Did we fit? 215 */ 216 if (buflen != 0) { 217 /* 218 * If there is a chained window, we will automatically 219 * fall back to it. 220 */ 221 return (EFBIG); /* XXX better return value here? */ 222 } 223 224 return (0); 225 } 226 227 /* 228 * Common function for loading a direct-mapped DMA map with a linear 229 * buffer. Called by bus-specific DMA map load functions with the 230 * OR value appropriate for indicating "direct-mapped" for that 231 * chipset. 232 */ 233 int 234 _bus_dmamap_load_direct(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 235 bus_size_t buflen, struct proc *p, int flags) 236 { 237 paddr_t lastaddr; 238 int seg, error; 239 240 /* 241 * Make sure that on error condition we return "no valid mappings". 242 */ 243 map->dm_mapsize = 0; 244 map->dm_nsegs = 0; 245 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz); 246 KASSERT((map->_dm_flags & (BUS_DMA_READ|BUS_DMA_WRITE)) == 0); 247 248 if (buflen > map->_dm_size) 249 return (EINVAL); 250 251 seg = 0; 252 error = _bus_dmamap_load_buffer_direct(t, map, buf, buflen, 253 p, flags, &lastaddr, &seg, 1); 254 if (error == 0) { 255 map->dm_mapsize = buflen; 256 map->dm_nsegs = seg + 1; 257 map->_dm_window = t; 258 } else if (t->_next_window != NULL) { 259 /* 260 * Give the next window a chance. 261 */ 262 error = bus_dmamap_load(t->_next_window, map, buf, buflen, 263 p, flags); 264 } 265 return (error); 266 } 267 268 /* 269 * Like _bus_dmamap_load_direct(), but for mbufs. 270 */ 271 int 272 _bus_dmamap_load_mbuf_direct(bus_dma_tag_t t, bus_dmamap_t map, 273 struct mbuf *m0, int flags) 274 { 275 paddr_t lastaddr; 276 int seg, error, first; 277 struct mbuf *m; 278 279 /* 280 * Make sure that on error condition we return "no valid mappings." 281 */ 282 map->dm_mapsize = 0; 283 map->dm_nsegs = 0; 284 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz); 285 KASSERT((map->_dm_flags & (BUS_DMA_READ|BUS_DMA_WRITE)) == 0); 286 287 #ifdef DIAGNOSTIC 288 if ((m0->m_flags & M_PKTHDR) == 0) 289 panic("_bus_dmamap_load_mbuf_direct: no packet header"); 290 #endif 291 292 if (m0->m_pkthdr.len > map->_dm_size) 293 return (EINVAL); 294 295 first = 1; 296 seg = 0; 297 error = 0; 298 for (m = m0; m != NULL && error == 0; m = m->m_next) { 299 if (m->m_len == 0) 300 continue; 301 /* XXX Could be better about coalescing. */ 302 /* XXX Doesn't check boundaries. */ 303 switch (m->m_flags & (M_EXT|M_EXT_CLUSTER)) { 304 case M_EXT|M_EXT_CLUSTER: 305 /* XXX KDASSERT */ 306 KASSERT(m->m_ext.ext_paddr != M_PADDR_INVALID); 307 lastaddr = m->m_ext.ext_paddr + 308 (m->m_data - m->m_ext.ext_buf); 309 have_addr: 310 if (first == 0 && 311 ++seg >= map->_dm_segcnt) { 312 error = EFBIG; 313 break; 314 } 315 316 /* 317 * If we're beyond the current DMA window, indicate 318 * that and try to fall back into SGMAPs. 319 */ 320 if (t->_wsize != 0 && lastaddr >= t->_wsize) { 321 error = EINVAL; 322 break; 323 } 324 lastaddr |= t->_wbase; 325 326 map->dm_segs[seg].ds_addr = lastaddr; 327 map->dm_segs[seg].ds_len = m->m_len; 328 lastaddr += m->m_len; 329 break; 330 331 case 0: 332 lastaddr = m->m_paddr + M_BUFOFFSET(m) + 333 (m->m_data - M_BUFADDR(m)); 334 goto have_addr; 335 336 default: 337 error = _bus_dmamap_load_buffer_direct(t, map, 338 m->m_data, m->m_len, NULL, flags, &lastaddr, 339 &seg, first); 340 } 341 first = 0; 342 } 343 if (error == 0) { 344 map->dm_mapsize = m0->m_pkthdr.len; 345 map->dm_nsegs = seg + 1; 346 map->_dm_window = t; 347 } else if (t->_next_window != NULL) { 348 /* 349 * Give the next window a chance. 350 */ 351 error = bus_dmamap_load_mbuf(t->_next_window, map, m0, flags); 352 } 353 return (error); 354 } 355 356 /* 357 * Like _bus_dmamap_load_direct(), but for uios. 358 */ 359 int 360 _bus_dmamap_load_uio_direct(bus_dma_tag_t t, bus_dmamap_t map, 361 struct uio *uio, int flags) 362 { 363 paddr_t lastaddr; 364 int seg, i, error, first; 365 bus_size_t minlen, resid; 366 struct proc *p = NULL; 367 struct iovec *iov; 368 caddr_t addr; 369 370 /* 371 * Make sure that on error condition we return "no valid mappings." 372 */ 373 map->dm_mapsize = 0; 374 map->dm_nsegs = 0; 375 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz); 376 KASSERT((map->_dm_flags & (BUS_DMA_READ|BUS_DMA_WRITE)) == 0); 377 378 resid = uio->uio_resid; 379 iov = uio->uio_iov; 380 381 if (uio->uio_segflg == UIO_USERSPACE) { 382 p = uio->uio_procp; 383 #ifdef DIAGNOSTIC 384 if (p == NULL) 385 panic("_bus_dmamap_load_uio_direct: " 386 "USERSPACE but no proc"); 387 #endif 388 } 389 390 first = 1; 391 seg = 0; 392 error = 0; 393 for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) { 394 /* 395 * Now at the first iovec to load. Load each iovec 396 * until we have exhausted the residual count. 397 */ 398 minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len; 399 addr = (caddr_t)iov[i].iov_base; 400 401 error = _bus_dmamap_load_buffer_direct(t, map, 402 addr, minlen, p, flags, &lastaddr, &seg, first); 403 first = 0; 404 405 resid -= minlen; 406 } 407 if (error == 0) { 408 map->dm_mapsize = uio->uio_resid; 409 map->dm_nsegs = seg + 1; 410 map->_dm_window = t; 411 } else if (t->_next_window != NULL) { 412 /* 413 * Give the next window a chance. 414 */ 415 error = bus_dmamap_load_uio(t->_next_window, map, uio, flags); 416 } 417 return (error); 418 } 419 420 /* 421 * Like _bus_dmamap_load_direct(), but for raw memory. 422 */ 423 int 424 _bus_dmamap_load_raw_direct(bus_dma_tag_t t, bus_dmamap_t map, 425 bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags) 426 { 427 428 panic("_bus_dmamap_load_raw_direct: not implemented"); 429 } 430 431 /* 432 * Common function for unloading a DMA map. May be called by 433 * chipset-specific DMA map unload functions. 434 */ 435 void 436 _bus_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) 437 { 438 439 /* 440 * No resources to free; just mark the mappings as 441 * invalid. 442 */ 443 map->dm_maxsegsz = map->_dm_maxmaxsegsz; 444 map->dm_mapsize = 0; 445 map->dm_nsegs = 0; 446 map->_dm_window = NULL; 447 map->_dm_flags &= ~(BUS_DMA_READ|BUS_DMA_WRITE); 448 } 449 450 /* 451 * Common function for DMA map synchronization. May be called 452 * by chipset-specific DMA map synchronization functions. 453 */ 454 void 455 _bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 456 bus_size_t len, int ops) 457 { 458 459 /* 460 * Flush the store buffer. 461 */ 462 alpha_mb(); 463 } 464 465 /* 466 * Common function for DMA-safe memory allocation. May be called 467 * by bus-specific DMA memory allocation functions. 468 */ 469 int 470 _bus_dmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, 471 bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, 472 int flags) 473 { 474 475 return (_bus_dmamem_alloc_range(t, size, alignment, boundary, 476 segs, nsegs, rsegs, flags, 0, trunc_page(avail_end))); 477 } 478 479 /* 480 * Allocate physical memory from the given physical address range. 481 * Called by DMA-safe memory allocation methods. 482 */ 483 int 484 _bus_dmamem_alloc_range(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, 485 bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, 486 int flags, paddr_t low, paddr_t high) 487 { 488 paddr_t curaddr, lastaddr; 489 struct vm_page *m; 490 struct pglist mlist; 491 int curseg, error; 492 493 /* Always round the size. */ 494 size = round_page(size); 495 496 /* 497 * Allocate pages from the VM system. 498 */ 499 error = uvm_pglistalloc(size, low, high, alignment, boundary, 500 &mlist, nsegs, (flags & BUS_DMA_NOWAIT) == 0); 501 if (error) 502 return (error); 503 504 /* 505 * Compute the location, size, and number of segments actually 506 * returned by the VM code. 507 */ 508 m = mlist.tqh_first; 509 curseg = 0; 510 lastaddr = segs[curseg].ds_addr = VM_PAGE_TO_PHYS(m); 511 segs[curseg].ds_len = PAGE_SIZE; 512 m = m->pageq.tqe_next; 513 514 for (; m != NULL; m = m->pageq.tqe_next) { 515 curaddr = VM_PAGE_TO_PHYS(m); 516 #ifdef DIAGNOSTIC 517 if (curaddr < avail_start || curaddr >= high) { 518 printf("uvm_pglistalloc returned non-sensical" 519 " address 0x%lx\n", curaddr); 520 panic("_bus_dmamem_alloc"); 521 } 522 #endif 523 if (curaddr == (lastaddr + PAGE_SIZE)) 524 segs[curseg].ds_len += PAGE_SIZE; 525 else { 526 curseg++; 527 segs[curseg].ds_addr = curaddr; 528 segs[curseg].ds_len = PAGE_SIZE; 529 } 530 lastaddr = curaddr; 531 } 532 533 *rsegs = curseg + 1; 534 535 return (0); 536 } 537 538 /* 539 * Common function for freeing DMA-safe memory. May be called by 540 * bus-specific DMA memory free functions. 541 */ 542 void 543 _bus_dmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs) 544 { 545 struct vm_page *m; 546 bus_addr_t addr; 547 struct pglist mlist; 548 int curseg; 549 550 /* 551 * Build a list of pages to free back to the VM system. 552 */ 553 TAILQ_INIT(&mlist); 554 for (curseg = 0; curseg < nsegs; curseg++) { 555 for (addr = segs[curseg].ds_addr; 556 addr < (segs[curseg].ds_addr + segs[curseg].ds_len); 557 addr += PAGE_SIZE) { 558 m = PHYS_TO_VM_PAGE(addr); 559 TAILQ_INSERT_TAIL(&mlist, m, pageq); 560 } 561 } 562 563 uvm_pglistfree(&mlist); 564 } 565 566 /* 567 * Common function for mapping DMA-safe memory. May be called by 568 * bus-specific DMA memory map functions. 569 */ 570 int 571 _bus_dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, 572 size_t size, caddr_t *kvap, int flags) 573 { 574 vaddr_t va; 575 bus_addr_t addr; 576 int curseg; 577 578 /* 579 * If we're only mapping 1 segment, use K0SEG, to avoid 580 * TLB thrashing. 581 */ 582 if (nsegs == 1) { 583 *kvap = (caddr_t)ALPHA_PHYS_TO_K0SEG(segs[0].ds_addr); 584 return (0); 585 } 586 587 size = round_page(size); 588 589 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY); 590 591 if (va == 0) 592 return (ENOMEM); 593 594 *kvap = (caddr_t)va; 595 596 for (curseg = 0; curseg < nsegs; curseg++) { 597 for (addr = segs[curseg].ds_addr; 598 addr < (segs[curseg].ds_addr + segs[curseg].ds_len); 599 addr += PAGE_SIZE, va += PAGE_SIZE, size -= PAGE_SIZE) { 600 if (size == 0) 601 panic("_bus_dmamem_map: size botch"); 602 pmap_enter(pmap_kernel(), va, addr, 603 VM_PROT_READ | VM_PROT_WRITE, 604 PMAP_WIRED | VM_PROT_READ | VM_PROT_WRITE); 605 } 606 } 607 pmap_update(pmap_kernel()); 608 609 return (0); 610 } 611 612 /* 613 * Common function for unmapping DMA-safe memory. May be called by 614 * bus-specific DMA memory unmapping functions. 615 */ 616 void 617 _bus_dmamem_unmap(bus_dma_tag_t t, caddr_t kva, size_t size) 618 { 619 620 #ifdef DIAGNOSTIC 621 if ((u_long)kva & PGOFSET) 622 panic("_bus_dmamem_unmap"); 623 #endif 624 625 /* 626 * Nothing to do if we mapped it with K0SEG. 627 */ 628 if (kva >= (caddr_t)ALPHA_K0SEG_BASE && 629 kva <= (caddr_t)ALPHA_K0SEG_END) 630 return; 631 632 size = round_page(size); 633 pmap_remove(pmap_kernel(), (vaddr_t)kva, (vaddr_t)kva + size); 634 pmap_update(pmap_kernel()); 635 uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY); 636 } 637 638 /* 639 * Common functin for mmap(2)'ing DMA-safe memory. May be called by 640 * bus-specific DMA mmap(2)'ing functions. 641 */ 642 paddr_t 643 _bus_dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, 644 off_t off, int prot, int flags) 645 { 646 int i; 647 648 for (i = 0; i < nsegs; i++) { 649 #ifdef DIAGNOSTIC 650 if (off & PGOFSET) 651 panic("_bus_dmamem_mmap: offset unaligned"); 652 if (segs[i].ds_addr & PGOFSET) 653 panic("_bus_dmamem_mmap: segment unaligned"); 654 if (segs[i].ds_len & PGOFSET) 655 panic("_bus_dmamem_mmap: segment size not multiple" 656 " of page size"); 657 #endif 658 if (off >= segs[i].ds_len) { 659 off -= segs[i].ds_len; 660 continue; 661 } 662 663 return (alpha_btop((caddr_t)segs[i].ds_addr + off)); 664 } 665 666 /* Page not found. */ 667 return (-1); 668 } 669