1 /* $NetBSD: bus_dma.c,v 1.59 2012/09/22 01:48:50 matt Exp $ */ 2 3 /*- 4 * Copyright (c) 1996, 1997, 1998 The NetBSD Foundation, Inc. 5 * All rights reserved. 6 * 7 * This code is derived from software contributed to The NetBSD Foundation 8 * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility, 9 * NASA Ames Research Center. 10 * 11 * Redistribution and use in source and binary forms, with or without 12 * modification, are permitted provided that the following conditions 13 * are met: 14 * 1. Redistributions of source code must retain the above copyright 15 * notice, this list of conditions and the following disclaimer. 16 * 2. Redistributions in binary form must reproduce the above copyright 17 * notice, this list of conditions and the following disclaimer in the 18 * documentation and/or other materials provided with the distribution. 19 * 20 * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS 21 * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED 22 * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR 23 * PURPOSE ARE DISCLAIMED. IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS 24 * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR 25 * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF 26 * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS 27 * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN 28 * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) 29 * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE 30 * POSSIBILITY OF SUCH DAMAGE. 31 */ 32 33 #define _ARM32_BUS_DMA_PRIVATE 34 35 #include <sys/cdefs.h> 36 __KERNEL_RCSID(0, "$NetBSD: bus_dma.c,v 1.59 2012/09/22 01:48:50 matt Exp $"); 37 38 #include <sys/param.h> 39 #include <sys/systm.h> 40 #include <sys/kernel.h> 41 #include <sys/proc.h> 42 #include <sys/buf.h> 43 #include <sys/reboot.h> 44 #include <sys/conf.h> 45 #include <sys/file.h> 46 #include <sys/malloc.h> 47 #include <sys/mbuf.h> 48 #include <sys/vnode.h> 49 #include <sys/device.h> 50 51 #include <uvm/uvm.h> 52 53 #include <sys/bus.h> 54 #include <machine/cpu.h> 55 56 #include <arm/cpufunc.h> 57 58 static struct evcnt bus_dma_creates = 59 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "creates"); 60 static struct evcnt bus_dma_bounced_creates = 61 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced creates"); 62 static struct evcnt bus_dma_loads = 63 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "loads"); 64 static struct evcnt bus_dma_bounced_loads = 65 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced loads"); 66 static struct evcnt bus_dma_read_bounces = 67 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "read bounces"); 68 static struct evcnt bus_dma_write_bounces = 69 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "write bounces"); 70 static struct evcnt bus_dma_bounced_unloads = 71 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced unloads"); 72 static struct evcnt bus_dma_unloads = 73 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "unloads"); 74 static struct evcnt bus_dma_bounced_destroys = 75 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced destroys"); 76 static struct evcnt bus_dma_destroys = 77 EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "destroys"); 78 79 EVCNT_ATTACH_STATIC(bus_dma_creates); 80 EVCNT_ATTACH_STATIC(bus_dma_bounced_creates); 81 EVCNT_ATTACH_STATIC(bus_dma_loads); 82 EVCNT_ATTACH_STATIC(bus_dma_bounced_loads); 83 EVCNT_ATTACH_STATIC(bus_dma_read_bounces); 84 EVCNT_ATTACH_STATIC(bus_dma_write_bounces); 85 EVCNT_ATTACH_STATIC(bus_dma_unloads); 86 EVCNT_ATTACH_STATIC(bus_dma_bounced_unloads); 87 EVCNT_ATTACH_STATIC(bus_dma_destroys); 88 EVCNT_ATTACH_STATIC(bus_dma_bounced_destroys); 89 90 #define STAT_INCR(x) (bus_dma_ ## x.ev_count++) 91 92 int _bus_dmamap_load_buffer(bus_dma_tag_t, bus_dmamap_t, void *, 93 bus_size_t, struct vmspace *, int); 94 static struct arm32_dma_range * 95 _bus_dma_paddr_inrange(struct arm32_dma_range *, int, paddr_t); 96 97 /* 98 * Check to see if the specified page is in an allowed DMA range. 99 */ 100 inline struct arm32_dma_range * 101 _bus_dma_paddr_inrange(struct arm32_dma_range *ranges, int nranges, 102 bus_addr_t curaddr) 103 { 104 struct arm32_dma_range *dr; 105 int i; 106 107 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 108 if (curaddr >= dr->dr_sysbase && 109 round_page(curaddr) <= (dr->dr_sysbase + dr->dr_len)) 110 return (dr); 111 } 112 113 return (NULL); 114 } 115 116 /* 117 * Check to see if the specified busaddr is in an allowed DMA range. 118 */ 119 static inline paddr_t 120 _bus_dma_busaddr_to_paddr(bus_dma_tag_t t, bus_addr_t curaddr) 121 { 122 struct arm32_dma_range *dr; 123 u_int i; 124 125 if (t->_nranges == 0) 126 return curaddr; 127 128 for (i = 0, dr = t->_ranges; i < t->_nranges; i++, dr++) { 129 if (dr->dr_busbase <= curaddr 130 && round_page(curaddr) <= dr->dr_busbase + dr->dr_len) 131 return curaddr - dr->dr_busbase + dr->dr_sysbase; 132 } 133 panic("%s: curaddr %#lx not in range", __func__, curaddr); 134 } 135 136 /* 137 * Common function to load the specified physical address into the 138 * DMA map, coalescing segments and boundary checking as necessary. 139 */ 140 static int 141 _bus_dmamap_load_paddr(bus_dma_tag_t t, bus_dmamap_t map, 142 bus_addr_t paddr, bus_size_t size) 143 { 144 bus_dma_segment_t * const segs = map->dm_segs; 145 int nseg = map->dm_nsegs; 146 bus_addr_t lastaddr; 147 bus_addr_t bmask = ~(map->_dm_boundary - 1); 148 bus_addr_t curaddr; 149 bus_size_t sgsize; 150 151 if (nseg > 0) 152 lastaddr = segs[nseg-1].ds_addr + segs[nseg-1].ds_len; 153 else 154 lastaddr = 0xdead; 155 156 again: 157 sgsize = size; 158 159 /* Make sure we're in an allowed DMA range. */ 160 if (t->_ranges != NULL) { 161 /* XXX cache last result? */ 162 const struct arm32_dma_range * const dr = 163 _bus_dma_paddr_inrange(t->_ranges, t->_nranges, paddr); 164 if (dr == NULL) 165 return (EINVAL); 166 167 /* 168 * In a valid DMA range. Translate the physical 169 * memory address to an address in the DMA window. 170 */ 171 curaddr = (paddr - dr->dr_sysbase) + dr->dr_busbase; 172 } else 173 curaddr = paddr; 174 175 /* 176 * Make sure we don't cross any boundaries. 177 */ 178 if (map->_dm_boundary > 0) { 179 bus_addr_t baddr; /* next boundary address */ 180 181 baddr = (curaddr + map->_dm_boundary) & bmask; 182 if (sgsize > (baddr - curaddr)) 183 sgsize = (baddr - curaddr); 184 } 185 186 /* 187 * Insert chunk into a segment, coalescing with the 188 * previous segment if possible. 189 */ 190 if (nseg > 0 && curaddr == lastaddr && 191 segs[nseg-1].ds_len + sgsize <= map->dm_maxsegsz && 192 (map->_dm_boundary == 0 || 193 (segs[nseg-1].ds_addr & bmask) == (curaddr & bmask))) { 194 /* coalesce */ 195 segs[nseg-1].ds_len += sgsize; 196 } else if (nseg >= map->_dm_segcnt) { 197 return (EFBIG); 198 } else { 199 /* new segment */ 200 segs[nseg].ds_addr = curaddr; 201 segs[nseg].ds_len = sgsize; 202 nseg++; 203 } 204 205 lastaddr = curaddr + sgsize; 206 207 paddr += sgsize; 208 size -= sgsize; 209 if (size > 0) 210 goto again; 211 212 map->dm_nsegs = nseg; 213 return (0); 214 } 215 216 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 217 static int _bus_dma_alloc_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map, 218 bus_size_t size, int flags); 219 static void _bus_dma_free_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map); 220 static int _bus_dma_uiomove(void *buf, struct uio *uio, size_t n, 221 int direction); 222 223 static int 224 _bus_dma_load_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 225 size_t buflen, int buftype, int flags) 226 { 227 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 228 struct vmspace * const vm = vmspace_kernel(); 229 int error; 230 231 KASSERT(cookie != NULL); 232 KASSERT(cookie->id_flags & _BUS_DMA_MIGHT_NEED_BOUNCE); 233 234 /* 235 * Allocate bounce pages, if necessary. 236 */ 237 if ((cookie->id_flags & _BUS_DMA_HAS_BOUNCE) == 0) { 238 error = _bus_dma_alloc_bouncebuf(t, map, buflen, flags); 239 if (error) 240 return (error); 241 } 242 243 /* 244 * Cache a pointer to the caller's buffer and load the DMA map 245 * with the bounce buffer. 246 */ 247 cookie->id_origbuf = buf; 248 cookie->id_origbuflen = buflen; 249 error = _bus_dmamap_load_buffer(t, map, cookie->id_bouncebuf, 250 buflen, vm, flags); 251 if (error) 252 return (error); 253 254 STAT_INCR(bounced_loads); 255 map->dm_mapsize = buflen; 256 map->_dm_vmspace = vm; 257 map->_dm_buftype = buftype; 258 259 /* ...so _bus_dmamap_sync() knows we're bouncing */ 260 cookie->id_flags |= _BUS_DMA_IS_BOUNCING; 261 return 0; 262 } 263 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 264 265 /* 266 * Common function for DMA map creation. May be called by bus-specific 267 * DMA map creation functions. 268 */ 269 int 270 _bus_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments, 271 bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp) 272 { 273 struct arm32_bus_dmamap *map; 274 void *mapstore; 275 size_t mapsize; 276 277 #ifdef DEBUG_DMA 278 printf("dmamap_create: t=%p size=%lx nseg=%x msegsz=%lx boundary=%lx flags=%x\n", 279 t, size, nsegments, maxsegsz, boundary, flags); 280 #endif /* DEBUG_DMA */ 281 282 /* 283 * Allocate and initialize the DMA map. The end of the map 284 * is a variable-sized array of segments, so we allocate enough 285 * room for them in one shot. 286 * 287 * Note we don't preserve the WAITOK or NOWAIT flags. Preservation 288 * of ALLOCNOW notifies others that we've reserved these resources, 289 * and they are not to be freed. 290 * 291 * The bus_dmamap_t includes one bus_dma_segment_t, hence 292 * the (nsegments - 1). 293 */ 294 mapsize = sizeof(struct arm32_bus_dmamap) + 295 (sizeof(bus_dma_segment_t) * (nsegments - 1)); 296 const int mallocflags = M_ZERO|(flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK; 297 if ((mapstore = malloc(mapsize, M_DMAMAP, mallocflags)) == NULL) 298 return (ENOMEM); 299 300 map = (struct arm32_bus_dmamap *)mapstore; 301 map->_dm_size = size; 302 map->_dm_segcnt = nsegments; 303 map->_dm_maxmaxsegsz = maxsegsz; 304 map->_dm_boundary = boundary; 305 map->_dm_flags = flags & ~(BUS_DMA_WAITOK|BUS_DMA_NOWAIT); 306 map->_dm_origbuf = NULL; 307 map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID; 308 map->_dm_vmspace = vmspace_kernel(); 309 map->_dm_cookie = NULL; 310 map->dm_maxsegsz = maxsegsz; 311 map->dm_mapsize = 0; /* no valid mappings */ 312 map->dm_nsegs = 0; 313 314 *dmamp = map; 315 316 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 317 struct arm32_bus_dma_cookie *cookie; 318 int cookieflags; 319 void *cookiestore; 320 size_t cookiesize; 321 int error; 322 323 cookieflags = 0; 324 325 if (t->_may_bounce != NULL) { 326 error = (*t->_may_bounce)(t, map, flags, &cookieflags); 327 if (error != 0) 328 goto out; 329 } 330 331 if (t->_ranges != NULL) 332 cookieflags |= _BUS_DMA_MIGHT_NEED_BOUNCE; 333 334 if ((cookieflags & _BUS_DMA_MIGHT_NEED_BOUNCE) == 0) { 335 STAT_INCR(creates); 336 return 0; 337 } 338 339 cookiesize = sizeof(struct arm32_bus_dma_cookie) + 340 (sizeof(bus_dma_segment_t) * map->_dm_segcnt); 341 342 /* 343 * Allocate our cookie. 344 */ 345 if ((cookiestore = malloc(cookiesize, M_DMAMAP, mallocflags)) == NULL) { 346 error = ENOMEM; 347 goto out; 348 } 349 cookie = (struct arm32_bus_dma_cookie *)cookiestore; 350 cookie->id_flags = cookieflags; 351 map->_dm_cookie = cookie; 352 STAT_INCR(bounced_creates); 353 354 error = _bus_dma_alloc_bouncebuf(t, map, size, flags); 355 out: 356 if (error) 357 _bus_dmamap_destroy(t, map); 358 #else 359 STAT_INCR(creates); 360 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 361 362 #ifdef DEBUG_DMA 363 printf("dmamap_create:map=%p\n", map); 364 #endif /* DEBUG_DMA */ 365 return (0); 366 } 367 368 /* 369 * Common function for DMA map destruction. May be called by bus-specific 370 * DMA map destruction functions. 371 */ 372 void 373 _bus_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map) 374 { 375 376 #ifdef DEBUG_DMA 377 printf("dmamap_destroy: t=%p map=%p\n", t, map); 378 #endif /* DEBUG_DMA */ 379 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 380 struct arm32_bus_dma_cookie *cookie = map->_dm_cookie; 381 382 /* 383 * Free any bounce pages this map might hold. 384 */ 385 if (cookie != NULL) { 386 if (cookie->id_flags & _BUS_DMA_IS_BOUNCING) 387 STAT_INCR(bounced_unloads); 388 map->dm_nsegs = 0; 389 if (cookie->id_flags & _BUS_DMA_HAS_BOUNCE) 390 _bus_dma_free_bouncebuf(t, map); 391 STAT_INCR(bounced_destroys); 392 free(cookie, M_DMAMAP); 393 } else 394 #endif 395 STAT_INCR(destroys); 396 397 if (map->dm_nsegs > 0) 398 STAT_INCR(unloads); 399 400 free(map, M_DMAMAP); 401 } 402 403 /* 404 * Common function for loading a DMA map with a linear buffer. May 405 * be called by bus-specific DMA map load functions. 406 */ 407 int 408 _bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 409 bus_size_t buflen, struct proc *p, int flags) 410 { 411 struct vmspace *vm; 412 int error; 413 414 #ifdef DEBUG_DMA 415 printf("dmamap_load: t=%p map=%p buf=%p len=%lx p=%p f=%d\n", 416 t, map, buf, buflen, p, flags); 417 #endif /* DEBUG_DMA */ 418 419 if (map->dm_nsegs > 0) { 420 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 421 struct arm32_bus_dma_cookie *cookie = map->_dm_cookie; 422 if (cookie != NULL) { 423 if (cookie->id_flags & _BUS_DMA_IS_BOUNCING) { 424 STAT_INCR(bounced_unloads); 425 cookie->id_flags &= ~_BUS_DMA_IS_BOUNCING; 426 } 427 } else 428 #endif 429 STAT_INCR(unloads); 430 } 431 432 /* 433 * Make sure that on error condition we return "no valid mappings". 434 */ 435 map->dm_mapsize = 0; 436 map->dm_nsegs = 0; 437 map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID; 438 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz); 439 440 if (buflen > map->_dm_size) 441 return (EINVAL); 442 443 if (p != NULL) { 444 vm = p->p_vmspace; 445 } else { 446 vm = vmspace_kernel(); 447 } 448 449 /* _bus_dmamap_load_buffer() clears this if we're not... */ 450 map->_dm_flags |= _BUS_DMAMAP_COHERENT; 451 452 error = _bus_dmamap_load_buffer(t, map, buf, buflen, vm, flags); 453 if (error == 0) { 454 map->dm_mapsize = buflen; 455 map->_dm_vmspace = vm; 456 map->_dm_origbuf = buf; 457 map->_dm_buftype = _BUS_DMA_BUFTYPE_LINEAR; 458 return 0; 459 } 460 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 461 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 462 if (cookie != NULL && (cookie->id_flags & _BUS_DMA_MIGHT_NEED_BOUNCE)) { 463 error = _bus_dma_load_bouncebuf(t, map, buf, buflen, 464 _BUS_DMA_BUFTYPE_LINEAR, flags); 465 } 466 #endif 467 return (error); 468 } 469 470 /* 471 * Like _bus_dmamap_load(), but for mbufs. 472 */ 473 int 474 _bus_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0, 475 int flags) 476 { 477 int error; 478 struct mbuf *m; 479 480 #ifdef DEBUG_DMA 481 printf("dmamap_load_mbuf: t=%p map=%p m0=%p f=%d\n", 482 t, map, m0, flags); 483 #endif /* DEBUG_DMA */ 484 485 if (map->dm_nsegs > 0) { 486 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 487 struct arm32_bus_dma_cookie *cookie = map->_dm_cookie; 488 if (cookie != NULL) { 489 if (cookie->id_flags & _BUS_DMA_IS_BOUNCING) { 490 STAT_INCR(bounced_unloads); 491 cookie->id_flags &= ~_BUS_DMA_IS_BOUNCING; 492 } 493 } else 494 #endif 495 STAT_INCR(unloads); 496 } 497 498 /* 499 * Make sure that on error condition we return "no valid mappings." 500 */ 501 map->dm_mapsize = 0; 502 map->dm_nsegs = 0; 503 map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID; 504 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz); 505 506 #ifdef DIAGNOSTIC 507 if ((m0->m_flags & M_PKTHDR) == 0) 508 panic("_bus_dmamap_load_mbuf: no packet header"); 509 #endif /* DIAGNOSTIC */ 510 511 if (m0->m_pkthdr.len > map->_dm_size) 512 return (EINVAL); 513 514 /* 515 * Mbuf chains should almost never have coherent (i.e. 516 * un-cached) mappings, so clear that flag now. 517 */ 518 map->_dm_flags &= ~_BUS_DMAMAP_COHERENT; 519 520 error = 0; 521 for (m = m0; m != NULL && error == 0; m = m->m_next) { 522 int offset; 523 int remainbytes; 524 const struct vm_page * const *pgs; 525 paddr_t paddr; 526 int size; 527 528 if (m->m_len == 0) 529 continue; 530 /* 531 * Don't allow reads in read-only mbufs. 532 */ 533 if (M_ROMAP(m) && (flags & BUS_DMA_READ)) { 534 error = EFAULT; 535 break; 536 } 537 switch (m->m_flags & (M_EXT|M_CLUSTER|M_EXT_PAGES)) { 538 case M_EXT|M_CLUSTER: 539 /* XXX KDASSERT */ 540 KASSERT(m->m_ext.ext_paddr != M_PADDR_INVALID); 541 paddr = m->m_ext.ext_paddr + 542 (m->m_data - m->m_ext.ext_buf); 543 size = m->m_len; 544 error = _bus_dmamap_load_paddr(t, map, paddr, size); 545 break; 546 547 case M_EXT|M_EXT_PAGES: 548 KASSERT(m->m_ext.ext_buf <= m->m_data); 549 KASSERT(m->m_data <= 550 m->m_ext.ext_buf + m->m_ext.ext_size); 551 552 offset = (vaddr_t)m->m_data - 553 trunc_page((vaddr_t)m->m_ext.ext_buf); 554 remainbytes = m->m_len; 555 556 /* skip uninteresting pages */ 557 pgs = (const struct vm_page * const *) 558 m->m_ext.ext_pgs + (offset >> PAGE_SHIFT); 559 560 offset &= PAGE_MASK; /* offset in the first page */ 561 562 /* load each page */ 563 while (remainbytes > 0) { 564 const struct vm_page *pg; 565 566 size = MIN(remainbytes, PAGE_SIZE - offset); 567 568 pg = *pgs++; 569 KASSERT(pg); 570 paddr = VM_PAGE_TO_PHYS(pg) + offset; 571 572 error = _bus_dmamap_load_paddr(t, map, 573 paddr, size); 574 if (error) 575 break; 576 offset = 0; 577 remainbytes -= size; 578 } 579 break; 580 581 case 0: 582 paddr = m->m_paddr + M_BUFOFFSET(m) + 583 (m->m_data - M_BUFADDR(m)); 584 size = m->m_len; 585 error = _bus_dmamap_load_paddr(t, map, paddr, size); 586 break; 587 588 default: 589 error = _bus_dmamap_load_buffer(t, map, m->m_data, 590 m->m_len, vmspace_kernel(), flags); 591 } 592 } 593 if (error == 0) { 594 map->dm_mapsize = m0->m_pkthdr.len; 595 map->_dm_origbuf = m0; 596 map->_dm_buftype = _BUS_DMA_BUFTYPE_MBUF; 597 map->_dm_vmspace = vmspace_kernel(); /* always kernel */ 598 return 0; 599 } 600 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 601 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 602 if (cookie != NULL && (cookie->id_flags & _BUS_DMA_MIGHT_NEED_BOUNCE)) { 603 error = _bus_dma_load_bouncebuf(t, map, m0, m0->m_pkthdr.len, 604 _BUS_DMA_BUFTYPE_MBUF, flags); 605 } 606 #endif 607 return (error); 608 } 609 610 /* 611 * Like _bus_dmamap_load(), but for uios. 612 */ 613 int 614 _bus_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio, 615 int flags) 616 { 617 int i, error; 618 bus_size_t minlen, resid; 619 struct iovec *iov; 620 void *addr; 621 622 /* 623 * Make sure that on error condition we return "no valid mappings." 624 */ 625 map->dm_mapsize = 0; 626 map->dm_nsegs = 0; 627 KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz); 628 629 resid = uio->uio_resid; 630 iov = uio->uio_iov; 631 632 /* _bus_dmamap_load_buffer() clears this if we're not... */ 633 map->_dm_flags |= _BUS_DMAMAP_COHERENT; 634 635 error = 0; 636 for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) { 637 /* 638 * Now at the first iovec to load. Load each iovec 639 * until we have exhausted the residual count. 640 */ 641 minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len; 642 addr = (void *)iov[i].iov_base; 643 644 error = _bus_dmamap_load_buffer(t, map, addr, minlen, 645 uio->uio_vmspace, flags); 646 647 resid -= minlen; 648 } 649 if (error == 0) { 650 map->dm_mapsize = uio->uio_resid; 651 map->_dm_origbuf = uio; 652 map->_dm_buftype = _BUS_DMA_BUFTYPE_UIO; 653 map->_dm_vmspace = uio->uio_vmspace; 654 } 655 return (error); 656 } 657 658 /* 659 * Like _bus_dmamap_load(), but for raw memory allocated with 660 * bus_dmamem_alloc(). 661 */ 662 int 663 _bus_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map, 664 bus_dma_segment_t *segs, int nsegs, bus_size_t size, int flags) 665 { 666 667 panic("_bus_dmamap_load_raw: not implemented"); 668 } 669 670 /* 671 * Common function for unloading a DMA map. May be called by 672 * bus-specific DMA map unload functions. 673 */ 674 void 675 _bus_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map) 676 { 677 678 #ifdef DEBUG_DMA 679 printf("dmamap_unload: t=%p map=%p\n", t, map); 680 #endif /* DEBUG_DMA */ 681 682 /* 683 * No resources to free; just mark the mappings as 684 * invalid. 685 */ 686 map->dm_mapsize = 0; 687 map->dm_nsegs = 0; 688 map->_dm_origbuf = NULL; 689 map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID; 690 map->_dm_vmspace = NULL; 691 } 692 693 static void 694 _bus_dmamap_sync_segment(vaddr_t va, paddr_t pa, vsize_t len, int ops, bool readonly_p) 695 { 696 KASSERT((va & PAGE_MASK) == (pa & PAGE_MASK)); 697 698 switch (ops) { 699 case BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE: 700 if (!readonly_p) { 701 cpu_dcache_wbinv_range(va, len); 702 cpu_sdcache_wbinv_range(va, pa, len); 703 break; 704 } 705 /* FALLTHROUGH */ 706 707 case BUS_DMASYNC_PREREAD: { 708 const size_t line_size = arm_dcache_align; 709 const size_t line_mask = arm_dcache_align_mask; 710 vsize_t misalignment = va & line_mask; 711 if (misalignment) { 712 va -= misalignment; 713 pa -= misalignment; 714 len += misalignment; 715 cpu_dcache_wbinv_range(va, line_size); 716 cpu_sdcache_wbinv_range(va, pa, line_size); 717 if (len <= line_size) 718 break; 719 va += line_size; 720 pa += line_size; 721 len -= line_size; 722 } 723 misalignment = len & line_mask; 724 len -= misalignment; 725 cpu_dcache_inv_range(va, len); 726 cpu_sdcache_inv_range(va, pa, len); 727 if (misalignment) { 728 va += len; 729 pa += len; 730 cpu_dcache_wbinv_range(va, line_size); 731 cpu_sdcache_wbinv_range(va, pa, line_size); 732 } 733 break; 734 } 735 736 case BUS_DMASYNC_PREWRITE: 737 cpu_dcache_wb_range(va, len); 738 cpu_sdcache_wb_range(va, pa, len); 739 break; 740 } 741 } 742 743 static inline void 744 _bus_dmamap_sync_linear(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 745 bus_size_t len, int ops) 746 { 747 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 748 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 749 bool bouncing = (cookie != NULL && (cookie->id_flags & _BUS_DMA_IS_BOUNCING)); 750 #endif 751 bus_dma_segment_t *ds = map->dm_segs; 752 vaddr_t va = (vaddr_t) map->_dm_origbuf; 753 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 754 if (bouncing) { 755 va = (vaddr_t) cookie->id_bouncebuf; 756 } 757 #endif 758 759 while (len > 0) { 760 while (offset >= ds->ds_len) { 761 offset -= ds->ds_len; 762 va += ds->ds_len; 763 ds++; 764 } 765 766 paddr_t pa = _bus_dma_busaddr_to_paddr(t, ds->ds_addr + offset); 767 size_t seglen = min(len, ds->ds_len - offset); 768 769 _bus_dmamap_sync_segment(va + offset, pa, seglen, ops, false); 770 771 offset += seglen; 772 len -= seglen; 773 } 774 } 775 776 static inline void 777 _bus_dmamap_sync_mbuf(bus_dma_tag_t t, bus_dmamap_t map, bus_size_t offset, 778 bus_size_t len, int ops) 779 { 780 bus_dma_segment_t *ds = map->dm_segs; 781 struct mbuf *m = map->_dm_origbuf; 782 bus_size_t voff = offset; 783 bus_size_t ds_off = offset; 784 785 while (len > 0) { 786 /* Find the current dma segment */ 787 while (ds_off >= ds->ds_len) { 788 ds_off -= ds->ds_len; 789 ds++; 790 } 791 /* Find the current mbuf. */ 792 while (voff >= m->m_len) { 793 voff -= m->m_len; 794 m = m->m_next; 795 } 796 797 /* 798 * Now at the first mbuf to sync; nail each one until 799 * we have exhausted the length. 800 */ 801 vsize_t seglen = min(len, min(m->m_len - voff, ds->ds_len - ds_off)); 802 vaddr_t va = mtod(m, vaddr_t) + voff; 803 paddr_t pa = _bus_dma_busaddr_to_paddr(t, ds->ds_addr + ds_off); 804 805 /* 806 * We can save a lot of work here if we know the mapping 807 * is read-only at the MMU: 808 * 809 * If a mapping is read-only, no dirty cache blocks will 810 * exist for it. If a writable mapping was made read-only, 811 * we know any dirty cache lines for the range will have 812 * been cleaned for us already. Therefore, if the upper 813 * layer can tell us we have a read-only mapping, we can 814 * skip all cache cleaning. 815 * 816 * NOTE: This only works if we know the pmap cleans pages 817 * before making a read-write -> read-only transition. If 818 * this ever becomes non-true (e.g. Physically Indexed 819 * cache), this will have to be revisited. 820 */ 821 822 _bus_dmamap_sync_segment(va, pa, seglen, ops, M_ROMAP(m)); 823 voff += seglen; 824 ds_off += seglen; 825 len -= seglen; 826 } 827 } 828 829 static inline void 830 _bus_dmamap_sync_uio(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 831 bus_size_t len, int ops) 832 { 833 bus_dma_segment_t *ds = map->dm_segs; 834 struct uio *uio = map->_dm_origbuf; 835 struct iovec *iov = uio->uio_iov; 836 bus_size_t voff = offset; 837 bus_size_t ds_off = offset; 838 839 while (len > 0) { 840 /* Find the current dma segment */ 841 while (ds_off >= ds->ds_len) { 842 ds_off -= ds->ds_len; 843 ds++; 844 } 845 846 /* Find the current iovec. */ 847 while (voff >= iov->iov_len) { 848 voff -= iov->iov_len; 849 iov++; 850 } 851 852 /* 853 * Now at the first iovec to sync; nail each one until 854 * we have exhausted the length. 855 */ 856 vsize_t seglen = min(len, min(iov->iov_len - voff, ds->ds_len - ds_off)); 857 vaddr_t va = (vaddr_t) iov->iov_base + voff; 858 paddr_t pa = _bus_dma_busaddr_to_paddr(t, ds->ds_addr + ds_off); 859 860 _bus_dmamap_sync_segment(va, pa, seglen, ops, false); 861 862 voff += seglen; 863 ds_off += seglen; 864 len -= seglen; 865 } 866 } 867 868 /* 869 * Common function for DMA map synchronization. May be called 870 * by bus-specific DMA map synchronization functions. 871 * 872 * This version works for the Virtually Indexed Virtually Tagged 873 * cache found on 32-bit ARM processors. 874 * 875 * XXX Should have separate versions for write-through vs. 876 * XXX write-back caches. We currently assume write-back 877 * XXX here, which is not as efficient as it could be for 878 * XXX the write-through case. 879 */ 880 void 881 _bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset, 882 bus_size_t len, int ops) 883 { 884 bool bouncing = false; 885 886 #ifdef DEBUG_DMA 887 printf("dmamap_sync: t=%p map=%p offset=%lx len=%lx ops=%x\n", 888 t, map, offset, len, ops); 889 #endif /* DEBUG_DMA */ 890 891 /* 892 * Mixing of PRE and POST operations is not allowed. 893 */ 894 if ((ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE)) != 0 && 895 (ops & (BUS_DMASYNC_POSTREAD|BUS_DMASYNC_POSTWRITE)) != 0) 896 panic("_bus_dmamap_sync: mix PRE and POST"); 897 898 #ifdef DIAGNOSTIC 899 if (offset >= map->dm_mapsize) 900 panic("_bus_dmamap_sync: bad offset %lu (map size is %lu)", 901 offset, map->dm_mapsize); 902 if (len == 0 || (offset + len) > map->dm_mapsize) 903 panic("_bus_dmamap_sync: bad length"); 904 #endif 905 906 /* 907 * For a virtually-indexed write-back cache, we need 908 * to do the following things: 909 * 910 * PREREAD -- Invalidate the D-cache. We do this 911 * here in case a write-back is required by the back-end. 912 * 913 * PREWRITE -- Write-back the D-cache. Note that if 914 * we are doing a PREREAD|PREWRITE, we can collapse 915 * the whole thing into a single Wb-Inv. 916 * 917 * POSTREAD -- Nothing. 918 * 919 * POSTWRITE -- Nothing. 920 */ 921 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 922 struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie; 923 bouncing = (cookie != NULL && (cookie->id_flags & _BUS_DMA_IS_BOUNCING)); 924 #endif 925 926 const int pre_ops = ops & (BUS_DMASYNC_PREREAD|BUS_DMASYNC_PREWRITE); 927 if (!bouncing && pre_ops == 0) 928 return; 929 930 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 931 if (bouncing && (ops & BUS_DMASYNC_PREWRITE)) { 932 STAT_INCR(write_bounces); 933 char * const dataptr = (char *)cookie->id_bouncebuf + offset; 934 /* 935 * Copy the caller's buffer to the bounce buffer. 936 */ 937 switch (map->_dm_buftype) { 938 case _BUS_DMA_BUFTYPE_LINEAR: 939 memcpy(dataptr, cookie->id_origlinearbuf + offset, len); 940 break; 941 case _BUS_DMA_BUFTYPE_MBUF: 942 m_copydata(cookie->id_origmbuf, offset, len, dataptr); 943 break; 944 case _BUS_DMA_BUFTYPE_UIO: 945 _bus_dma_uiomove(dataptr, cookie->id_origuio, len, UIO_WRITE); 946 break; 947 #ifdef DIAGNOSTIC 948 case _BUS_DMA_BUFTYPE_RAW: 949 panic("_bus_dmamap_sync(pre): _BUS_DMA_BUFTYPE_RAW"); 950 break; 951 952 case _BUS_DMA_BUFTYPE_INVALID: 953 panic("_bus_dmamap_sync(pre): _BUS_DMA_BUFTYPE_INVALID"); 954 break; 955 956 default: 957 panic("_bus_dmamap_sync(pre): map %p: unknown buffer type %d\n", 958 map, map->_dm_buftype); 959 break; 960 #endif /* DIAGNOSTIC */ 961 } 962 } 963 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 964 965 /* Skip cache frobbing if mapping was COHERENT. */ 966 if (!bouncing && (map->_dm_flags & _BUS_DMAMAP_COHERENT)) { 967 /* Drain the write buffer. */ 968 cpu_drain_writebuf(); 969 return; 970 } 971 972 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 973 if (bouncing && ((map->_dm_flags & _BUS_DMAMAP_COHERENT) || pre_ops == 0)) { 974 goto bounce_it; 975 } 976 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 977 978 /* 979 * If the mapping belongs to a non-kernel vmspace, and the 980 * vmspace has not been active since the last time a full 981 * cache flush was performed, we don't need to do anything. 982 */ 983 if (__predict_false(!VMSPACE_IS_KERNEL_P(map->_dm_vmspace) && 984 vm_map_pmap(&map->_dm_vmspace->vm_map)->pm_cstate.cs_cache_d == 0)) 985 return; 986 987 int buftype = map->_dm_buftype; 988 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 989 if (bouncing) { 990 buftype = _BUS_DMA_BUFTYPE_LINEAR; 991 } 992 #endif 993 994 switch (buftype) { 995 case _BUS_DMA_BUFTYPE_LINEAR: 996 _bus_dmamap_sync_linear(t, map, offset, len, ops); 997 break; 998 999 case _BUS_DMA_BUFTYPE_MBUF: 1000 _bus_dmamap_sync_mbuf(t, map, offset, len, ops); 1001 break; 1002 1003 case _BUS_DMA_BUFTYPE_UIO: 1004 _bus_dmamap_sync_uio(t, map, offset, len, ops); 1005 break; 1006 1007 case _BUS_DMA_BUFTYPE_RAW: 1008 panic("_bus_dmamap_sync: _BUS_DMA_BUFTYPE_RAW"); 1009 break; 1010 1011 case _BUS_DMA_BUFTYPE_INVALID: 1012 panic("_bus_dmamap_sync: _BUS_DMA_BUFTYPE_INVALID"); 1013 break; 1014 1015 default: 1016 panic("_bus_dmamap_sync: map %p: unknown buffer type %d\n", 1017 map, map->_dm_buftype); 1018 } 1019 1020 /* Drain the write buffer. */ 1021 cpu_drain_writebuf(); 1022 1023 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 1024 bounce_it: 1025 if ((ops & BUS_DMASYNC_POSTREAD) == 0 1026 || cookie == NULL 1027 || (cookie->id_flags & _BUS_DMA_IS_BOUNCING) == 0) 1028 return; 1029 1030 char * const dataptr = (char *)cookie->id_bouncebuf + offset; 1031 STAT_INCR(read_bounces); 1032 /* 1033 * Copy the bounce buffer to the caller's buffer. 1034 */ 1035 switch (map->_dm_buftype) { 1036 case _BUS_DMA_BUFTYPE_LINEAR: 1037 memcpy(cookie->id_origlinearbuf + offset, dataptr, len); 1038 break; 1039 1040 case _BUS_DMA_BUFTYPE_MBUF: 1041 m_copyback(cookie->id_origmbuf, offset, len, dataptr); 1042 break; 1043 1044 case _BUS_DMA_BUFTYPE_UIO: 1045 _bus_dma_uiomove(dataptr, cookie->id_origuio, len, UIO_READ); 1046 break; 1047 #ifdef DIAGNOSTIC 1048 case _BUS_DMA_BUFTYPE_RAW: 1049 panic("_bus_dmamap_sync(post): _BUS_DMA_BUFTYPE_RAW"); 1050 break; 1051 1052 case _BUS_DMA_BUFTYPE_INVALID: 1053 panic("_bus_dmamap_sync(post): _BUS_DMA_BUFTYPE_INVALID"); 1054 break; 1055 1056 default: 1057 panic("_bus_dmamap_sync(post): map %p: unknown buffer type %d\n", 1058 map, map->_dm_buftype); 1059 break; 1060 #endif 1061 } 1062 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 1063 } 1064 1065 /* 1066 * Common function for DMA-safe memory allocation. May be called 1067 * by bus-specific DMA memory allocation functions. 1068 */ 1069 1070 extern paddr_t physical_start; 1071 extern paddr_t physical_end; 1072 1073 int 1074 _bus_dmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, 1075 bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, 1076 int flags) 1077 { 1078 struct arm32_dma_range *dr; 1079 int error, i; 1080 1081 #ifdef DEBUG_DMA 1082 printf("dmamem_alloc t=%p size=%lx align=%lx boundary=%lx " 1083 "segs=%p nsegs=%x rsegs=%p flags=%x\n", t, size, alignment, 1084 boundary, segs, nsegs, rsegs, flags); 1085 #endif 1086 1087 if ((dr = t->_ranges) != NULL) { 1088 error = ENOMEM; 1089 for (i = 0; i < t->_nranges; i++, dr++) { 1090 if (dr->dr_len == 0) 1091 continue; 1092 error = _bus_dmamem_alloc_range(t, size, alignment, 1093 boundary, segs, nsegs, rsegs, flags, 1094 trunc_page(dr->dr_sysbase), 1095 trunc_page(dr->dr_sysbase + dr->dr_len)); 1096 if (error == 0) 1097 break; 1098 } 1099 } else { 1100 error = _bus_dmamem_alloc_range(t, size, alignment, boundary, 1101 segs, nsegs, rsegs, flags, trunc_page(physical_start), 1102 trunc_page(physical_end)); 1103 } 1104 1105 #ifdef DEBUG_DMA 1106 printf("dmamem_alloc: =%d\n", error); 1107 #endif 1108 1109 return(error); 1110 } 1111 1112 /* 1113 * Common function for freeing DMA-safe memory. May be called by 1114 * bus-specific DMA memory free functions. 1115 */ 1116 void 1117 _bus_dmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs) 1118 { 1119 struct vm_page *m; 1120 bus_addr_t addr; 1121 struct pglist mlist; 1122 int curseg; 1123 1124 #ifdef DEBUG_DMA 1125 printf("dmamem_free: t=%p segs=%p nsegs=%x\n", t, segs, nsegs); 1126 #endif /* DEBUG_DMA */ 1127 1128 /* 1129 * Build a list of pages to free back to the VM system. 1130 */ 1131 TAILQ_INIT(&mlist); 1132 for (curseg = 0; curseg < nsegs; curseg++) { 1133 for (addr = segs[curseg].ds_addr; 1134 addr < (segs[curseg].ds_addr + segs[curseg].ds_len); 1135 addr += PAGE_SIZE) { 1136 m = PHYS_TO_VM_PAGE(addr); 1137 TAILQ_INSERT_TAIL(&mlist, m, pageq.queue); 1138 } 1139 } 1140 uvm_pglistfree(&mlist); 1141 } 1142 1143 /* 1144 * Common function for mapping DMA-safe memory. May be called by 1145 * bus-specific DMA memory map functions. 1146 */ 1147 int 1148 _bus_dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, 1149 size_t size, void **kvap, int flags) 1150 { 1151 vaddr_t va; 1152 paddr_t pa; 1153 int curseg; 1154 pt_entry_t *ptep/*, pte*/; 1155 const uvm_flag_t kmflags = 1156 (flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0; 1157 1158 #ifdef DEBUG_DMA 1159 printf("dmamem_map: t=%p segs=%p nsegs=%x size=%lx flags=%x\n", t, 1160 segs, nsegs, (unsigned long)size, flags); 1161 #endif /* DEBUG_DMA */ 1162 1163 size = round_page(size); 1164 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY | kmflags); 1165 1166 if (va == 0) 1167 return (ENOMEM); 1168 1169 *kvap = (void *)va; 1170 1171 for (curseg = 0; curseg < nsegs; curseg++) { 1172 for (pa = segs[curseg].ds_addr; 1173 pa < (segs[curseg].ds_addr + segs[curseg].ds_len); 1174 pa += PAGE_SIZE, va += PAGE_SIZE, size -= PAGE_SIZE) { 1175 #ifdef DEBUG_DMA 1176 printf("wiring p%lx to v%lx", pa, va); 1177 #endif /* DEBUG_DMA */ 1178 if (size == 0) 1179 panic("_bus_dmamem_map: size botch"); 1180 pmap_enter(pmap_kernel(), va, pa, 1181 VM_PROT_READ | VM_PROT_WRITE, 1182 VM_PROT_READ | VM_PROT_WRITE | PMAP_WIRED); 1183 1184 /* 1185 * If the memory must remain coherent with the 1186 * cache then we must make the memory uncacheable 1187 * in order to maintain virtual cache coherency. 1188 * We must also guarantee the cache does not already 1189 * contain the virtal addresses we are making 1190 * uncacheable. 1191 */ 1192 if (flags & BUS_DMA_COHERENT) { 1193 cpu_dcache_wbinv_range(va, PAGE_SIZE); 1194 cpu_sdcache_wbinv_range(va, pa, PAGE_SIZE); 1195 cpu_drain_writebuf(); 1196 ptep = vtopte(va); 1197 *ptep &= ~L2_S_CACHE_MASK; 1198 PTE_SYNC(ptep); 1199 tlb_flush(); 1200 } 1201 #ifdef DEBUG_DMA 1202 ptep = vtopte(va); 1203 printf(" pte=v%p *pte=%x\n", ptep, *ptep); 1204 #endif /* DEBUG_DMA */ 1205 } 1206 } 1207 pmap_update(pmap_kernel()); 1208 #ifdef DEBUG_DMA 1209 printf("dmamem_map: =%p\n", *kvap); 1210 #endif /* DEBUG_DMA */ 1211 return (0); 1212 } 1213 1214 /* 1215 * Common function for unmapping DMA-safe memory. May be called by 1216 * bus-specific DMA memory unmapping functions. 1217 */ 1218 void 1219 _bus_dmamem_unmap(bus_dma_tag_t t, void *kva, size_t size) 1220 { 1221 1222 #ifdef DEBUG_DMA 1223 printf("dmamem_unmap: t=%p kva=%p size=%lx\n", t, kva, 1224 (unsigned long)size); 1225 #endif /* DEBUG_DMA */ 1226 #ifdef DIAGNOSTIC 1227 if ((u_long)kva & PGOFSET) 1228 panic("_bus_dmamem_unmap"); 1229 #endif /* DIAGNOSTIC */ 1230 1231 size = round_page(size); 1232 pmap_remove(pmap_kernel(), (vaddr_t)kva, (vaddr_t)kva + size); 1233 pmap_update(pmap_kernel()); 1234 uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY); 1235 } 1236 1237 /* 1238 * Common functin for mmap(2)'ing DMA-safe memory. May be called by 1239 * bus-specific DMA mmap(2)'ing functions. 1240 */ 1241 paddr_t 1242 _bus_dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs, 1243 off_t off, int prot, int flags) 1244 { 1245 int i; 1246 1247 for (i = 0; i < nsegs; i++) { 1248 #ifdef DIAGNOSTIC 1249 if (off & PGOFSET) 1250 panic("_bus_dmamem_mmap: offset unaligned"); 1251 if (segs[i].ds_addr & PGOFSET) 1252 panic("_bus_dmamem_mmap: segment unaligned"); 1253 if (segs[i].ds_len & PGOFSET) 1254 panic("_bus_dmamem_mmap: segment size not multiple" 1255 " of page size"); 1256 #endif /* DIAGNOSTIC */ 1257 if (off >= segs[i].ds_len) { 1258 off -= segs[i].ds_len; 1259 continue; 1260 } 1261 1262 return (arm_btop((u_long)segs[i].ds_addr + off)); 1263 } 1264 1265 /* Page not found. */ 1266 return (-1); 1267 } 1268 1269 /********************************************************************** 1270 * DMA utility functions 1271 **********************************************************************/ 1272 1273 /* 1274 * Utility function to load a linear buffer. lastaddrp holds state 1275 * between invocations (for multiple-buffer loads). segp contains 1276 * the starting segment on entrace, and the ending segment on exit. 1277 * first indicates if this is the first invocation of this function. 1278 */ 1279 int 1280 _bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf, 1281 bus_size_t buflen, struct vmspace *vm, int flags) 1282 { 1283 bus_size_t sgsize; 1284 bus_addr_t curaddr; 1285 vaddr_t vaddr = (vaddr_t)buf; 1286 pd_entry_t *pde; 1287 pt_entry_t pte; 1288 int error; 1289 pmap_t pmap; 1290 pt_entry_t *ptep; 1291 1292 #ifdef DEBUG_DMA 1293 printf("_bus_dmamem_load_buffer(buf=%p, len=%lx, flags=%d)\n", 1294 buf, buflen, flags); 1295 #endif /* DEBUG_DMA */ 1296 1297 pmap = vm_map_pmap(&vm->vm_map); 1298 1299 while (buflen > 0) { 1300 /* 1301 * Get the physical address for this segment. 1302 * 1303 * XXX Doesn't support checking for coherent mappings 1304 * XXX in user address space. 1305 */ 1306 if (__predict_true(pmap == pmap_kernel())) { 1307 (void) pmap_get_pde_pte(pmap, vaddr, &pde, &ptep); 1308 if (__predict_false(pmap_pde_section(pde))) { 1309 paddr_t s_frame = L1_S_FRAME; 1310 paddr_t s_offset = L1_S_OFFSET; 1311 #if (ARM_MMU_V6 + ARM_MMU_V7) > 0 1312 if (__predict_false(pmap_pde_supersection(pde))) { 1313 s_frame = L1_SS_FRAME; 1314 s_frame = L1_SS_OFFSET; 1315 } 1316 #endif 1317 curaddr = (*pde & s_frame) | (vaddr & s_offset); 1318 if (*pde & L1_S_CACHE_MASK) { 1319 map->_dm_flags &= ~_BUS_DMAMAP_COHERENT; 1320 } 1321 } else { 1322 pte = *ptep; 1323 KDASSERT((pte & L2_TYPE_MASK) != L2_TYPE_INV); 1324 if (__predict_false((pte & L2_TYPE_MASK) 1325 == L2_TYPE_L)) { 1326 curaddr = (pte & L2_L_FRAME) | 1327 (vaddr & L2_L_OFFSET); 1328 if (pte & L2_L_CACHE_MASK) { 1329 map->_dm_flags &= 1330 ~_BUS_DMAMAP_COHERENT; 1331 } 1332 } else { 1333 curaddr = (pte & L2_S_FRAME) | 1334 (vaddr & L2_S_OFFSET); 1335 if (pte & L2_S_CACHE_MASK) { 1336 map->_dm_flags &= 1337 ~_BUS_DMAMAP_COHERENT; 1338 } 1339 } 1340 } 1341 } else { 1342 (void) pmap_extract(pmap, vaddr, &curaddr); 1343 map->_dm_flags &= ~_BUS_DMAMAP_COHERENT; 1344 } 1345 1346 /* 1347 * Compute the segment size, and adjust counts. 1348 */ 1349 sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET); 1350 if (buflen < sgsize) 1351 sgsize = buflen; 1352 1353 error = _bus_dmamap_load_paddr(t, map, curaddr, sgsize); 1354 if (error) 1355 return (error); 1356 1357 vaddr += sgsize; 1358 buflen -= sgsize; 1359 } 1360 1361 return (0); 1362 } 1363 1364 /* 1365 * Allocate physical memory from the given physical address range. 1366 * Called by DMA-safe memory allocation methods. 1367 */ 1368 int 1369 _bus_dmamem_alloc_range(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment, 1370 bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs, 1371 int flags, paddr_t low, paddr_t high) 1372 { 1373 paddr_t curaddr, lastaddr; 1374 struct vm_page *m; 1375 struct pglist mlist; 1376 int curseg, error; 1377 1378 #ifdef DEBUG_DMA 1379 printf("alloc_range: t=%p size=%lx align=%lx boundary=%lx segs=%p nsegs=%x rsegs=%p flags=%x lo=%lx hi=%lx\n", 1380 t, size, alignment, boundary, segs, nsegs, rsegs, flags, low, high); 1381 #endif /* DEBUG_DMA */ 1382 1383 /* Always round the size. */ 1384 size = round_page(size); 1385 1386 /* 1387 * Allocate pages from the VM system. 1388 */ 1389 error = uvm_pglistalloc(size, low, high, alignment, boundary, 1390 &mlist, nsegs, (flags & BUS_DMA_NOWAIT) == 0); 1391 if (error) 1392 return (error); 1393 1394 /* 1395 * Compute the location, size, and number of segments actually 1396 * returned by the VM code. 1397 */ 1398 m = TAILQ_FIRST(&mlist); 1399 curseg = 0; 1400 lastaddr = segs[curseg].ds_addr = VM_PAGE_TO_PHYS(m); 1401 segs[curseg].ds_len = PAGE_SIZE; 1402 #ifdef DEBUG_DMA 1403 printf("alloc: page %lx\n", lastaddr); 1404 #endif /* DEBUG_DMA */ 1405 m = TAILQ_NEXT(m, pageq.queue); 1406 1407 for (; m != NULL; m = TAILQ_NEXT(m, pageq.queue)) { 1408 curaddr = VM_PAGE_TO_PHYS(m); 1409 #ifdef DIAGNOSTIC 1410 if (curaddr < low || curaddr >= high) { 1411 printf("uvm_pglistalloc returned non-sensical" 1412 " address 0x%lx\n", curaddr); 1413 panic("_bus_dmamem_alloc_range"); 1414 } 1415 #endif /* DIAGNOSTIC */ 1416 #ifdef DEBUG_DMA 1417 printf("alloc: page %lx\n", curaddr); 1418 #endif /* DEBUG_DMA */ 1419 if (curaddr == (lastaddr + PAGE_SIZE)) 1420 segs[curseg].ds_len += PAGE_SIZE; 1421 else { 1422 curseg++; 1423 segs[curseg].ds_addr = curaddr; 1424 segs[curseg].ds_len = PAGE_SIZE; 1425 } 1426 lastaddr = curaddr; 1427 } 1428 1429 *rsegs = curseg + 1; 1430 1431 return (0); 1432 } 1433 1434 /* 1435 * Check if a memory region intersects with a DMA range, and return the 1436 * page-rounded intersection if it does. 1437 */ 1438 int 1439 arm32_dma_range_intersect(struct arm32_dma_range *ranges, int nranges, 1440 paddr_t pa, psize_t size, paddr_t *pap, psize_t *sizep) 1441 { 1442 struct arm32_dma_range *dr; 1443 int i; 1444 1445 if (ranges == NULL) 1446 return (0); 1447 1448 for (i = 0, dr = ranges; i < nranges; i++, dr++) { 1449 if (dr->dr_sysbase <= pa && 1450 pa < (dr->dr_sysbase + dr->dr_len)) { 1451 /* 1452 * Beginning of region intersects with this range. 1453 */ 1454 *pap = trunc_page(pa); 1455 *sizep = round_page(min(pa + size, 1456 dr->dr_sysbase + dr->dr_len) - pa); 1457 return (1); 1458 } 1459 if (pa < dr->dr_sysbase && dr->dr_sysbase < (pa + size)) { 1460 /* 1461 * End of region intersects with this range. 1462 */ 1463 *pap = trunc_page(dr->dr_sysbase); 1464 *sizep = round_page(min((pa + size) - dr->dr_sysbase, 1465 dr->dr_len)); 1466 return (1); 1467 } 1468 } 1469 1470 /* No intersection found. */ 1471 return (0); 1472 } 1473 1474 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 1475 static int 1476 _bus_dma_alloc_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map, 1477 bus_size_t size, int flags) 1478 { 1479 struct arm32_bus_dma_cookie *cookie = map->_dm_cookie; 1480 int error = 0; 1481 1482 #ifdef DIAGNOSTIC 1483 if (cookie == NULL) 1484 panic("_bus_dma_alloc_bouncebuf: no cookie"); 1485 #endif 1486 1487 cookie->id_bouncebuflen = round_page(size); 1488 error = _bus_dmamem_alloc(t, cookie->id_bouncebuflen, 1489 PAGE_SIZE, map->_dm_boundary, cookie->id_bouncesegs, 1490 map->_dm_segcnt, &cookie->id_nbouncesegs, flags); 1491 if (error) 1492 goto out; 1493 error = _bus_dmamem_map(t, cookie->id_bouncesegs, 1494 cookie->id_nbouncesegs, cookie->id_bouncebuflen, 1495 (void **)&cookie->id_bouncebuf, flags); 1496 1497 out: 1498 if (error) { 1499 _bus_dmamem_free(t, cookie->id_bouncesegs, 1500 cookie->id_nbouncesegs); 1501 cookie->id_bouncebuflen = 0; 1502 cookie->id_nbouncesegs = 0; 1503 } else { 1504 cookie->id_flags |= _BUS_DMA_HAS_BOUNCE; 1505 } 1506 1507 return (error); 1508 } 1509 1510 static void 1511 _bus_dma_free_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map) 1512 { 1513 struct arm32_bus_dma_cookie *cookie = map->_dm_cookie; 1514 1515 #ifdef DIAGNOSTIC 1516 if (cookie == NULL) 1517 panic("_bus_dma_alloc_bouncebuf: no cookie"); 1518 #endif 1519 1520 _bus_dmamem_unmap(t, cookie->id_bouncebuf, cookie->id_bouncebuflen); 1521 _bus_dmamem_free(t, cookie->id_bouncesegs, 1522 cookie->id_nbouncesegs); 1523 cookie->id_bouncebuflen = 0; 1524 cookie->id_nbouncesegs = 0; 1525 cookie->id_flags &= ~_BUS_DMA_HAS_BOUNCE; 1526 } 1527 1528 /* 1529 * This function does the same as uiomove, but takes an explicit 1530 * direction, and does not update the uio structure. 1531 */ 1532 static int 1533 _bus_dma_uiomove(void *buf, struct uio *uio, size_t n, int direction) 1534 { 1535 struct iovec *iov; 1536 int error; 1537 struct vmspace *vm; 1538 char *cp; 1539 size_t resid, cnt; 1540 int i; 1541 1542 iov = uio->uio_iov; 1543 vm = uio->uio_vmspace; 1544 cp = buf; 1545 resid = n; 1546 1547 for (i = 0; i < uio->uio_iovcnt && resid > 0; i++) { 1548 iov = &uio->uio_iov[i]; 1549 if (iov->iov_len == 0) 1550 continue; 1551 cnt = MIN(resid, iov->iov_len); 1552 1553 if (!VMSPACE_IS_KERNEL_P(vm) && 1554 (curlwp->l_cpu->ci_schedstate.spc_flags & SPCF_SHOULDYIELD) 1555 != 0) { 1556 preempt(); 1557 } 1558 if (direction == UIO_READ) { 1559 error = copyout_vmspace(vm, cp, iov->iov_base, cnt); 1560 } else { 1561 error = copyin_vmspace(vm, iov->iov_base, cp, cnt); 1562 } 1563 if (error) 1564 return (error); 1565 cp += cnt; 1566 resid -= cnt; 1567 } 1568 return (0); 1569 } 1570 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 1571 1572 int 1573 _bus_dmatag_subregion(bus_dma_tag_t tag, bus_addr_t min_addr, 1574 bus_addr_t max_addr, bus_dma_tag_t *newtag, int flags) 1575 { 1576 1577 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 1578 struct arm32_dma_range *dr; 1579 bool subset = false; 1580 size_t nranges = 0; 1581 size_t i; 1582 for (i = 0, dr = tag->_ranges; i < tag->_nranges; i++, dr++) { 1583 if (dr->dr_sysbase <= min_addr 1584 && max_addr <= dr->dr_sysbase + dr->dr_len - 1) { 1585 subset = true; 1586 } 1587 if (min_addr <= dr->dr_sysbase + dr->dr_len 1588 && max_addr >= dr->dr_sysbase) { 1589 nranges++; 1590 } 1591 } 1592 if (subset) { 1593 *newtag = tag; 1594 /* if the tag must be freed, add a reference */ 1595 if (tag->_tag_needs_free) 1596 (tag->_tag_needs_free)++; 1597 return 0; 1598 } 1599 if (nranges == 0) { 1600 nranges = 1; 1601 } 1602 1603 size_t mallocsize = sizeof(*tag) + nranges * sizeof(*dr); 1604 if ((*newtag = malloc(mallocsize, M_DMAMAP, 1605 (flags & BUS_DMA_NOWAIT) ? M_NOWAIT : M_WAITOK)) == NULL) 1606 return ENOMEM; 1607 1608 dr = (void *)(*newtag + 1); 1609 **newtag = *tag; 1610 (*newtag)->_tag_needs_free = 1; 1611 (*newtag)->_ranges = dr; 1612 (*newtag)->_nranges = nranges; 1613 1614 if (tag->_ranges == NULL) { 1615 dr->dr_sysbase = min_addr; 1616 dr->dr_busbase = min_addr; 1617 dr->dr_len = max_addr + 1 - min_addr; 1618 } else { 1619 for (i = 0; i < nranges; i++) { 1620 if (min_addr > dr->dr_sysbase + dr->dr_len 1621 || max_addr < dr->dr_sysbase) 1622 continue; 1623 dr[0] = tag->_ranges[i]; 1624 if (dr->dr_sysbase < min_addr) { 1625 psize_t diff = min_addr - dr->dr_sysbase; 1626 dr->dr_busbase += diff; 1627 dr->dr_len -= diff; 1628 dr->dr_sysbase += diff; 1629 } 1630 if (max_addr != 0xffffffff 1631 && max_addr + 1 < dr->dr_sysbase + dr->dr_len) { 1632 dr->dr_len = max_addr + 1 - dr->dr_sysbase; 1633 } 1634 dr++; 1635 } 1636 } 1637 1638 return 0; 1639 #else 1640 return EOPNOTSUPP; 1641 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */ 1642 } 1643 1644 void 1645 _bus_dmatag_destroy(bus_dma_tag_t tag) 1646 { 1647 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE 1648 switch (tag->_tag_needs_free) { 1649 case 0: 1650 break; /* not allocated with malloc */ 1651 case 1: 1652 free(tag, M_DMAMAP); /* last reference to tag */ 1653 break; 1654 default: 1655 (tag->_tag_needs_free)--; /* one less reference */ 1656 } 1657 #endif 1658 } 1659