xref: /netbsd-src/sys/arch/arm/arm32/bus_dma.c (revision c92dfce00230eefdddc782314f1f71d0e8a504d6)
1 /*	$NetBSD: bus_dma.c,v 1.148 2024/12/10 00:41:30 msaitoh Exp $	*/
2 
3 /*-
4  * Copyright (c) 1996, 1997, 1998, 2020 The NetBSD Foundation, Inc.
5  * All rights reserved.
6  *
7  * This code is derived from software contributed to The NetBSD Foundation
8  * by Jason R. Thorpe of the Numerical Aerospace Simulation Facility,
9  * NASA Ames Research Center.
10  *
11  * Redistribution and use in source and binary forms, with or without
12  * modification, are permitted provided that the following conditions
13  * are met:
14  * 1. Redistributions of source code must retain the above copyright
15  *    notice, this list of conditions and the following disclaimer.
16  * 2. Redistributions in binary form must reproduce the above copyright
17  *    notice, this list of conditions and the following disclaimer in the
18  *    documentation and/or other materials provided with the distribution.
19  *
20  * THIS SOFTWARE IS PROVIDED BY THE NETBSD FOUNDATION, INC. AND CONTRIBUTORS
21  * ``AS IS'' AND ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED
22  * TO, THE IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR
23  * PURPOSE ARE DISCLAIMED.  IN NO EVENT SHALL THE FOUNDATION OR CONTRIBUTORS
24  * BE LIABLE FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR
25  * CONSEQUENTIAL DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF
26  * SUBSTITUTE GOODS OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS
27  * INTERRUPTION) HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN
28  * CONTRACT, STRICT LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE)
29  * ARISING IN ANY WAY OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE
30  * POSSIBILITY OF SUCH DAMAGE.
31  */
32 
33 #define _ARM32_BUS_DMA_PRIVATE
34 
35 #include "opt_arm_bus_space.h"
36 #include "opt_cputypes.h"
37 
38 #include <sys/cdefs.h>
39 __KERNEL_RCSID(0, "$NetBSD: bus_dma.c,v 1.148 2024/12/10 00:41:30 msaitoh Exp $");
40 
41 #include <sys/param.h>
42 
43 #include <sys/bus.h>
44 #include <sys/cpu.h>
45 #include <sys/kmem.h>
46 #include <sys/mbuf.h>
47 
48 #include <uvm/uvm.h>
49 
50 #include <arm/cpuconf.h>
51 #include <arm/cpufunc.h>
52 
53 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
54 #include <dev/mm.h>
55 #endif
56 
57 #ifdef BUSDMA_COUNTERS
58 static struct evcnt bus_dma_creates =
59 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "creates");
60 static struct evcnt bus_dma_bounced_creates =
61 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced creates");
62 static struct evcnt bus_dma_loads =
63 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "loads");
64 static struct evcnt bus_dma_bounced_loads =
65 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced loads");
66 static struct evcnt bus_dma_coherent_loads =
67 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "coherent loads");
68 static struct evcnt bus_dma_read_bounces =
69 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "read bounces");
70 static struct evcnt bus_dma_write_bounces =
71 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "write bounces");
72 static struct evcnt bus_dma_bounced_unloads =
73 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced unloads");
74 static struct evcnt bus_dma_bounced_mbuf_loads =
75 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced mbuf loads");
76 static struct evcnt bus_dma_unloads =
77 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "unloads");
78 static struct evcnt bus_dma_bounced_destroys =
79 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "bounced destroys");
80 static struct evcnt bus_dma_destroys =
81 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "destroys");
82 static struct evcnt bus_dma_sync_prereadwrite =
83 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync prereadwrite");
84 static struct evcnt bus_dma_sync_preread_begin =
85 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync preread begin");
86 static struct evcnt bus_dma_sync_preread =
87 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync preread");
88 static struct evcnt bus_dma_sync_preread_tail =
89 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync preread tail");
90 static struct evcnt bus_dma_sync_prewrite =
91 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync prewrite");
92 static struct evcnt bus_dma_sync_postread =
93 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync postread");
94 static struct evcnt bus_dma_sync_postreadwrite =
95 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync postreadwrite");
96 static struct evcnt bus_dma_sync_postwrite =
97 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync postwrite");
98 static struct evcnt bus_dma_inrange_fail =
99 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "inrange check failed");
100 
101 static struct evcnt bus_dma_sync_coherent_prereadwrite =
102 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync coherent prereadwrite");
103 static struct evcnt bus_dma_sync_coherent_preread =
104 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync coherent preread");
105 static struct evcnt bus_dma_sync_coherent_prewrite =
106 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync coherent prewrite");
107 static struct evcnt bus_dma_sync_coherent_postread =
108 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync coherent postread");
109 static struct evcnt bus_dma_sync_coherent_postreadwrite =
110 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync coherent postreadwrite");
111 static struct evcnt bus_dma_sync_coherent_postwrite =
112 	EVCNT_INITIALIZER(EVCNT_TYPE_MISC, NULL, "busdma", "sync coherent postwrite");
113 
114 EVCNT_ATTACH_STATIC(bus_dma_creates);
115 EVCNT_ATTACH_STATIC(bus_dma_bounced_creates);
116 EVCNT_ATTACH_STATIC(bus_dma_loads);
117 EVCNT_ATTACH_STATIC(bus_dma_bounced_loads);
118 EVCNT_ATTACH_STATIC(bus_dma_coherent_loads);
119 EVCNT_ATTACH_STATIC(bus_dma_read_bounces);
120 EVCNT_ATTACH_STATIC(bus_dma_write_bounces);
121 EVCNT_ATTACH_STATIC(bus_dma_unloads);
122 EVCNT_ATTACH_STATIC(bus_dma_bounced_unloads);
123 EVCNT_ATTACH_STATIC(bus_dma_destroys);
124 EVCNT_ATTACH_STATIC(bus_dma_bounced_destroys);
125 EVCNT_ATTACH_STATIC(bus_dma_bounced_mbuf_loads);
126 EVCNT_ATTACH_STATIC(bus_dma_sync_prereadwrite);
127 EVCNT_ATTACH_STATIC(bus_dma_sync_preread_begin);
128 EVCNT_ATTACH_STATIC(bus_dma_sync_preread);
129 EVCNT_ATTACH_STATIC(bus_dma_sync_preread_tail);
130 EVCNT_ATTACH_STATIC(bus_dma_sync_prewrite);
131 EVCNT_ATTACH_STATIC(bus_dma_sync_postread);
132 EVCNT_ATTACH_STATIC(bus_dma_sync_postreadwrite);
133 EVCNT_ATTACH_STATIC(bus_dma_sync_postwrite);
134 EVCNT_ATTACH_STATIC(bus_dma_inrange_fail);
135 
136 EVCNT_ATTACH_STATIC(bus_dma_sync_coherent_prereadwrite);
137 EVCNT_ATTACH_STATIC(bus_dma_sync_coherent_preread);
138 EVCNT_ATTACH_STATIC(bus_dma_sync_coherent_prewrite);
139 EVCNT_ATTACH_STATIC(bus_dma_sync_coherent_postread);
140 EVCNT_ATTACH_STATIC(bus_dma_sync_coherent_postreadwrite);
141 EVCNT_ATTACH_STATIC(bus_dma_sync_coherent_postwrite);
142 
143 #define	STAT_INCR(x)	(bus_dma_ ## x.ev_count++)
144 #else
145 #define	STAT_INCR(x)	__nothing
146 #endif
147 
148 int	_bus_dmamap_load_buffer(bus_dma_tag_t, bus_dmamap_t, void *,
149 	    bus_size_t, struct vmspace *, int);
150 
151 /*
152  * Check to see if the specified page is in an allowed DMA range.
153  */
154 static inline struct arm32_dma_range *
155 _bus_dma_paddr_inrange(struct arm32_dma_range *ranges, int nranges,
156     bus_addr_t curaddr)
157 {
158 	struct arm32_dma_range *dr;
159 	int i;
160 
161 	for (i = 0, dr = ranges; i < nranges; i++, dr++) {
162 		if (curaddr >= dr->dr_sysbase &&
163 		    curaddr < (dr->dr_sysbase + dr->dr_len))
164 			return dr;
165 	}
166 
167 	return NULL;
168 }
169 
170 /*
171  * Check to see if the specified busaddr is in an allowed DMA range.
172  */
173 static inline paddr_t
174 _bus_dma_busaddr_to_paddr(bus_dma_tag_t t, bus_addr_t curaddr)
175 {
176 	struct arm32_dma_range *dr;
177 	u_int i;
178 
179 	if (t->_nranges == 0)
180 		return curaddr;
181 
182 	for (i = 0, dr = t->_ranges; i < t->_nranges; i++, dr++) {
183 		if (dr->dr_busbase <= curaddr
184 		    && curaddr < dr->dr_busbase + dr->dr_len)
185 			return curaddr - dr->dr_busbase + dr->dr_sysbase;
186 	}
187 	panic("%s: curaddr %#" PRIxBUSADDR "not in range", __func__, curaddr);
188 }
189 
190 /*
191  * Common function to load the specified physical address into the
192  * DMA map, coalescing segments and boundary checking as necessary.
193  */
194 static int
195 _bus_dmamap_load_paddr(bus_dma_tag_t t, bus_dmamap_t map,
196     bus_addr_t paddr, bus_size_t size, bool coherent)
197 {
198 	bus_dma_segment_t * const segs = map->dm_segs;
199 	int nseg = map->dm_nsegs;
200 	bus_addr_t lastaddr;
201 	bus_addr_t bmask = ~(map->_dm_boundary - 1);
202 	bus_addr_t curaddr;
203 	bus_size_t sgsize;
204 	uint32_t _ds_flags = coherent ? _BUS_DMAMAP_COHERENT : 0;
205 
206 	if (nseg > 0)
207 		lastaddr = segs[nseg - 1].ds_addr + segs[nseg - 1].ds_len;
208 	else
209 		lastaddr = 0xdead;
210 
211  again:
212 	sgsize = size;
213 
214 	/* Make sure we're in an allowed DMA range. */
215 	if (t->_ranges != NULL) {
216 		/* XXX cache last result? */
217 		const struct arm32_dma_range * const dr =
218 		    _bus_dma_paddr_inrange(t->_ranges, t->_nranges, paddr);
219 		if (__predict_false(dr == NULL)) {
220 			STAT_INCR(inrange_fail);
221 			return EINVAL;
222 		}
223 
224 		/*
225 		 * If this region is coherent, mark the segment as coherent.
226 		 */
227 		_ds_flags |= dr->dr_flags & _BUS_DMAMAP_COHERENT;
228 
229 		/*
230 		 * In a valid DMA range.  Translate the physical
231 		 * memory address to an address in the DMA window.
232 		 */
233 		curaddr = (paddr - dr->dr_sysbase) + dr->dr_busbase;
234 #if 0
235 		printf("%p: %#" PRIxPADDR
236 		    ": range %#" PRIxPADDR "/%#" PRIxBUSADDR
237 		    "/%#" PRIxBUSSIZE "/%#" PRIx32 ": %#" PRIx32
238 		    " <-- %#" PRIxBUSADDR "\n",
239 		    t, paddr, dr->dr_sysbase, dr->dr_busbase,
240 		    dr->dr_len, dr->dr_flags, _ds_flags, curaddr);
241 #endif
242 	} else
243 		curaddr = paddr;
244 
245 	/*
246 	 * Make sure we don't cross any boundaries.
247 	 */
248 	if (map->_dm_boundary > 0) {
249 		bus_addr_t baddr;	/* next boundary address */
250 
251 		baddr = (curaddr + map->_dm_boundary) & bmask;
252 		if (sgsize > (baddr - curaddr))
253 			sgsize = (baddr - curaddr);
254 	}
255 
256 	/*
257 	 * Insert chunk into a segment, coalescing with the
258 	 * previous segment if possible.
259 	 */
260 	if (nseg > 0 && curaddr == lastaddr &&
261 	    segs[nseg - 1].ds_len + sgsize <= map->dm_maxsegsz &&
262 	    ((segs[nseg - 1]._ds_flags ^ _ds_flags) & _BUS_DMAMAP_COHERENT) == 0 &&
263 	    (map->_dm_boundary == 0 ||
264 	     (segs[nseg - 1].ds_addr & bmask) == (curaddr & bmask))) {
265 		/* coalesce */
266 		segs[nseg - 1].ds_len += sgsize;
267 	} else if (__predict_false(nseg >= map->_dm_segcnt)) {
268 		return EFBIG;
269 	} else {
270 		/* new segment */
271 		segs[nseg].ds_addr = curaddr;
272 		segs[nseg].ds_len = sgsize;
273 		segs[nseg]._ds_paddr = curaddr;
274 		segs[nseg]._ds_flags = _ds_flags;
275 		nseg++;
276 	}
277 
278 	lastaddr = curaddr + sgsize;
279 
280 	paddr += sgsize;
281 	size -= sgsize;
282 	if (size > 0)
283 		goto again;
284 
285 	map->_dm_flags &= (_ds_flags & _BUS_DMAMAP_COHERENT);
286 	map->dm_nsegs = nseg;
287 	return 0;
288 }
289 
290 static int _bus_dma_uiomove(void *buf, struct uio *uio, size_t n,
291 	    int direction);
292 
293 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE
294 static int _bus_dma_alloc_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map,
295 	    bus_size_t size, int flags);
296 static void _bus_dma_free_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map);
297 
298 static int
299 _bus_dma_load_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
300 	size_t buflen, int buftype, int flags)
301 {
302 	struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie;
303 	struct vmspace * const vm = vmspace_kernel();
304 	int error;
305 
306 	KASSERT(cookie != NULL);
307 	KASSERT(cookie->id_flags & _BUS_DMA_MIGHT_NEED_BOUNCE);
308 
309 	/*
310 	 * Allocate bounce pages, if necessary.
311 	 */
312 	if ((cookie->id_flags & _BUS_DMA_HAS_BOUNCE) == 0) {
313 		error = _bus_dma_alloc_bouncebuf(t, map, buflen, flags);
314 		if (__predict_false(error))
315 			return error;
316 	}
317 
318 	/*
319 	 * Since we're trying again, clear the previous attempt.
320 	 */
321 	map->dm_mapsize = 0;
322 	map->dm_nsegs = 0;
323 	map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID;
324 	/* _bus_dmamap_load_buffer() clears this if we're not... */
325 	map->_dm_flags |= _BUS_DMAMAP_COHERENT;
326 
327 	/*
328 	 * Cache a pointer to the caller's buffer and load the DMA map
329 	 * with the bounce buffer.
330 	 */
331 	cookie->id_origbuf = buf;
332 	cookie->id_origbuflen = buflen;
333 	error = _bus_dmamap_load_buffer(t, map, cookie->id_bouncebuf,
334 	    buflen, vm, flags);
335 	if (__predict_false(error))
336 		return error;
337 
338 	STAT_INCR(bounced_loads);
339 	map->dm_mapsize = buflen;
340 	map->_dm_vmspace = vm;
341 	map->_dm_buftype = buftype;
342 
343 	/* ...so _bus_dmamap_sync() knows we're bouncing */
344 	map->_dm_flags |= _BUS_DMAMAP_IS_BOUNCING;
345 	cookie->id_flags |= _BUS_DMA_IS_BOUNCING;
346 	return 0;
347 }
348 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */
349 
350 /*
351  * Common function for DMA map creation.  May be called by bus-specific
352  * DMA map creation functions.
353  */
354 int
355 _bus_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments,
356     bus_size_t maxsegsz, bus_size_t boundary, int flags, bus_dmamap_t *dmamp)
357 {
358 	struct arm32_bus_dmamap *map;
359 	void *mapstore;
360 	int error = 0;
361 
362 #ifdef DEBUG_DMA
363 	printf("dmamap_create: t=%p size=%#" PRIxBUSSIZE
364 	    " nseg=%#x msegsz=%#" PRIxBUSSIZE
365 	    " boundary=%#" PRIxBUSSIZE
366 	    " flags=%#x\n", t, size, nsegments, maxsegsz, boundary, flags);
367 #endif	/* DEBUG_DMA */
368 
369 	/*
370 	 * Allocate and initialize the DMA map.  The end of the map
371 	 * is a variable-sized array of segments, so we allocate enough
372 	 * room for them in one shot.
373 	 *
374 	 * Note we don't preserve the WAITOK or NOWAIT flags.  Preservation
375 	 * of ALLOCNOW notifies others that we've reserved these resources,
376 	 * and they are not to be freed.
377 	 *
378 	 * The bus_dmamap_t includes one bus_dma_segment_t, hence
379 	 * the (nsegments - 1).
380 	 */
381 	const size_t mapsize = sizeof(struct arm32_bus_dmamap) +
382 	    (sizeof(bus_dma_segment_t) * (nsegments - 1));
383 	const int zallocflags = (flags & BUS_DMA_NOWAIT) ? KM_NOSLEEP : KM_SLEEP;
384 	if ((mapstore = kmem_intr_zalloc(mapsize, zallocflags)) == NULL)
385 		return ENOMEM;
386 
387 	map = (struct arm32_bus_dmamap *)mapstore;
388 	map->_dm_size = size;
389 	map->_dm_segcnt = nsegments;
390 	map->_dm_maxmaxsegsz = maxsegsz;
391 	map->_dm_boundary = boundary;
392 	map->_dm_flags = flags & ~(BUS_DMA_WAITOK | BUS_DMA_NOWAIT);
393 	map->_dm_origbuf = NULL;
394 	map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID;
395 	map->_dm_vmspace = vmspace_kernel();
396 	map->_dm_cookie = NULL;
397 	map->dm_maxsegsz = maxsegsz;
398 	map->dm_mapsize = 0;		/* no valid mappings */
399 	map->dm_nsegs = 0;
400 
401 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE
402 	struct arm32_bus_dma_cookie *cookie;
403 	int cookieflags;
404 	void *cookiestore;
405 
406 	cookieflags = 0;
407 
408 	if (t->_may_bounce != NULL) {
409 		error = (*t->_may_bounce)(t, map, flags, &cookieflags);
410 		if (error != 0)
411 			goto out;
412 	}
413 
414 	if (t->_ranges != NULL) {
415 		/*
416 		 * If ranges are defined, we may have to bounce. The only
417 		 * exception is if there is exactly one range that covers
418 		 * all of physical memory.
419 		 */
420 		switch (t->_nranges) {
421 		case 1:
422 			if (t->_ranges[0].dr_sysbase == 0 &&
423 			    t->_ranges[0].dr_len == UINTPTR_MAX) {
424 				break;
425 			}
426 			/* FALLTHROUGH */
427 		default:
428 			cookieflags |= _BUS_DMA_MIGHT_NEED_BOUNCE;
429 		}
430 	}
431 
432 	if ((cookieflags & _BUS_DMA_MIGHT_NEED_BOUNCE) == 0) {
433 		STAT_INCR(creates);
434 		*dmamp = map;
435 		return 0;
436 	}
437 
438 	const size_t cookiesize = sizeof(struct arm32_bus_dma_cookie) +
439 	    (sizeof(bus_dma_segment_t) * map->_dm_segcnt);
440 
441 	/*
442 	 * Allocate our cookie.
443 	 */
444 	if ((cookiestore = kmem_intr_zalloc(cookiesize, zallocflags)) == NULL) {
445 		error = ENOMEM;
446 		goto out;
447 	}
448 	cookie = (struct arm32_bus_dma_cookie *)cookiestore;
449 	cookie->id_flags = cookieflags;
450 	map->_dm_cookie = cookie;
451 	STAT_INCR(bounced_creates);
452 
453 	error = _bus_dma_alloc_bouncebuf(t, map, size, flags);
454  out:
455 	if (error)
456 		_bus_dmamap_destroy(t, map);
457 	else
458 		*dmamp = map;
459 #else
460 	*dmamp = map;
461 	STAT_INCR(creates);
462 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */
463 #ifdef DEBUG_DMA
464 	printf("dmamap_create:map=%p\n", map);
465 #endif	/* DEBUG_DMA */
466 	return error;
467 }
468 
469 /*
470  * Common function for DMA map destruction.  May be called by bus-specific
471  * DMA map destruction functions.
472  */
473 void
474 _bus_dmamap_destroy(bus_dma_tag_t t, bus_dmamap_t map)
475 {
476 
477 #ifdef DEBUG_DMA
478 	printf("dmamap_destroy: t=%p map=%p\n", t, map);
479 #endif	/* DEBUG_DMA */
480 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE
481 	struct arm32_bus_dma_cookie *cookie = map->_dm_cookie;
482 
483 	/*
484 	 * Free any bounce pages this map might hold.
485 	 */
486 	if (cookie != NULL) {
487 		const size_t cookiesize = sizeof(struct arm32_bus_dma_cookie) +
488 		    (sizeof(bus_dma_segment_t) * map->_dm_segcnt);
489 
490 		if (cookie->id_flags & _BUS_DMA_IS_BOUNCING)
491 			STAT_INCR(bounced_unloads);
492 		map->dm_nsegs = 0;
493 		if (cookie->id_flags & _BUS_DMA_HAS_BOUNCE)
494 			_bus_dma_free_bouncebuf(t, map);
495 		STAT_INCR(bounced_destroys);
496 		kmem_intr_free(cookie, cookiesize);
497 	} else
498 #endif
499 	STAT_INCR(destroys);
500 
501 	if (map->dm_nsegs > 0)
502 		STAT_INCR(unloads);
503 
504 	const size_t mapsize = sizeof(struct arm32_bus_dmamap) +
505 	    (sizeof(bus_dma_segment_t) * (map->_dm_segcnt - 1));
506 	kmem_intr_free(map, mapsize);
507 }
508 
509 /*
510  * Common function for loading a DMA map with a linear buffer.  May
511  * be called by bus-specific DMA map load functions.
512  */
513 int
514 _bus_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
515     bus_size_t buflen, struct proc *p, int flags)
516 {
517 	struct vmspace *vm;
518 	int error;
519 
520 #ifdef DEBUG_DMA
521 	printf("dmamap_load: t=%p map=%p buf=%p len=%#" PRIxBUSSIZE
522 	    " p=%p f=%#x\n", t, map, buf, buflen, p, flags);
523 #endif	/* DEBUG_DMA */
524 
525 	if (map->dm_nsegs > 0) {
526 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE
527 		struct arm32_bus_dma_cookie *cookie = map->_dm_cookie;
528 		if (cookie != NULL) {
529 			if (cookie->id_flags & _BUS_DMA_IS_BOUNCING) {
530 				STAT_INCR(bounced_unloads);
531 				cookie->id_flags &= ~_BUS_DMA_IS_BOUNCING;
532 				map->_dm_flags &= ~_BUS_DMAMAP_IS_BOUNCING;
533 			}
534 		} else
535 #endif
536 		STAT_INCR(unloads);
537 	}
538 
539 	/*
540 	 * Make sure that on error condition we return "no valid mappings".
541 	 */
542 	map->dm_mapsize = 0;
543 	map->dm_nsegs = 0;
544 	map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID;
545 	KASSERTMSG(map->dm_maxsegsz <= map->_dm_maxmaxsegsz,
546 	    "dm_maxsegsz %" PRIuBUSSIZE " _dm_maxmaxsegsz %" PRIuBUSSIZE,
547 	    map->dm_maxsegsz, map->_dm_maxmaxsegsz);
548 
549 	if (__predict_false(buflen > map->_dm_size))
550 		return EINVAL;
551 
552 	if (p != NULL) {
553 		vm = p->p_vmspace;
554 	} else {
555 		vm = vmspace_kernel();
556 	}
557 
558 	/* _bus_dmamap_load_buffer() clears this if we're not... */
559 	map->_dm_flags |= _BUS_DMAMAP_COHERENT;
560 
561 	error = _bus_dmamap_load_buffer(t, map, buf, buflen, vm, flags);
562 	if (__predict_true(error == 0)) {
563 		map->dm_mapsize = buflen;
564 		map->_dm_vmspace = vm;
565 		map->_dm_origbuf = buf;
566 		map->_dm_buftype = _BUS_DMA_BUFTYPE_LINEAR;
567 		if (map->_dm_flags & _BUS_DMAMAP_COHERENT) {
568 			STAT_INCR(coherent_loads);
569 		} else {
570 			STAT_INCR(loads);
571 		}
572 		return 0;
573 	}
574 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE
575 	struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie;
576 	if (cookie != NULL && (cookie->id_flags & _BUS_DMA_MIGHT_NEED_BOUNCE)) {
577 		error = _bus_dma_load_bouncebuf(t, map, buf, buflen,
578 		    _BUS_DMA_BUFTYPE_LINEAR, flags);
579 	}
580 #endif
581 	return error;
582 }
583 
584 /*
585  * Like _bus_dmamap_load(), but for mbufs.
586  */
587 int
588 _bus_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map, struct mbuf *m0,
589     int flags)
590 {
591 	struct mbuf *m;
592 	int error;
593 
594 #ifdef DEBUG_DMA
595 	printf("dmamap_load_mbuf: t=%p map=%p m0=%p f=%#x\n",
596 	    t, map, m0, flags);
597 #endif	/* DEBUG_DMA */
598 
599 	if (map->dm_nsegs > 0) {
600 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE
601 		struct arm32_bus_dma_cookie *cookie = map->_dm_cookie;
602 		if (cookie != NULL) {
603 			if (cookie->id_flags & _BUS_DMA_IS_BOUNCING) {
604 				STAT_INCR(bounced_unloads);
605 				cookie->id_flags &= ~_BUS_DMA_IS_BOUNCING;
606 				map->_dm_flags &= ~_BUS_DMAMAP_IS_BOUNCING;
607 			}
608 		} else
609 #endif
610 		STAT_INCR(unloads);
611 	}
612 
613 	/*
614 	 * Make sure that on error condition we return "no valid mappings."
615 	 */
616 	map->dm_mapsize = 0;
617 	map->dm_nsegs = 0;
618 	map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID;
619 	KASSERTMSG(map->dm_maxsegsz <= map->_dm_maxmaxsegsz,
620 	    "dm_maxsegsz %" PRIuBUSSIZE " _dm_maxmaxsegsz %" PRIuBUSSIZE,
621 	    map->dm_maxsegsz, map->_dm_maxmaxsegsz);
622 
623 	KASSERT(m0->m_flags & M_PKTHDR);
624 
625 	if (__predict_false(m0->m_pkthdr.len > map->_dm_size))
626 		return EINVAL;
627 
628 	/* _bus_dmamap_load_paddr() clears this if we're not... */
629 	map->_dm_flags |= _BUS_DMAMAP_COHERENT;
630 
631 	error = 0;
632 	for (m = m0; m != NULL && error == 0; m = m->m_next) {
633 		int offset;
634 		int remainbytes;
635 		const struct vm_page * const *pgs;
636 		paddr_t paddr;
637 		int size;
638 
639 		if (m->m_len == 0)
640 			continue;
641 		/*
642 		 * Don't allow reads in read-only mbufs.
643 		 */
644 		if (__predict_false(M_ROMAP(m) && (flags & BUS_DMA_READ))) {
645 			error = EFAULT;
646 			break;
647 		}
648 		switch (m->m_flags & (M_EXT | M_EXT_CLUSTER | M_EXT_PAGES)) {
649 		case M_EXT | M_EXT_CLUSTER:
650 			KASSERT(m->m_ext.ext_paddr != M_PADDR_INVALID);
651 			paddr = m->m_ext.ext_paddr +
652 			    (m->m_data - m->m_ext.ext_buf);
653 			size = m->m_len;
654 			error = _bus_dmamap_load_paddr(t, map, paddr, size,
655 			    false);
656 			break;
657 
658 		case M_EXT | M_EXT_PAGES:
659 			KASSERT(m->m_ext.ext_buf <= m->m_data);
660 			KASSERT(m->m_data <=
661 			    m->m_ext.ext_buf + m->m_ext.ext_size);
662 
663 			offset = (vaddr_t)m->m_data -
664 			    trunc_page((vaddr_t)m->m_ext.ext_buf);
665 			remainbytes = m->m_len;
666 
667 			/* skip uninteresting pages */
668 			pgs = (const struct vm_page * const *)
669 			    m->m_ext.ext_pgs + (offset >> PAGE_SHIFT);
670 
671 			offset &= PAGE_MASK;	/* offset in the first page */
672 
673 			/* load each page */
674 			while (remainbytes > 0) {
675 				const struct vm_page *pg;
676 
677 				size = MIN(remainbytes, PAGE_SIZE - offset);
678 
679 				pg = *pgs++;
680 				KASSERT(pg);
681 				paddr = VM_PAGE_TO_PHYS(pg) + offset;
682 
683 				error = _bus_dmamap_load_paddr(t, map,
684 				    paddr, size, false);
685 				if (__predict_false(error))
686 					break;
687 				offset = 0;
688 				remainbytes -= size;
689 			}
690 			break;
691 
692 		case 0:
693 			paddr = m->m_paddr + M_BUFOFFSET(m) +
694 			    (m->m_data - M_BUFADDR(m));
695 			size = m->m_len;
696 			error = _bus_dmamap_load_paddr(t, map, paddr, size,
697 			    false);
698 			break;
699 
700 		default:
701 			error = _bus_dmamap_load_buffer(t, map, m->m_data,
702 			    m->m_len, vmspace_kernel(), flags);
703 		}
704 	}
705 	if (__predict_true(error == 0)) {
706 		map->dm_mapsize = m0->m_pkthdr.len;
707 		map->_dm_origbuf = m0;
708 		map->_dm_buftype = _BUS_DMA_BUFTYPE_MBUF;
709 		map->_dm_vmspace = vmspace_kernel();	/* always kernel */
710 		if (map->_dm_flags & _BUS_DMAMAP_COHERENT) {
711 			STAT_INCR(coherent_loads);
712 		} else {
713 			STAT_INCR(loads);
714 		}
715 		return 0;
716 	}
717 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE
718 	struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie;
719 	if (cookie != NULL && (cookie->id_flags & _BUS_DMA_MIGHT_NEED_BOUNCE)) {
720 		error = _bus_dma_load_bouncebuf(t, map, m0, m0->m_pkthdr.len,
721 		    _BUS_DMA_BUFTYPE_MBUF, flags);
722 		STAT_INCR(bounced_mbuf_loads);
723 	}
724 #endif
725 	return error;
726 }
727 
728 /*
729  * Like _bus_dmamap_load(), but for uios.
730  */
731 int
732 _bus_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map, struct uio *uio,
733     int flags)
734 {
735 	bus_size_t minlen, resid;
736 	struct iovec *iov;
737 	void *addr;
738 	int i, error;
739 
740 	/*
741 	 * Make sure that on error condition we return "no valid mappings."
742 	 */
743 	map->dm_mapsize = 0;
744 	map->dm_nsegs = 0;
745 	KASSERTMSG(map->dm_maxsegsz <= map->_dm_maxmaxsegsz,
746 	    "dm_maxsegsz %" PRIuBUSSIZE " _dm_maxmaxsegsz %" PRIuBUSSIZE,
747 	    map->dm_maxsegsz, map->_dm_maxmaxsegsz);
748 
749 	resid = uio->uio_resid;
750 	iov = uio->uio_iov;
751 
752 	/* _bus_dmamap_load_buffer() clears this if we're not... */
753 	map->_dm_flags |= _BUS_DMAMAP_COHERENT;
754 
755 	error = 0;
756 	for (i = 0; i < uio->uio_iovcnt && resid != 0 && error == 0; i++) {
757 		/*
758 		 * Now at the first iovec to load.  Load each iovec
759 		 * until we have exhausted the residual count.
760 		 */
761 		minlen = resid < iov[i].iov_len ? resid : iov[i].iov_len;
762 		addr = (void *)iov[i].iov_base;
763 
764 		error = _bus_dmamap_load_buffer(t, map, addr, minlen,
765 		    uio->uio_vmspace, flags);
766 
767 		resid -= minlen;
768 	}
769 	if (__predict_true(error == 0)) {
770 		map->dm_mapsize = uio->uio_resid;
771 		map->_dm_origbuf = uio;
772 		map->_dm_buftype = _BUS_DMA_BUFTYPE_UIO;
773 		map->_dm_vmspace = uio->uio_vmspace;
774 		if (map->_dm_flags & _BUS_DMAMAP_COHERENT) {
775 			STAT_INCR(coherent_loads);
776 		} else {
777 			STAT_INCR(loads);
778 		}
779 	}
780 	return error;
781 }
782 
783 /*
784  * Like _bus_dmamap_load(), but for raw memory allocated with
785  * bus_dmamem_alloc().
786  */
787 int
788 _bus_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map,
789     bus_dma_segment_t *segs, int nsegs, bus_size_t size0, int flags)
790 {
791 
792 	bus_size_t size;
793 	int i, error = 0;
794 
795 	/*
796 	 * Make sure that on error conditions we return "no valid mappings."
797 	 */
798 	map->dm_mapsize = 0;
799 	map->dm_nsegs = 0;
800 	KASSERT(map->dm_maxsegsz <= map->_dm_maxmaxsegsz);
801 
802 	if (__predict_false(size0 > map->_dm_size))
803 		return EINVAL;
804 
805 	for (i = 0, size = size0; i < nsegs && size > 0; i++) {
806 		bus_dma_segment_t *ds = &segs[i];
807 		bus_size_t sgsize;
808 
809 		sgsize = MIN(ds->ds_len, size);
810 		if (sgsize == 0)
811 			continue;
812 		const bool coherent =
813 		    (ds->_ds_flags & _BUS_DMAMAP_COHERENT) != 0;
814 		error = _bus_dmamap_load_paddr(t, map, ds->ds_addr,
815 		    sgsize, coherent);
816 		if (__predict_false(error != 0))
817 			break;
818 		size -= sgsize;
819 	}
820 
821 	if (__predict_false(error != 0)) {
822 		map->dm_mapsize = 0;
823 		map->dm_nsegs = 0;
824 		return error;
825 	}
826 
827 	/* XXX TBD bounce */
828 
829 	map->dm_mapsize = size0;
830 	map->_dm_origbuf = NULL;
831 	map->_dm_buftype = _BUS_DMA_BUFTYPE_RAW;
832 	map->_dm_vmspace = NULL;
833 	return 0;
834 }
835 
836 /*
837  * Common function for unloading a DMA map.  May be called by
838  * bus-specific DMA map unload functions.
839  */
840 void
841 _bus_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
842 {
843 
844 #ifdef DEBUG_DMA
845 	printf("dmamap_unload: t=%p map=%p\n", t, map);
846 #endif	/* DEBUG_DMA */
847 
848 	/*
849 	 * No resources to free; just mark the mappings as
850 	 * invalid.
851 	 */
852 	map->dm_mapsize = 0;
853 	map->dm_nsegs = 0;
854 	map->_dm_origbuf = NULL;
855 	map->_dm_buftype = _BUS_DMA_BUFTYPE_INVALID;
856 	map->_dm_vmspace = NULL;
857 }
858 
859 static void
860 _bus_dmamap_sync_segment(vaddr_t va, paddr_t pa, vsize_t len, int ops,
861     bool readonly_p)
862 {
863 
864 #if defined(ARM_MMU_EXTENDED)
865 	/*
866 	 * No optimisations are available for readonly mbufs on armv6+, so
867 	 * assume it's not readonly from here on.
868 	 *
869  	 * See the comment in _bus_dmamap_sync_mbuf
870 	 */
871 	readonly_p = false;
872 #endif
873 
874 	KASSERTMSG((va & PAGE_MASK) == (pa & PAGE_MASK),
875 	    "va %#" PRIxVADDR " pa %#" PRIxPADDR, va, pa);
876 #if 0
877 	printf("sync_segment: va=%#" PRIxVADDR
878 	    " pa=%#" PRIxPADDR " len=%#" PRIxVSIZE " ops=%#x\n",
879 	    va, pa, len, ops);
880 #endif
881 
882 	switch (ops) {
883 	case BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE:
884 		if (!readonly_p) {
885 			STAT_INCR(sync_prereadwrite);
886 			cpu_dcache_wbinv_range(va, len);
887 			cpu_sdcache_wbinv_range(va, pa, len);
888 			break;
889 		}
890 		/* FALLTHROUGH */
891 
892 	case BUS_DMASYNC_PREREAD: {
893 		const size_t line_size = arm_dcache_align;
894 		const size_t line_mask = arm_dcache_align_mask;
895 		vsize_t misalignment = va & line_mask;
896 		if (misalignment) {
897 			va -= misalignment;
898 			pa -= misalignment;
899 			len += misalignment;
900 			STAT_INCR(sync_preread_begin);
901 			cpu_dcache_wbinv_range(va, line_size);
902 			cpu_sdcache_wbinv_range(va, pa, line_size);
903 			if (len <= line_size)
904 				break;
905 			va += line_size;
906 			pa += line_size;
907 			len -= line_size;
908 		}
909 		misalignment = len & line_mask;
910 		len -= misalignment;
911 		if (len > 0) {
912 			STAT_INCR(sync_preread);
913 			cpu_dcache_inv_range(va, len);
914 			cpu_sdcache_inv_range(va, pa, len);
915 		}
916 		if (misalignment) {
917 			va += len;
918 			pa += len;
919 			STAT_INCR(sync_preread_tail);
920 			cpu_dcache_wbinv_range(va, line_size);
921 			cpu_sdcache_wbinv_range(va, pa, line_size);
922 		}
923 		break;
924 	}
925 
926 	case BUS_DMASYNC_PREWRITE:
927 		STAT_INCR(sync_prewrite);
928 		cpu_dcache_wb_range(va, len);
929 		cpu_sdcache_wb_range(va, pa, len);
930 		break;
931 
932 #if defined(CPU_CORTEX) || defined(CPU_ARMV8)
933 
934 	/*
935 	 * Cortex CPUs can do speculative loads so we need to clean the cache
936 	 * after a DMA read to deal with any speculatively loaded cache lines.
937 	 * Since these can't be dirty, we can just invalidate them and don't
938 	 * have to worry about having to write back their contents.
939 	 */
940 	case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE:
941 		STAT_INCR(sync_postreadwrite);
942 		cpu_dcache_inv_range(va, len);
943 		cpu_sdcache_inv_range(va, pa, len);
944 		break;
945 
946 	case BUS_DMASYNC_POSTREAD:
947 		STAT_INCR(sync_postread);
948 		cpu_dcache_inv_range(va, len);
949 		cpu_sdcache_inv_range(va, pa, len);
950 		break;
951 #endif
952 	}
953 }
954 
955 static inline void
956 _bus_dmamap_sync_linear(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
957     bus_size_t len, int ops)
958 {
959 	bus_dma_segment_t *ds = map->dm_segs;
960 	vaddr_t va = (vaddr_t) map->_dm_origbuf;
961 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE
962 	if (map->_dm_flags & _BUS_DMAMAP_IS_BOUNCING) {
963 		struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie;
964 		va = (vaddr_t) cookie->id_bouncebuf;
965 	}
966 #endif
967 
968 	while (len > 0) {
969 		while (offset >= ds->ds_len) {
970 			offset -= ds->ds_len;
971 			va += ds->ds_len;
972 			ds++;
973 		}
974 
975 		paddr_t pa = _bus_dma_busaddr_to_paddr(t, ds->ds_addr + offset);
976 		size_t seglen = uimin(len, ds->ds_len - offset);
977 
978 		if ((ds->_ds_flags & _BUS_DMAMAP_COHERENT) == 0)
979 			_bus_dmamap_sync_segment(va + offset, pa, seglen, ops,
980 			    false);
981 
982 		offset += seglen;
983 		len -= seglen;
984 	}
985 }
986 
987 static inline void
988 _bus_dmamap_sync_mbuf(bus_dma_tag_t t, bus_dmamap_t map, bus_size_t offset,
989     bus_size_t len, int ops)
990 {
991 	bus_dma_segment_t *ds = map->dm_segs;
992 	struct mbuf *m = map->_dm_origbuf;
993 	bus_size_t voff = offset;
994 	bus_size_t ds_off = offset;
995 
996 	while (len > 0) {
997 		/* Find the current dma segment */
998 		while (ds_off >= ds->ds_len) {
999 			ds_off -= ds->ds_len;
1000 			ds++;
1001 		}
1002 		/* Find the current mbuf. */
1003 		while (voff >= m->m_len) {
1004 			voff -= m->m_len;
1005 			m = m->m_next;
1006 		}
1007 
1008 		/*
1009 		 * Now at the first mbuf to sync; nail each one until
1010 		 * we have exhausted the length.
1011 		 */
1012 		vsize_t seglen = uimin(len, uimin(m->m_len - voff, ds->ds_len - ds_off));
1013 		vaddr_t va = mtod(m, vaddr_t) + voff;
1014 		paddr_t pa = _bus_dma_busaddr_to_paddr(t, ds->ds_addr + ds_off);
1015 
1016 		/*
1017 		 * We can save a lot of work here if we know the mapping
1018 		 * is read-only at the MMU and we aren't using the armv6+
1019 		 * MMU:
1020 		 *
1021 		 * If a mapping is read-only, no dirty cache blocks will
1022 		 * exist for it.  If a writable mapping was made read-only,
1023 		 * we know any dirty cache lines for the range will have
1024 		 * been cleaned for us already.  Therefore, if the upper
1025 		 * layer can tell us we have a read-only mapping, we can
1026 		 * skip all cache cleaning.
1027 		 *
1028 		 * NOTE: This only works if we know the pmap cleans pages
1029 		 * before making a read-write -> read-only transition.  If
1030 		 * this ever becomes non-true (e.g. Physically Indexed
1031 		 * cache), this will have to be revisited.
1032 		 */
1033 
1034 		if ((ds->_ds_flags & _BUS_DMAMAP_COHERENT) == 0) {
1035 			/*
1036 			 * If we are doing preread (DMAing into the mbuf),
1037 			 * this mbuf better not be readonly,
1038 			 */
1039 			KASSERT(!(ops & BUS_DMASYNC_PREREAD) || !M_ROMAP(m));
1040 			_bus_dmamap_sync_segment(va, pa, seglen, ops,
1041 			    M_ROMAP(m));
1042 		}
1043 		voff += seglen;
1044 		ds_off += seglen;
1045 		len -= seglen;
1046 	}
1047 }
1048 
1049 static inline void
1050 _bus_dmamap_sync_uio(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
1051     bus_size_t len, int ops)
1052 {
1053 	bus_dma_segment_t *ds = map->dm_segs;
1054 	struct uio *uio = map->_dm_origbuf;
1055 	struct iovec *iov = uio->uio_iov;
1056 	bus_size_t voff = offset;
1057 	bus_size_t ds_off = offset;
1058 
1059 	while (len > 0) {
1060 		/* Find the current dma segment */
1061 		while (ds_off >= ds->ds_len) {
1062 			ds_off -= ds->ds_len;
1063 			ds++;
1064 		}
1065 
1066 		/* Find the current iovec. */
1067 		while (voff >= iov->iov_len) {
1068 			voff -= iov->iov_len;
1069 			iov++;
1070 		}
1071 
1072 		/*
1073 		 * Now at the first iovec to sync; nail each one until
1074 		 * we have exhausted the length.
1075 		 */
1076 		vsize_t seglen = uimin(len, uimin(iov->iov_len - voff, ds->ds_len - ds_off));
1077 		vaddr_t va = (vaddr_t) iov->iov_base + voff;
1078 		paddr_t pa = _bus_dma_busaddr_to_paddr(t, ds->ds_addr + ds_off);
1079 
1080 		if ((ds->_ds_flags & _BUS_DMAMAP_COHERENT) == 0)
1081 			_bus_dmamap_sync_segment(va, pa, seglen, ops, false);
1082 
1083 		voff += seglen;
1084 		ds_off += seglen;
1085 		len -= seglen;
1086 	}
1087 }
1088 
1089 /*
1090  * Common function for DMA map synchronization.  May be called
1091  * by bus-specific DMA map synchronization functions.
1092  *
1093  * XXX Should have separate versions for write-through vs.
1094  * XXX write-back caches.  We currently assume write-back
1095  * XXX here, which is not as efficient as it could be for
1096  * XXX the write-through case.
1097  */
1098 void
1099 _bus_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map, bus_addr_t offset,
1100     bus_size_t len, int ops)
1101 {
1102 #ifdef DEBUG_DMA
1103 	printf("dmamap_sync: t=%p map=%p offset=%#" PRIxBUSADDR
1104 	    " len=%#" PRIxBUSSIZE " ops=%#x\n", t, map, offset, len, ops);
1105 #endif	/* DEBUG_DMA */
1106 
1107 	/*
1108 	 * Mixing of PRE and POST operations is not allowed.
1109 	 */
1110 	KASSERTMSG((((ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE)) == 0)
1111 	    || ((ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE)) == 0)),
1112 	    "%s: mix PRE and POST", __func__);
1113 
1114 	KASSERTMSG(offset < map->dm_mapsize,
1115 	    "offset %" PRIxBUSADDR " mapsize %" PRIuBUSSIZE,
1116 	    offset, map->dm_mapsize);
1117 	KASSERTMSG(len > 0 && offset + len <= map->dm_mapsize,
1118 	    "len %" PRIuBUSSIZE " offset %" PRIxBUSADDR " mapsize %" PRIuBUSSIZE,
1119 	    len, offset, map->dm_mapsize);
1120 
1121 	/*
1122 	 * For a write-back cache, we need to do the following things:
1123 	 *
1124 	 *	PREREAD -- Invalidate the D-cache.  We do this
1125 	 *	here in case a write-back is required by the back-end.
1126 	 *
1127 	 *	PREWRITE -- Write-back the D-cache.  Note that if
1128 	 *	we are doing a PREREAD | PREWRITE, we can collapse
1129 	 *	the whole thing into a single Wb-Inv.
1130 	 *
1131 	 *	POSTREAD -- Re-invalidate the D-cache in case speculative
1132 	 *	memory accesses caused cachelines to become valid with now
1133 	 *	invalid data.
1134 	 *
1135 	 *	POSTWRITE -- Nothing.
1136 	 */
1137 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE
1138 	const bool bouncing = (map->_dm_flags & _BUS_DMAMAP_IS_BOUNCING);
1139 #else
1140 	const bool bouncing = false;
1141 #endif
1142 
1143 	const int pre_ops = ops & (BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE);
1144 #if defined(CPU_CORTEX) || defined(CPU_ARMV8)
1145 	const int post_ops = ops & (BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE);
1146 #else
1147 	const int post_ops = 0;
1148 #endif
1149 	if (pre_ops == 0 && post_ops == 0)
1150 		return;
1151 
1152 	if (post_ops == BUS_DMASYNC_POSTWRITE) {
1153 		KASSERT(pre_ops == 0);
1154 		if ((map->_dm_flags & _BUS_DMAMAP_COHERENT)) {
1155 			STAT_INCR(sync_coherent_postwrite);
1156 		} else {
1157 			STAT_INCR(sync_postwrite);
1158 		}
1159 		return;
1160 	}
1161 
1162 	KASSERTMSG(bouncing || pre_ops != 0 || (post_ops & BUS_DMASYNC_POSTREAD),
1163 	    "pre_ops %#x post_ops %#x", pre_ops, post_ops);
1164 
1165 	if (bouncing && (ops & BUS_DMASYNC_PREWRITE)) {
1166 		struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie;
1167 		STAT_INCR(write_bounces);
1168 		char * const dataptr = (char *)cookie->id_bouncebuf + offset;
1169 		/*
1170 		 * Copy the caller's buffer to the bounce buffer.
1171 		 */
1172 		switch (map->_dm_buftype) {
1173 		case _BUS_DMA_BUFTYPE_LINEAR:
1174 			memcpy(dataptr, cookie->id_origlinearbuf + offset, len);
1175 			break;
1176 
1177 		case _BUS_DMA_BUFTYPE_MBUF:
1178 			m_copydata(cookie->id_origmbuf, offset, len, dataptr);
1179 			break;
1180 
1181 		case _BUS_DMA_BUFTYPE_UIO:
1182 			_bus_dma_uiomove(dataptr, cookie->id_origuio, len,
1183 			    UIO_WRITE);
1184 			break;
1185 
1186 #ifdef DIAGNOSTIC
1187 		case _BUS_DMA_BUFTYPE_RAW:
1188 			panic("%s:(pre): _BUS_DMA_BUFTYPE_RAW", __func__);
1189 			break;
1190 
1191 		case _BUS_DMA_BUFTYPE_INVALID:
1192 			panic("%s(pre): _BUS_DMA_BUFTYPE_INVALID", __func__);
1193 			break;
1194 
1195 		default:
1196 			panic("%s(pre): map %p: unknown buffer type %d\n",
1197 			    __func__, map, map->_dm_buftype);
1198 			break;
1199 #endif /* DIAGNOSTIC */
1200 		}
1201 	}
1202 
1203 	/* Skip cache frobbing if mapping was COHERENT */
1204 	if ((map->_dm_flags & _BUS_DMAMAP_COHERENT)) {
1205 		switch (ops) {
1206 		case BUS_DMASYNC_PREREAD | BUS_DMASYNC_PREWRITE:
1207 			STAT_INCR(sync_coherent_prereadwrite);
1208 			break;
1209 
1210 		case BUS_DMASYNC_PREREAD:
1211 			STAT_INCR(sync_coherent_preread);
1212 			break;
1213 
1214 		case BUS_DMASYNC_PREWRITE:
1215 			STAT_INCR(sync_coherent_prewrite);
1216 			break;
1217 
1218 		case BUS_DMASYNC_POSTREAD | BUS_DMASYNC_POSTWRITE:
1219 			STAT_INCR(sync_coherent_postreadwrite);
1220 			break;
1221 
1222 		case BUS_DMASYNC_POSTREAD:
1223 			STAT_INCR(sync_coherent_postread);
1224 			break;
1225 
1226 		/* BUS_DMASYNC_POSTWRITE was aleady handled as a fastpath */
1227 		}
1228 		/*
1229 		 * Drain the write buffer of DMA operators.
1230 		 * 1) when cpu->device (prewrite)
1231 		 * 2) when device->cpu (postread)
1232 		 */
1233 		if ((pre_ops & BUS_DMASYNC_PREWRITE) || (post_ops & BUS_DMASYNC_POSTREAD))
1234 			cpu_drain_writebuf();
1235 
1236 		/*
1237 		 * Only thing left to do for COHERENT mapping is copy from bounce
1238 		 * in the POSTREAD case.
1239 		 */
1240 		if (bouncing && (post_ops & BUS_DMASYNC_POSTREAD))
1241 			goto bounce_it;
1242 
1243 		return;
1244 	}
1245 
1246 #if !defined(ARM_MMU_EXTENDED)
1247 	/*
1248 	 * If the mapping belongs to a non-kernel vmspace, and the
1249 	 * vmspace has not been active since the last time a full
1250 	 * cache flush was performed, we don't need to do anything.
1251 	 */
1252 	if (__predict_false(!VMSPACE_IS_KERNEL_P(map->_dm_vmspace) &&
1253 	    vm_map_pmap(&map->_dm_vmspace->vm_map)->pm_cstate.cs_cache_d == 0))
1254 		return;
1255 #endif
1256 
1257 	int buftype = map->_dm_buftype;
1258 	if (bouncing) {
1259 		buftype = _BUS_DMA_BUFTYPE_LINEAR;
1260 	}
1261 
1262 	switch (buftype) {
1263 	case _BUS_DMA_BUFTYPE_LINEAR:
1264 	case _BUS_DMA_BUFTYPE_RAW:
1265 		_bus_dmamap_sync_linear(t, map, offset, len, ops);
1266 		break;
1267 
1268 	case _BUS_DMA_BUFTYPE_MBUF:
1269 		_bus_dmamap_sync_mbuf(t, map, offset, len, ops);
1270 		break;
1271 
1272 	case _BUS_DMA_BUFTYPE_UIO:
1273 		_bus_dmamap_sync_uio(t, map, offset, len, ops);
1274 		break;
1275 
1276 	case _BUS_DMA_BUFTYPE_INVALID:
1277 		panic("%s: _BUS_DMA_BUFTYPE_INVALID", __func__);
1278 		break;
1279 
1280 	default:
1281 		panic("%s: map %p: unknown buffer type %d\n", __func__, map,
1282 		    map->_dm_buftype);
1283 	}
1284 
1285 	/* Drain the write buffer. */
1286 	cpu_drain_writebuf();
1287 
1288 	if (!bouncing || (ops & BUS_DMASYNC_POSTREAD) == 0)
1289 		return;
1290 
1291   bounce_it:
1292 	STAT_INCR(read_bounces);
1293 
1294 	struct arm32_bus_dma_cookie * const cookie = map->_dm_cookie;
1295 	char * const dataptr = (char *)cookie->id_bouncebuf + offset;
1296 	/*
1297 	 * Copy the bounce buffer to the caller's buffer.
1298 	 */
1299 	switch (map->_dm_buftype) {
1300 	case _BUS_DMA_BUFTYPE_LINEAR:
1301 		memcpy(cookie->id_origlinearbuf + offset, dataptr, len);
1302 		break;
1303 
1304 	case _BUS_DMA_BUFTYPE_MBUF:
1305 		m_copyback(cookie->id_origmbuf, offset, len, dataptr);
1306 		break;
1307 
1308 	case _BUS_DMA_BUFTYPE_UIO:
1309 		_bus_dma_uiomove(dataptr, cookie->id_origuio, len, UIO_READ);
1310 		break;
1311 
1312 #ifdef DIAGNOSTIC
1313 	case _BUS_DMA_BUFTYPE_RAW:
1314 		panic("%s(post): _BUS_DMA_BUFTYPE_RAW", __func__);
1315 		break;
1316 
1317 	case _BUS_DMA_BUFTYPE_INVALID:
1318 		panic("%s(post): _BUS_DMA_BUFTYPE_INVALID", __func__);
1319 		break;
1320 
1321 	default:
1322 		panic("%s(post): map %p: unknown buffer type %d\n", __func__,
1323 		    map, map->_dm_buftype);
1324 		break;
1325 #endif
1326 	}
1327 }
1328 
1329 /*
1330  * Common function for DMA-safe memory allocation.  May be called
1331  * by bus-specific DMA memory allocation functions.
1332  */
1333 
1334 int
1335 _bus_dmamem_alloc(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
1336     bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
1337     int flags)
1338 {
1339 	struct arm32_dma_range *dr;
1340 	int error, i;
1341 
1342 #ifdef DEBUG_DMA
1343 	printf("dmamem_alloc t=%p size=%#" PRIxBUSSIZE
1344 	    " align=%#" PRIxBUSSIZE
1345 	    " boundary=%#" PRIxBUSSIZE " "
1346 	    "segs=%p nsegs=%#x rsegs=%p flags=%#x\n", t, size, alignment,
1347 	    boundary, segs, nsegs, rsegs, flags);
1348 #endif
1349 
1350 	if ((dr = t->_ranges) != NULL) {
1351 		error = ENOMEM;
1352 		for (i = 0; i < t->_nranges; i++, dr++) {
1353 			if (dr->dr_len == 0
1354 			    || (dr->dr_flags & _BUS_DMAMAP_NOALLOC))
1355 				continue;
1356 			error = _bus_dmamem_alloc_range(t, size, alignment,
1357 			    boundary, segs, nsegs, rsegs, flags,
1358 			    trunc_page(dr->dr_sysbase),
1359 			    trunc_page(dr->dr_sysbase + dr->dr_len));
1360 			if (error == 0)
1361 				break;
1362 		}
1363 	} else {
1364 		error = _bus_dmamem_alloc_range(t, size, alignment, boundary,
1365 		    segs, nsegs, rsegs, flags, 0UL, ~0UL);
1366 	}
1367 
1368 #ifdef DEBUG_DMA
1369 	printf("dmamem_alloc: =%d\n", error);
1370 #endif
1371 
1372 	return error;
1373 }
1374 
1375 /*
1376  * Common function for freeing DMA-safe memory.  May be called by
1377  * bus-specific DMA memory free functions.
1378  */
1379 void
1380 _bus_dmamem_free(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs)
1381 {
1382 	struct vm_page *m;
1383 	bus_addr_t addr;
1384 	struct pglist mlist;
1385 	int curseg;
1386 
1387 #ifdef DEBUG_DMA
1388 	printf("dmamem_free: t=%p segs=%p nsegs=%#x\n", t, segs, nsegs);
1389 #endif	/* DEBUG_DMA */
1390 
1391 	/*
1392 	 * Build a list of pages to free back to the VM system.
1393 	 */
1394 	TAILQ_INIT(&mlist);
1395 	for (curseg = 0; curseg < nsegs; curseg++) {
1396 		for (addr = segs[curseg].ds_addr;
1397 		    addr < (segs[curseg].ds_addr + segs[curseg].ds_len);
1398 		    addr += PAGE_SIZE) {
1399 			m = PHYS_TO_VM_PAGE(addr);
1400 			TAILQ_INSERT_TAIL(&mlist, m, pageq.queue);
1401 		}
1402 	}
1403 	uvm_pglistfree(&mlist);
1404 }
1405 
1406 /*
1407  * Common function for mapping DMA-safe memory.  May be called by
1408  * bus-specific DMA memory map functions.
1409  */
1410 int
1411 _bus_dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs,
1412     size_t size, void **kvap, int flags)
1413 {
1414 	vaddr_t va;
1415 	paddr_t pa;
1416 	int curseg;
1417 	const uvm_flag_t kmflags = UVM_KMF_VAONLY
1418 	    | ((flags & BUS_DMA_NOWAIT) != 0 ? UVM_KMF_NOWAIT : 0);
1419 	vsize_t align = 0;
1420 
1421 #ifdef DEBUG_DMA
1422 	printf("dmamem_map: t=%p segs=%p nsegs=%#x size=%#zx flags=%#x\n", t,
1423 	    segs, nsegs, size, flags);
1424 #endif	/* DEBUG_DMA */
1425 
1426 #ifdef PMAP_MAP_POOLPAGE
1427 	/*
1428 	 * If all of memory is mapped, and we are mapping a single physically
1429 	 * contiguous area then this area is already mapped.  Let's see if we
1430 	 * avoid having a separate mapping for it.
1431 	 */
1432 	if (nsegs == 1 && (flags & BUS_DMA_PREFETCHABLE) == 0) {
1433 		/*
1434 		 * If this is a non-COHERENT mapping, then the existing kernel
1435 		 * mapping is already compatible with it.
1436 		 */
1437 		bool direct_mapable = (flags & BUS_DMA_COHERENT) == 0;
1438 		pa = segs[0].ds_addr;
1439 
1440 		/*
1441 		 * This is a COHERENT mapping which, unless this address is in
1442 		 * a COHERENT dma range, will not be compatible.
1443 		 */
1444 		if (t->_ranges != NULL) {
1445 			const struct arm32_dma_range * const dr =
1446 			    _bus_dma_paddr_inrange(t->_ranges, t->_nranges, pa);
1447 			if (dr != NULL
1448 			    && (dr->dr_flags & _BUS_DMAMAP_COHERENT)) {
1449 				direct_mapable = true;
1450 			}
1451 		}
1452 
1453 #ifdef PMAP_NEED_ALLOC_POOLPAGE
1454 		/*
1455 		 * The page can only be direct mapped if was allocated out
1456 		 * of the arm poolpage vm freelist.
1457 		 */
1458 		uvm_physseg_t upm = uvm_physseg_find(atop(pa), NULL);
1459 		KASSERT(uvm_physseg_valid_p(upm));
1460 		if (direct_mapable) {
1461 			direct_mapable =
1462 			    (arm_poolpage_vmfreelist == uvm_physseg_get_free_list(upm));
1463 		}
1464 #endif
1465 
1466 		if (direct_mapable) {
1467 			*kvap = (void *)PMAP_MAP_POOLPAGE(pa);
1468 #ifdef DEBUG_DMA
1469 			printf("dmamem_map: =%p\n", *kvap);
1470 #endif	/* DEBUG_DMA */
1471 			return 0;
1472 		}
1473 	}
1474 #endif
1475 
1476 	size = round_page(size);
1477 
1478 #ifdef PMAP_MAPSIZE1
1479 	if (size >= PMAP_MAPSIZE1)
1480 		align = PMAP_MAPSIZE1;
1481 
1482 #ifdef PMAP_MAPSIZE2
1483 
1484 #if PMAP_MAPSIZE1 > PMAP_MAPSIZE2
1485 #error PMAP_MAPSIZE1 must be smaller than PMAP_MAPSIZE2
1486 #endif
1487 
1488 	if (size >= PMAP_MAPSIZE2)
1489 		align = PMAP_MAPSIZE2;
1490 
1491 #ifdef PMAP_MAPSIZE3
1492 
1493 #if PMAP_MAPSIZE2 > PMAP_MAPSIZE3
1494 #error PMAP_MAPSIZE2 must be smaller than PMAP_MAPSIZE3
1495 #endif
1496 
1497 	if (size >= PMAP_MAPSIZE3)
1498 		align = PMAP_MAPSIZE3;
1499 #endif
1500 #endif
1501 #endif
1502 
1503 	va = uvm_km_alloc(kernel_map, size, align, kmflags);
1504 	if (__predict_false(va == 0 && align > 0)) {
1505 		align = 0;
1506 		va = uvm_km_alloc(kernel_map, size, 0, kmflags);
1507 	}
1508 
1509 	if (va == 0)
1510 		return ENOMEM;
1511 
1512 	*kvap = (void *)va;
1513 
1514 	for (curseg = 0; curseg < nsegs; curseg++) {
1515 		for (pa = segs[curseg].ds_addr;
1516 		    pa < (segs[curseg].ds_addr + segs[curseg].ds_len);
1517 		    pa += PAGE_SIZE, va += PAGE_SIZE, size -= PAGE_SIZE) {
1518 			bool uncached = (flags & BUS_DMA_COHERENT);
1519 			bool prefetchable = (flags & BUS_DMA_PREFETCHABLE);
1520 #ifdef DEBUG_DMA
1521 			printf("wiring P%#" PRIxPADDR
1522 			    " to V%#" PRIxVADDR "\n", pa, va);
1523 #endif	/* DEBUG_DMA */
1524 			if (size == 0)
1525 				panic("_bus_dmamem_map: size botch");
1526 
1527 			const struct arm32_dma_range * const dr =
1528 			    _bus_dma_paddr_inrange(t->_ranges, t->_nranges, pa);
1529 			/*
1530 			 * If this dma region is coherent then there is
1531 			 * no need for an uncached mapping.
1532 			 */
1533 			if (dr != NULL
1534 			    && (dr->dr_flags & _BUS_DMAMAP_COHERENT)) {
1535 				uncached = false;
1536 			}
1537 
1538 			u_int pmap_flags = PMAP_WIRED;
1539 			if (prefetchable)
1540 				pmap_flags |= PMAP_WRITE_COMBINE;
1541 			else if (uncached)
1542 				pmap_flags |= PMAP_NOCACHE;
1543 
1544 			pmap_kenter_pa(va, pa, VM_PROT_READ | VM_PROT_WRITE,
1545 			    pmap_flags);
1546 		}
1547 	}
1548 	pmap_update(pmap_kernel());
1549 #ifdef DEBUG_DMA
1550 	printf("dmamem_map: =%p\n", *kvap);
1551 #endif	/* DEBUG_DMA */
1552 	return 0;
1553 }
1554 
1555 /*
1556  * Common function for unmapping DMA-safe memory.  May be called by
1557  * bus-specific DMA memory unmapping functions.
1558  */
1559 void
1560 _bus_dmamem_unmap(bus_dma_tag_t t, void *kva, size_t size)
1561 {
1562 
1563 #ifdef DEBUG_DMA
1564 	printf("dmamem_unmap: t=%p kva=%p size=%#zx\n", t, kva, size);
1565 #endif	/* DEBUG_DMA */
1566 	KASSERTMSG(((uintptr_t)kva & PAGE_MASK) == 0,
1567 	    "kva %p (%#"PRIxPTR")", kva, ((uintptr_t)kva & PAGE_MASK));
1568 
1569 #ifdef __HAVE_MM_MD_DIRECT_MAPPED_PHYS
1570 	/*
1571 	 * Check to see if this used direct mapped memory.  Get its physical
1572 	 * address and try to map it.  If the resultant matches the kva, then
1573 	 * it was and so we can just return since we have nothing to free up.
1574 	 */
1575 	paddr_t pa;
1576 	vaddr_t va;
1577 	(void)pmap_extract(pmap_kernel(), (vaddr_t)kva, &pa);
1578 	if (mm_md_direct_mapped_phys(pa, &va) && va == (vaddr_t)kva)
1579 		return;
1580 #endif
1581 
1582 	size = round_page(size);
1583 	pmap_kremove((vaddr_t)kva, size);
1584 	pmap_update(pmap_kernel());
1585 	uvm_km_free(kernel_map, (vaddr_t)kva, size, UVM_KMF_VAONLY);
1586 }
1587 
1588 /*
1589  * Common function for mmap(2)'ing DMA-safe memory.  May be called by
1590  * bus-specific DMA mmap(2)'ing functions.
1591  */
1592 paddr_t
1593 _bus_dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs,
1594     off_t off, int prot, int flags)
1595 {
1596 	paddr_t map_flags;
1597 	int i;
1598 
1599 	for (i = 0; i < nsegs; i++) {
1600 		KASSERTMSG((off & PAGE_MASK) == 0,
1601 		    "off %#jx (%#x)", (uintmax_t)off, (int)off & PAGE_MASK);
1602 		KASSERTMSG((segs[i].ds_addr & PAGE_MASK) == 0,
1603 		    "ds_addr %#lx (%#x)", segs[i].ds_addr,
1604 		    (int)segs[i].ds_addr & PAGE_MASK);
1605 		KASSERTMSG((segs[i].ds_len & PAGE_MASK) == 0,
1606 		    "ds_len %#lx (%#x)", segs[i].ds_addr,
1607 		    (int)segs[i].ds_addr & PAGE_MASK);
1608 		if (off >= segs[i].ds_len) {
1609 			off -= segs[i].ds_len;
1610 			continue;
1611 		}
1612 
1613 		map_flags = 0;
1614 		if (flags & BUS_DMA_PREFETCHABLE)
1615 			map_flags |= ARM_MMAP_WRITECOMBINE;
1616 
1617 		return arm_btop((u_long)segs[i].ds_addr + off) | map_flags;
1618 
1619 	}
1620 
1621 	/* Page not found. */
1622 	return -1;
1623 }
1624 
1625 /**********************************************************************
1626  * DMA utility functions
1627  **********************************************************************/
1628 
1629 /*
1630  * Utility function to load a linear buffer.  lastaddrp holds state
1631  * between invocations (for multiple-buffer loads).  segp contains
1632  * the starting segment on entrance, and the ending segment on exit.
1633  * first indicates if this is the first invocation of this function.
1634  */
1635 int
1636 _bus_dmamap_load_buffer(bus_dma_tag_t t, bus_dmamap_t map, void *buf,
1637     bus_size_t buflen, struct vmspace *vm, int flags)
1638 {
1639 	bus_size_t sgsize;
1640 	bus_addr_t curaddr;
1641 	vaddr_t vaddr = (vaddr_t)buf;
1642 	int error;
1643 
1644 #ifdef DEBUG_DMA
1645 	printf("_bus_dmamap_load_buffer(buf=%p, len=%#" PRIxBUSSIZE
1646 	    ", flags=%#x)\n", buf, buflen, flags);
1647 #endif /* DEBUG_DMA */
1648 
1649 	pmap_t pmap = vm_map_pmap(&vm->vm_map);
1650 
1651 	while (buflen > 0) {
1652 		/*
1653 		 * Get the physical address for this segment.
1654 		 *
1655 		 */
1656 		bool coherent;
1657 		bool ok __diagused;
1658 		ok = pmap_extract_coherency(pmap, vaddr, &curaddr, &coherent);
1659 
1660 		/*
1661 		 * trying to bus_dmamap_load an unmapped buffer is a
1662 		 * programming error.
1663 		 */
1664 		KASSERT(ok);
1665 
1666 		KASSERTMSG((vaddr & PAGE_MASK) == (curaddr & PAGE_MASK),
1667 		    "va %#" PRIxVADDR " curaddr %#" PRIxBUSADDR, vaddr, curaddr);
1668 
1669 		/*
1670 		 * Compute the segment size, and adjust counts.
1671 		 */
1672 		sgsize = PAGE_SIZE - ((u_long)vaddr & PGOFSET);
1673 		if (buflen < sgsize)
1674 			sgsize = buflen;
1675 
1676 		error = _bus_dmamap_load_paddr(t, map, curaddr, sgsize,
1677 		    coherent);
1678 		if (__predict_false(error))
1679 			return error;
1680 
1681 		vaddr += sgsize;
1682 		buflen -= sgsize;
1683 	}
1684 
1685 	return 0;
1686 }
1687 
1688 /*
1689  * Allocate physical memory from the given physical address range.
1690  * Called by DMA-safe memory allocation methods.
1691  */
1692 int
1693 _bus_dmamem_alloc_range(bus_dma_tag_t t, bus_size_t size, bus_size_t alignment,
1694     bus_size_t boundary, bus_dma_segment_t *segs, int nsegs, int *rsegs,
1695     int flags, paddr_t low, paddr_t high)
1696 {
1697 	paddr_t curaddr, lastaddr;
1698 	struct vm_page *m;
1699 	struct pglist mlist;
1700 	int curseg, error;
1701 
1702 	KASSERTMSG(boundary == 0 || (boundary & (boundary - 1)) == 0,
1703 	    "invalid boundary %#" PRIxBUSSIZE, boundary);
1704 
1705 #ifdef DEBUG_DMA
1706 	printf("alloc_range: t=%p size=%#" PRIxBUSSIZE
1707 	    " align=%#" PRIxBUSSIZE " boundary=%#" PRIxBUSSIZE
1708 	    " segs=%p nsegs=%#x rsegs=%p flags=%#x"
1709 	    " lo=%#" PRIxPADDR " hi=%#" PRIxPADDR "\n",
1710 	    t, size, alignment, boundary, segs, nsegs, rsegs, flags, low, high);
1711 #endif	/* DEBUG_DMA */
1712 
1713 	/* Always round the size. */
1714 	size = round_page(size);
1715 
1716 	/*
1717 	 * We accept boundaries < size, splitting in multiple segments
1718 	 * if needed. uvm_pglistalloc does not, so compute an appropriate
1719 	 * boundary: next power of 2 >= size
1720 	 */
1721 	bus_size_t uboundary = boundary;
1722 	if (uboundary <= PAGE_SIZE) {
1723 		uboundary = 0;
1724 	} else {
1725 		while (uboundary < size) {
1726 			uboundary <<= 1;
1727 		}
1728 	}
1729 
1730 	/*
1731 	 * Allocate pages from the VM system.
1732 	 */
1733 	error = uvm_pglistalloc(size, low, high, alignment, uboundary,
1734 	    &mlist, nsegs, (flags & BUS_DMA_NOWAIT) == 0);
1735 	if (error)
1736 		return error;
1737 
1738 	/*
1739 	 * Compute the location, size, and number of segments actually
1740 	 * returned by the VM code.
1741 	 */
1742 	m = TAILQ_FIRST(&mlist);
1743 	curseg = 0;
1744 	lastaddr = segs[curseg].ds_addr = segs[curseg]._ds_paddr =
1745 	    VM_PAGE_TO_PHYS(m);
1746 	segs[curseg].ds_len = PAGE_SIZE;
1747 #ifdef DEBUG_DMA
1748 	printf("alloc: page %#" PRIxPADDR "\n", lastaddr);
1749 #endif	/* DEBUG_DMA */
1750 	m = TAILQ_NEXT(m, pageq.queue);
1751 
1752 	for (; m != NULL; m = TAILQ_NEXT(m, pageq.queue)) {
1753 		curaddr = VM_PAGE_TO_PHYS(m);
1754 		KASSERTMSG(low <= curaddr && curaddr < high,
1755 		    "uvm_pglistalloc returned non-sensicaladdress %#" PRIxPADDR
1756 		    "(low=%#" PRIxPADDR ", high=%#" PRIxPADDR "\n",
1757 		    curaddr, low, high);
1758 #ifdef DEBUG_DMA
1759 		printf("alloc: page %#" PRIxPADDR "\n", curaddr);
1760 #endif	/* DEBUG_DMA */
1761 		if (curaddr == lastaddr + PAGE_SIZE
1762 		    && (lastaddr & boundary) == (curaddr & boundary))
1763 			segs[curseg].ds_len += PAGE_SIZE;
1764 		else {
1765 			curseg++;
1766 			if (curseg >= nsegs) {
1767 				uvm_pglistfree(&mlist);
1768 				return EFBIG;
1769 			}
1770 			segs[curseg].ds_addr = curaddr;
1771 			segs[curseg]._ds_paddr = curaddr;
1772 			segs[curseg].ds_len = PAGE_SIZE;
1773 		}
1774 		lastaddr = curaddr;
1775 	}
1776 
1777 	*rsegs = curseg + 1;
1778 
1779 	return 0;
1780 }
1781 
1782 /*
1783  * Check if a memory region intersects with a DMA range, and return the
1784  * page-rounded intersection if it does.
1785  */
1786 int
1787 arm32_dma_range_intersect(struct arm32_dma_range *ranges, int nranges,
1788     paddr_t pa, psize_t size, paddr_t *pap, psize_t *sizep)
1789 {
1790 	struct arm32_dma_range *dr;
1791 	int i;
1792 
1793 	if (ranges == NULL)
1794 		return 0;
1795 
1796 	for (i = 0, dr = ranges; i < nranges; i++, dr++) {
1797 		if (dr->dr_sysbase <= pa &&
1798 		    pa < (dr->dr_sysbase + dr->dr_len)) {
1799 			/*
1800 			 * Beginning of region intersects with this range.
1801 			 */
1802 			*pap = trunc_page(pa);
1803 			*sizep = round_page(uimin(pa + size,
1804 			    dr->dr_sysbase + dr->dr_len) - pa);
1805 			return 1;
1806 		}
1807 		if (pa < dr->dr_sysbase && dr->dr_sysbase < (pa + size)) {
1808 			/*
1809 			 * End of region intersects with this range.
1810 			 */
1811 			*pap = trunc_page(dr->dr_sysbase);
1812 			*sizep = round_page(uimin((pa + size) - dr->dr_sysbase,
1813 			    dr->dr_len));
1814 			return 1;
1815 		}
1816 	}
1817 
1818 	/* No intersection found. */
1819 	return 0;
1820 }
1821 
1822 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE
1823 static int
1824 _bus_dma_alloc_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map,
1825     bus_size_t size, int flags)
1826 {
1827 	struct arm32_bus_dma_cookie *cookie = map->_dm_cookie;
1828 	int error = 0;
1829 
1830 	KASSERT(cookie != NULL);
1831 
1832 	cookie->id_bouncebuflen = round_page(size);
1833 	error = _bus_dmamem_alloc(t, cookie->id_bouncebuflen,
1834 	    PAGE_SIZE, map->_dm_boundary, cookie->id_bouncesegs,
1835 	    map->_dm_segcnt, &cookie->id_nbouncesegs, flags);
1836 	if (error == 0) {
1837 		error = _bus_dmamem_map(t, cookie->id_bouncesegs,
1838 		    cookie->id_nbouncesegs, cookie->id_bouncebuflen,
1839 		    (void **)&cookie->id_bouncebuf, flags);
1840 		if (error) {
1841 			_bus_dmamem_free(t, cookie->id_bouncesegs,
1842 			    cookie->id_nbouncesegs);
1843 			cookie->id_bouncebuflen = 0;
1844 			cookie->id_nbouncesegs = 0;
1845 		} else {
1846 			cookie->id_flags |= _BUS_DMA_HAS_BOUNCE;
1847 		}
1848 	} else {
1849 		cookie->id_bouncebuflen = 0;
1850 		cookie->id_nbouncesegs = 0;
1851 	}
1852 
1853 	return error;
1854 }
1855 
1856 static void
1857 _bus_dma_free_bouncebuf(bus_dma_tag_t t, bus_dmamap_t map)
1858 {
1859 	struct arm32_bus_dma_cookie *cookie = map->_dm_cookie;
1860 
1861 	KASSERT(cookie != NULL);
1862 
1863 	_bus_dmamem_unmap(t, cookie->id_bouncebuf, cookie->id_bouncebuflen);
1864 	_bus_dmamem_free(t, cookie->id_bouncesegs, cookie->id_nbouncesegs);
1865 	cookie->id_bouncebuflen = 0;
1866 	cookie->id_nbouncesegs = 0;
1867 	cookie->id_flags &= ~_BUS_DMA_HAS_BOUNCE;
1868 }
1869 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */
1870 
1871 /*
1872  * This function does the same as uiomove, but takes an explicit
1873  * direction, and does not update the uio structure.
1874  */
1875 static int
1876 _bus_dma_uiomove(void *buf, struct uio *uio, size_t n, int direction)
1877 {
1878 	struct iovec *iov;
1879 	int error;
1880 	struct vmspace *vm;
1881 	char *cp;
1882 	size_t resid, cnt;
1883 	int i;
1884 
1885 	iov = uio->uio_iov;
1886 	vm = uio->uio_vmspace;
1887 	cp = buf;
1888 	resid = n;
1889 
1890 	for (i = 0; i < uio->uio_iovcnt && resid > 0; i++) {
1891 		iov = &uio->uio_iov[i];
1892 		if (iov->iov_len == 0)
1893 			continue;
1894 		cnt = MIN(resid, iov->iov_len);
1895 
1896 		if (!VMSPACE_IS_KERNEL_P(vm)) {
1897 			preempt_point();
1898 		}
1899 		if (direction == UIO_READ) {
1900 			error = copyout_vmspace(vm, cp, iov->iov_base, cnt);
1901 		} else {
1902 			error = copyin_vmspace(vm, iov->iov_base, cp, cnt);
1903 		}
1904 		if (error)
1905 			return error;
1906 		cp += cnt;
1907 		resid -= cnt;
1908 	}
1909 	return 0;
1910 }
1911 
1912 int
1913 _bus_dmatag_subregion(bus_dma_tag_t tag, bus_addr_t min_addr,
1914     bus_addr_t max_addr, bus_dma_tag_t *newtag, int flags)
1915 {
1916 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE
1917 	if (min_addr >= max_addr)
1918 		return EOPNOTSUPP;
1919 
1920 	struct arm32_dma_range *dr;
1921 	bool psubset = true;
1922 	size_t nranges = 0;
1923 	size_t i;
1924 	for (i = 0, dr = tag->_ranges; i < tag->_nranges; i++, dr++) {
1925 		/*
1926 		 * If the new {min,max}_addr are narrower than any of the
1927 		 * ranges in the parent tag then we need a new tag;
1928 		 * otherwise the parent tag is a subset of the new
1929 		 * range and can continue to be used.
1930 		 */
1931 		if (min_addr > dr->dr_sysbase
1932 		    || max_addr < dr->dr_sysbase + dr->dr_len - 1) {
1933 			psubset = false;
1934 		}
1935 		if (min_addr <= dr->dr_sysbase + dr->dr_len
1936 		    && max_addr >= dr->dr_sysbase) {
1937 			nranges++;
1938 		}
1939 	}
1940 	if (nranges == 0) {
1941 		nranges = 1;
1942 		psubset = false;
1943 	}
1944 	if (psubset) {
1945 		*newtag = tag;
1946 		/* if the tag must be freed, add a reference */
1947 		if (tag->_tag_needs_free)
1948 			(tag->_tag_needs_free)++;
1949 		return 0;
1950 	}
1951 
1952 	const size_t tagsize = sizeof(*tag) + nranges * sizeof(*dr);
1953 	if ((*newtag = kmem_intr_zalloc(tagsize,
1954 	    (flags & BUS_DMA_NOWAIT) ? KM_NOSLEEP : KM_SLEEP)) == NULL)
1955 		return ENOMEM;
1956 
1957 	dr = (void *)(*newtag + 1);
1958 	**newtag = *tag;
1959 	(*newtag)->_tag_needs_free = 1;
1960 	(*newtag)->_ranges = dr;
1961 	(*newtag)->_nranges = nranges;
1962 
1963 	if (tag->_ranges == NULL) {
1964 		dr->dr_sysbase = min_addr;
1965 		dr->dr_busbase = min_addr;
1966 		dr->dr_len = max_addr + 1 - min_addr;
1967 	} else {
1968 		struct arm32_dma_range *pdr;
1969 
1970 		for (i = 0, pdr = tag->_ranges; i < tag->_nranges; i++, pdr++) {
1971 			KASSERT(nranges != 0);
1972 
1973 			if (min_addr > pdr->dr_sysbase + pdr->dr_len
1974 			    || max_addr < pdr->dr_sysbase) {
1975 				/*
1976 				 * this range doesn't overlap with new limits,
1977 				 * so skip.
1978 				 */
1979 				continue;
1980 			}
1981 			/*
1982 			 * Copy the range and adjust to fit within the new
1983 			 * limits
1984 			 */
1985 			dr[0] = pdr[0];
1986 			if (dr->dr_sysbase < min_addr) {
1987 				psize_t diff = min_addr - dr->dr_sysbase;
1988 				dr->dr_busbase += diff;
1989 				dr->dr_len -= diff;
1990 				dr->dr_sysbase += diff;
1991 			}
1992 			if (max_addr <= dr->dr_sysbase + dr->dr_len - 1) {
1993 				dr->dr_len = max_addr + 1 - dr->dr_sysbase;
1994 			}
1995 			dr++;
1996 			nranges--;
1997 		}
1998 	}
1999 
2000 	return 0;
2001 #else
2002 	return EOPNOTSUPP;
2003 #endif /* _ARM32_NEED_BUS_DMA_BOUNCE */
2004 }
2005 
2006 void
2007 _bus_dmatag_destroy(bus_dma_tag_t tag)
2008 {
2009 #ifdef _ARM32_NEED_BUS_DMA_BOUNCE
2010 	switch (tag->_tag_needs_free) {
2011 	case 0:
2012 		break;				/* not allocated with kmem */
2013 	case 1: {
2014 		const size_t tagsize = sizeof(*tag)
2015 		    + tag->_nranges * sizeof(*tag->_ranges);
2016 		kmem_intr_free(tag, tagsize);	/* last reference to tag */
2017 		break;
2018 	}
2019 	default:
2020 		(tag->_tag_needs_free)--;	/* one less reference */
2021 	}
2022 #endif
2023 }
2024