1 /* $NetBSD: iommu.c,v 1.102 2023/12/01 05:22:01 thorpej Exp $ */
2
3 /*
4 * Copyright (c) 1996
5 * The President and Fellows of Harvard College. All rights reserved.
6 * Copyright (c) 1995 Paul Kranenburg
7 *
8 * Redistribution and use in source and binary forms, with or without
9 * modification, are permitted provided that the following conditions
10 * are met:
11 * 1. Redistributions of source code must retain the above copyright
12 * notice, this list of conditions and the following disclaimer.
13 * 2. Redistributions in binary form must reproduce the above copyright
14 * notice, this list of conditions and the following disclaimer in the
15 * documentation and/or other materials provided with the distribution.
16 * 3. All advertising materials mentioning features or use of this software
17 * must display the following acknowledgement:
18 * This product includes software developed by Aaron Brown and
19 * Harvard University.
20 * This product includes software developed by Paul Kranenburg.
21 * 4. Neither the name of the University nor the names of its contributors
22 * may be used to endorse or promote products derived from this software
23 * without specific prior written permission.
24 *
25 * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26 * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27 * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28 * ARE DISCLAIMED. IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29 * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30 * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31 * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32 * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33 * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34 * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35 * SUCH DAMAGE.
36 *
37 */
38
39 #include <sys/cdefs.h>
40 __KERNEL_RCSID(0, "$NetBSD: iommu.c,v 1.102 2023/12/01 05:22:01 thorpej Exp $");
41
42 #include "opt_sparc_arch.h"
43
44 #include <sys/param.h>
45 #include <sys/malloc.h>
46 #include <sys/queue.h>
47 #include <sys/systm.h>
48 #include <sys/device.h>
49 #include <sys/proc.h>
50 #include <sys/vmem.h>
51
52 #include <uvm/uvm.h>
53
54 #define _SPARC_BUS_DMA_PRIVATE
55 #include <sys/bus.h>
56 #include <machine/autoconf.h>
57 #include <machine/ctlreg.h>
58 #include <sparc/sparc/asm.h>
59 #include <sparc/sparc/vaddrs.h>
60 #include <sparc/sparc/cpuvar.h>
61 #include <sparc/sparc/iommureg.h>
62 #include <sparc/sparc/iommuvar.h>
63
64 struct iommu_softc {
65 struct iommureg *sc_reg;
66 u_int sc_pagesize;
67 u_int sc_range;
68 bus_addr_t sc_dvmabase;
69 iopte_t *sc_ptes;
70 int sc_cachecoherent;
71 /*
72 * Note: operations on the extent map are being protected with
73 * splhigh(), since we cannot predict at which interrupt priority
74 * our clients will run.
75 */
76 struct sparc_bus_dma_tag sc_dmatag;
77 vmem_t *sc_dvmamap;
78 };
79
80 /* autoconfiguration driver */
81 int iommu_print(void *, const char *);
82 void iommu_attach(device_t, device_t, void *);
83 int iommu_match(device_t, cfdata_t, void *);
84
85 #if defined(SUN4M)
86 static void iommu_copy_prom_entries(struct iommu_softc *);
87 #endif
88
89 CFATTACH_DECL_NEW(iommu, sizeof(struct iommu_softc),
90 iommu_match, iommu_attach, NULL, NULL);
91
92 /* IOMMU DMA map functions */
93 int iommu_dmamap_create(bus_dma_tag_t, bus_size_t, int, bus_size_t,
94 bus_size_t, int, bus_dmamap_t *);
95 int iommu_dmamap_load(bus_dma_tag_t, bus_dmamap_t, void *,
96 bus_size_t, struct proc *, int);
97 int iommu_dmamap_load_mbuf(bus_dma_tag_t, bus_dmamap_t,
98 struct mbuf *, int);
99 int iommu_dmamap_load_uio(bus_dma_tag_t, bus_dmamap_t,
100 struct uio *, int);
101 int iommu_dmamap_load_raw(bus_dma_tag_t, bus_dmamap_t,
102 bus_dma_segment_t *, int, bus_size_t, int);
103 void iommu_dmamap_unload(bus_dma_tag_t, bus_dmamap_t);
104 void iommu_dmamap_sync(bus_dma_tag_t, bus_dmamap_t, bus_addr_t,
105 bus_size_t, int);
106
107 int iommu_dmamem_map(bus_dma_tag_t, bus_dma_segment_t *,
108 int, size_t, void **, int);
109 void iommu_dmamem_unmap(bus_dma_tag_t, void *, size_t);
110 paddr_t iommu_dmamem_mmap(bus_dma_tag_t, bus_dma_segment_t *,
111 int, off_t, int, int);
112 int iommu_dvma_alloc(struct iommu_softc *, bus_dmamap_t, vaddr_t,
113 bus_size_t, int, bus_addr_t *, bus_size_t *);
114
115 /*
116 * Print the location of some iommu-attached device (called just
117 * before attaching that device). If `iommu' is not NULL, the
118 * device was found but not configured; print the iommu as well.
119 * Return UNCONF (config_find ignores this if the device was configured).
120 */
121 int
iommu_print(void * args,const char * iommu)122 iommu_print(void *args, const char *iommu)
123 {
124 struct iommu_attach_args *ia = args;
125
126 if (iommu)
127 aprint_normal("%s at %s", ia->iom_name, iommu);
128 return (UNCONF);
129 }
130
131 int
iommu_match(device_t parent,cfdata_t cf,void * aux)132 iommu_match(device_t parent, cfdata_t cf, void *aux)
133 {
134 struct mainbus_attach_args *ma = aux;
135
136 if (CPU_ISSUN4 || CPU_ISSUN4C)
137 return (0);
138 return (strcmp(cf->cf_name, ma->ma_name) == 0);
139 }
140
141 /*
142 * Attach the iommu.
143 */
144 void
iommu_attach(device_t parent,device_t self,void * aux)145 iommu_attach(device_t parent, device_t self, void *aux)
146 {
147 #if defined(SUN4M)
148 struct iommu_softc *sc = device_private(self);
149 struct mainbus_attach_args *ma = aux;
150 struct sparc_bus_dma_tag *dmat = &sc->sc_dmatag;
151 bus_space_handle_t bh;
152 int node;
153 int js1_implicit_iommu;
154 int i, s;
155 u_int iopte_table_pa;
156 struct pglist mlist;
157 u_int size;
158 struct vm_page *m;
159 vaddr_t va;
160
161 dmat->_cookie = sc;
162 dmat->_dmamap_create = iommu_dmamap_create;
163 dmat->_dmamap_destroy = _bus_dmamap_destroy;
164 dmat->_dmamap_load = iommu_dmamap_load;
165 dmat->_dmamap_load_mbuf = iommu_dmamap_load_mbuf;
166 dmat->_dmamap_load_uio = iommu_dmamap_load_uio;
167 dmat->_dmamap_load_raw = iommu_dmamap_load_raw;
168 dmat->_dmamap_unload = iommu_dmamap_unload;
169 dmat->_dmamap_sync = iommu_dmamap_sync;
170
171 dmat->_dmamem_alloc = _bus_dmamem_alloc;
172 dmat->_dmamem_free = _bus_dmamem_free;
173 dmat->_dmamem_map = iommu_dmamem_map;
174 dmat->_dmamem_unmap = _bus_dmamem_unmap;
175 dmat->_dmamem_mmap = iommu_dmamem_mmap;
176
177 /*
178 * JS1/OF device tree does not have an iommu node and sbus
179 * node is directly under root. mainbus_attach detects this
180 * and calls us with sbus node instead so that we can attach
181 * implicit iommu and attach that sbus node under it.
182 */
183 node = ma->ma_node;
184 if (strcmp(prom_getpropstring(node, "name"), "sbus") == 0)
185 js1_implicit_iommu = 1;
186 else
187 js1_implicit_iommu = 0;
188
189 /*
190 * Map registers into our space. The PROM may have done this
191 * already, but I feel better if we have our own copy. Plus, the
192 * prom doesn't map the entire register set.
193 *
194 * XXX struct iommureg is bigger than ra->ra_len; what are the
195 * other fields for?
196 */
197 if (bus_space_map(ma->ma_bustag, ma->ma_paddr,
198 sizeof(struct iommureg), 0, &bh) != 0) {
199 printf("iommu_attach: cannot map registers\n");
200 return;
201 }
202 sc->sc_reg = (struct iommureg *)bh;
203
204 sc->sc_cachecoherent = js1_implicit_iommu ? 0
205 : node_has_property(node, "cache-coherence?");
206 if (CACHEINFO.c_enabled == 0) /* XXX - is this correct? */
207 sc->sc_cachecoherent = 0;
208
209 sc->sc_pagesize = js1_implicit_iommu ? PAGE_SIZE
210 : prom_getpropint(node, "page-size", PAGE_SIZE),
211
212 /*
213 * Allocate memory for I/O pagetables.
214 * This takes 64K of contiguous physical memory to map 64M of
215 * DVMA space (starting at IOMMU_DVMA_BASE).
216 * The table must be aligned on a (-IOMMU_DVMA_BASE/pagesize)
217 * boundary (i.e. 64K for 64M of DVMA space).
218 */
219
220 size = ((0 - IOMMU_DVMA_BASE) / sc->sc_pagesize) * sizeof(iopte_t);
221 if (uvm_pglistalloc(size, vm_first_phys, vm_first_phys+vm_num_phys,
222 size, 0, &mlist, 1, 0) != 0)
223 panic("iommu_attach: no memory");
224
225 va = uvm_km_alloc(kernel_map, size, 0, UVM_KMF_VAONLY);
226 if (va == 0)
227 panic("iommu_attach: no memory");
228
229 sc->sc_ptes = (iopte_t *)va;
230
231 m = TAILQ_FIRST(&mlist);
232 iopte_table_pa = VM_PAGE_TO_PHYS(m);
233
234 /* Map the pages */
235 for (; m != NULL; m = TAILQ_NEXT(m,pageq.queue)) {
236 paddr_t pa = VM_PAGE_TO_PHYS(m);
237 pmap_kenter_pa(va, pa | PMAP_NC,
238 VM_PROT_READ | VM_PROT_WRITE, 0);
239 va += PAGE_SIZE;
240 }
241 pmap_update(pmap_kernel());
242
243 /*
244 * Copy entries from current IOMMU table.
245 * XXX - Why do we need to do this?
246 */
247 iommu_copy_prom_entries(sc);
248
249 /*
250 * Now we can install our new pagetable into the IOMMU
251 */
252 sc->sc_range = 0 - IOMMU_DVMA_BASE;
253 sc->sc_dvmabase = IOMMU_DVMA_BASE;
254
255 /* calculate log2(sc->sc_range/16MB) */
256 i = ffs(sc->sc_range/(1 << 24)) - 1;
257 if ((1 << i) != (sc->sc_range/(1 << 24)))
258 panic("iommu: bad range: %d", i);
259
260 s = splhigh();
261 IOMMU_FLUSHALL(sc);
262
263 /* Load range and physical address of PTEs */
264 sc->sc_reg->io_cr = (sc->sc_reg->io_cr & ~IOMMU_CTL_RANGE) |
265 (i << IOMMU_CTL_RANGESHFT) | IOMMU_CTL_ME;
266 sc->sc_reg->io_bar = (iopte_table_pa >> 4) & IOMMU_BAR_IBA;
267
268 IOMMU_FLUSHALL(sc);
269 splx(s);
270
271 printf(": version 0x%x/0x%x, page-size %d, range %dMB\n",
272 (sc->sc_reg->io_cr & IOMMU_CTL_VER) >> 24,
273 (sc->sc_reg->io_cr & IOMMU_CTL_IMPL) >> 28,
274 sc->sc_pagesize,
275 sc->sc_range >> 20);
276
277 sc->sc_dvmamap = vmem_create("iommudvma",
278 IOMMU_DVMA_BASE,
279 IOMMU_DVMA_END - IOMMU_DVMA_BASE,
280 PAGE_SIZE, /* quantum */
281 NULL, /* importfn */
282 NULL, /* releasefn */
283 NULL, /* source */
284 0, /* qcache_max */
285 VM_SLEEP,
286 IPL_VM);
287
288 devhandle_t selfh = device_handle(self);
289
290 /*
291 * If we are attaching implicit iommu on JS1/OF we do not have
292 * an iommu node to traverse, instead mainbus_attach passed us
293 * sbus node in ma.ma_node. Attach it as the only iommu child.
294 */
295 if (js1_implicit_iommu) {
296 struct iommu_attach_args ia;
297 struct openprom_addr sbus_iommu_reg = { 0, 0x10001000, 0x28 };
298
299 memset(&ia, 0, sizeof ia);
300
301 /* Propagate BUS & DMA tags */
302 ia.iom_bustag = ma->ma_bustag;
303 ia.iom_dmatag = &sc->sc_dmatag;
304
305 ia.iom_name = "sbus";
306 ia.iom_node = node;
307 ia.iom_reg = &sbus_iommu_reg;
308 ia.iom_nreg = 1;
309
310 config_found(self, (void *)&ia, iommu_print,
311 CFARGS(.devhandle = prom_node_to_devhandle(selfh, node)));
312 return;
313 }
314
315 /*
316 * Loop through ROM children (expect Sbus among them).
317 */
318 for (node = firstchild(node); node; node = nextsibling(node)) {
319 struct iommu_attach_args ia;
320
321 memset(&ia, 0, sizeof ia);
322 ia.iom_name = prom_getpropstring(node, "name");
323
324 /* Propagate BUS & DMA tags */
325 ia.iom_bustag = ma->ma_bustag;
326 ia.iom_dmatag = &sc->sc_dmatag;
327
328 ia.iom_node = node;
329
330 ia.iom_reg = NULL;
331 prom_getprop(node, "reg", sizeof(struct openprom_addr),
332 &ia.iom_nreg, &ia.iom_reg);
333
334 config_found(self, (void *)&ia, iommu_print,
335 CFARGS(.devhandle = prom_node_to_devhandle(selfh, node)));
336 if (ia.iom_reg != NULL)
337 free(ia.iom_reg, M_DEVBUF);
338 }
339 #endif
340 }
341
342 #if defined(SUN4M)
343 static void
iommu_copy_prom_entries(struct iommu_softc * sc)344 iommu_copy_prom_entries(struct iommu_softc *sc)
345 {
346 u_int pbase, pa;
347 u_int range;
348 iopte_t *tpte_p;
349 u_int pagesz = sc->sc_pagesize;
350 int use_ac = (cpuinfo.cpu_impl == 4 && cpuinfo.mxcc);
351 u_int mmupcr_save;
352
353 /*
354 * We read in the original table using MMU bypass and copy all
355 * of its entries to the appropriate place in our new table,
356 * even if the sizes are different.
357 * This is pretty easy since we know DVMA ends at 0xffffffff.
358 */
359
360 range = (1 << 24) <<
361 ((sc->sc_reg->io_cr & IOMMU_CTL_RANGE) >> IOMMU_CTL_RANGESHFT);
362
363 pbase = (sc->sc_reg->io_bar & IOMMU_BAR_IBA) <<
364 (14 - IOMMU_BAR_IBASHFT);
365
366 if (use_ac) {
367 /*
368 * Set MMU AC bit so we'll still read from the cache
369 * in by-pass mode.
370 */
371 mmupcr_save = lda(SRMMU_PCR, ASI_SRMMU);
372 sta(SRMMU_PCR, ASI_SRMMU, mmupcr_save | VIKING_PCR_AC);
373 } else
374 mmupcr_save = 0; /* XXX - avoid GCC `uninitialized' warning */
375
376 /* Flush entire IOMMU TLB before messing with the in-memory tables */
377 IOMMU_FLUSHALL(sc);
378
379 /*
380 * tpte_p = top of our PTE table
381 * pa = top of current PTE table
382 * Then work downwards and copy entries until we hit the bottom
383 * of either table.
384 */
385 for (tpte_p = &sc->sc_ptes[((0 - IOMMU_DVMA_BASE)/pagesz) - 1],
386 pa = (u_int)pbase + (range/pagesz - 1)*sizeof(iopte_t);
387 tpte_p >= &sc->sc_ptes[0] && pa >= (u_int)pbase;
388 tpte_p--, pa -= sizeof(iopte_t)) {
389
390 *tpte_p = lda(pa, ASI_BYPASS);
391 }
392
393 if (use_ac) {
394 /* restore mmu after bug-avoidance */
395 sta(SRMMU_PCR, ASI_SRMMU, mmupcr_save);
396 }
397 }
398 #endif
399
400 static void
iommu_enter(struct iommu_softc * sc,bus_addr_t dva,paddr_t pa)401 iommu_enter(struct iommu_softc *sc, bus_addr_t dva, paddr_t pa)
402 {
403 int pte;
404
405 /* This routine relies on the fact that sc->sc_pagesize == PAGE_SIZE */
406
407 #ifdef DIAGNOSTIC
408 if (dva < sc->sc_dvmabase)
409 panic("iommu_enter: dva 0x%lx not in DVMA space", (long)dva);
410 #endif
411
412 pte = atop(pa) << IOPTE_PPNSHFT;
413 pte &= IOPTE_PPN;
414 pte |= IOPTE_V | IOPTE_W | (sc->sc_cachecoherent ? IOPTE_C : 0);
415 sc->sc_ptes[atop(dva - sc->sc_dvmabase)] = pte;
416 IOMMU_FLUSHPAGE(sc, dva);
417 }
418
419 /*
420 * iommu_remove: removes mappings created by iommu_enter
421 */
422 static void
iommu_remove(struct iommu_softc * sc,bus_addr_t dva,bus_size_t len)423 iommu_remove(struct iommu_softc *sc, bus_addr_t dva, bus_size_t len)
424 {
425 u_int pagesz = sc->sc_pagesize;
426 bus_addr_t base = sc->sc_dvmabase;
427
428 #ifdef DEBUG
429 if (dva < base)
430 panic("iommu_remove: va 0x%lx not in DVMA space", (long)dva);
431 #endif
432
433 while ((long)len > 0) {
434 #ifdef notyet
435 #ifdef DEBUG
436 if ((sc->sc_ptes[atop(dva - base)] & IOPTE_V) == 0)
437 panic("iommu_remove: clearing invalid pte at dva 0x%lx",
438 (long)dva);
439 #endif
440 #endif
441 sc->sc_ptes[atop(dva - base)] = 0;
442 IOMMU_FLUSHPAGE(sc, dva);
443 len -= pagesz;
444 dva += pagesz;
445 }
446 }
447
448 #if 0 /* These registers aren't there??? */
449 void
450 iommu_error(void)
451 {
452 struct iommu_softc *sc = X;
453 struct iommureg *iop = sc->sc_reg;
454
455 printf("iommu: afsr 0x%x, afar 0x%x\n", iop->io_afsr, iop->io_afar);
456 printf("iommu: mfsr 0x%x, mfar 0x%x\n", iop->io_mfsr, iop->io_mfar);
457 }
458
459 int
460 iommu_alloc(u_int va, u_int len)
461 {
462 struct iommu_softc *sc = X;
463 int off, tva, iovaddr, pte;
464 paddr_t pa;
465
466 off = (int)va & PGOFSET;
467 len = round_page(len + off);
468 va -= off;
469
470 if ((int)sc->sc_dvmacur + len > 0)
471 sc->sc_dvmacur = sc->sc_dvmabase;
472
473 iovaddr = tva = sc->sc_dvmacur;
474 sc->sc_dvmacur += len;
475 while (len) {
476 (void) pmap_extract(pmap_kernel(), va, &pa);
477
478 #define IOMMU_PPNSHIFT 8
479 #define IOMMU_V 0x00000002
480 #define IOMMU_W 0x00000004
481
482 pte = atop(pa) << IOMMU_PPNSHIFT;
483 pte |= IOMMU_V | IOMMU_W;
484 sta(sc->sc_ptes + atop(tva - sc->sc_dvmabase), ASI_BYPASS, pte);
485 sc->sc_reg->io_flushpage = tva;
486 len -= PAGE_SIZE;
487 va += PAGE_SIZE;
488 tva += PAGE_SIZE;
489 }
490 return iovaddr + off;
491 }
492 #endif
493
494
495 /*
496 * IOMMU DMA map functions.
497 */
498 int
iommu_dmamap_create(bus_dma_tag_t t,bus_size_t size,int nsegments,bus_size_t maxsegsz,bus_size_t boundary,int flags,bus_dmamap_t * dmamp)499 iommu_dmamap_create(bus_dma_tag_t t, bus_size_t size, int nsegments,
500 bus_size_t maxsegsz, bus_size_t boundary, int flags,
501 bus_dmamap_t *dmamp)
502 {
503 bus_dmamap_t map;
504 int error;
505
506 if ((error = _bus_dmamap_create(t, size, nsegments, maxsegsz,
507 boundary, flags, &map)) != 0)
508 return (error);
509
510 if ((flags & BUS_DMA_24BIT) != 0) {
511 /* Limit this map to the range usable by `24-bit' devices */
512 map->_dm_ex_start = D24_DVMA_BASE;
513 map->_dm_ex_end = D24_DVMA_END - 1;
514 } else {
515 /* Enable allocations from the entire map */
516 map->_dm_ex_start = VMEM_ADDR_MIN;
517 map->_dm_ex_end = VMEM_ADDR_MAX;
518 }
519
520 *dmamp = map;
521 return (0);
522 }
523
524 /*
525 * Internal routine to allocate space in the IOMMU map.
526 */
527 int
iommu_dvma_alloc(struct iommu_softc * sc,bus_dmamap_t map,vaddr_t va,bus_size_t len,int flags,bus_addr_t * dvap,bus_size_t * sgsizep)528 iommu_dvma_alloc(struct iommu_softc *sc, bus_dmamap_t map,
529 vaddr_t va, bus_size_t len, int flags,
530 bus_addr_t *dvap, bus_size_t *sgsizep)
531 {
532 bus_size_t sgsize;
533 u_long align, voff;
534 vmem_addr_t dvaddr;
535 int error;
536 int pagesz = PAGE_SIZE;
537
538 /*
539 * Remember page offset, then truncate the buffer address to
540 * a page boundary.
541 */
542 voff = va & (pagesz - 1);
543 va &= -pagesz;
544
545 if (len > map->_dm_size)
546 return (EINVAL);
547
548 sgsize = (len + voff + pagesz - 1) & -pagesz;
549 align = dvma_cachealign ? dvma_cachealign : map->_dm_align;
550
551 const vm_flag_t vmflags = VM_BESTFIT |
552 ((flags & BUS_DMA_NOWAIT) ? VM_NOSLEEP : VM_SLEEP);
553
554 error = vmem_xalloc(sc->sc_dvmamap, sgsize,
555 align, /* alignment */
556 va & (align-1), /* phase */
557 map->_dm_boundary, /* nocross */
558 map->_dm_ex_start, /* minaddr */
559 map->_dm_ex_end, /* maxaddr */
560 vmflags,
561 &dvaddr);
562
563 *dvap = (bus_addr_t)dvaddr;
564 *sgsizep = sgsize;
565 return (error);
566 }
567
568 /*
569 * Prepare buffer for DMA transfer.
570 */
571 int
iommu_dmamap_load(bus_dma_tag_t t,bus_dmamap_t map,void * buf,bus_size_t buflen,struct proc * p,int flags)572 iommu_dmamap_load(bus_dma_tag_t t, bus_dmamap_t map,
573 void *buf, bus_size_t buflen,
574 struct proc *p, int flags)
575 {
576 struct iommu_softc *sc = t->_cookie;
577 bus_size_t sgsize;
578 bus_addr_t dva;
579 vaddr_t va = (vaddr_t)buf;
580 int pagesz = PAGE_SIZE;
581 pmap_t pmap;
582 int error;
583
584 /*
585 * Make sure that on error condition we return "no valid mappings".
586 */
587 map->dm_nsegs = 0;
588
589 /* Allocate IOMMU resources */
590 if ((error = iommu_dvma_alloc(sc, map, va, buflen, flags,
591 &dva, &sgsize)) != 0)
592 return (error);
593
594 if ((sc->sc_cachecoherent == 0) ||
595 (curcpu()->cacheinfo.ec_totalsize == 0))
596 cache_flush(buf, buflen); /* XXX - move to bus_dma_sync? */
597
598 /*
599 * We always use just one segment.
600 */
601 map->dm_mapsize = buflen;
602 map->dm_nsegs = 1;
603 map->dm_segs[0].ds_addr = dva + (va & (pagesz - 1));
604 map->dm_segs[0].ds_len = buflen;
605 map->dm_segs[0]._ds_sgsize = sgsize;
606
607 if (p != NULL)
608 pmap = p->p_vmspace->vm_map.pmap;
609 else
610 pmap = pmap_kernel();
611
612 for (; sgsize != 0; ) {
613 paddr_t pa;
614 /*
615 * Get the physical address for this page.
616 */
617 if (!pmap_extract(pmap, va, &pa)) {
618 iommu_dmamap_unload(t, map);
619 return (EFAULT);
620 }
621
622 iommu_enter(sc, dva, pa);
623
624 dva += pagesz;
625 va += pagesz;
626 sgsize -= pagesz;
627 }
628
629 return (0);
630 }
631
632 /*
633 * Like _bus_dmamap_load(), but for mbufs.
634 */
635 int
iommu_dmamap_load_mbuf(bus_dma_tag_t t,bus_dmamap_t map,struct mbuf * m,int flags)636 iommu_dmamap_load_mbuf(bus_dma_tag_t t, bus_dmamap_t map,
637 struct mbuf *m, int flags)
638 {
639
640 panic("_bus_dmamap_load_mbuf: not implemented");
641 }
642
643 /*
644 * Like _bus_dmamap_load(), but for uios.
645 */
646 int
iommu_dmamap_load_uio(bus_dma_tag_t t,bus_dmamap_t map,struct uio * uio,int flags)647 iommu_dmamap_load_uio(bus_dma_tag_t t, bus_dmamap_t map,
648 struct uio *uio, int flags)
649 {
650
651 panic("_bus_dmamap_load_uio: not implemented");
652 }
653
654 /*
655 * Like _bus_dmamap_load(), but for raw memory allocated with
656 * bus_dmamem_alloc().
657 */
658 int
iommu_dmamap_load_raw(bus_dma_tag_t t,bus_dmamap_t map,bus_dma_segment_t * segs,int nsegs,bus_size_t size,int flags)659 iommu_dmamap_load_raw(bus_dma_tag_t t, bus_dmamap_t map,
660 bus_dma_segment_t *segs, int nsegs, bus_size_t size,
661 int flags)
662 {
663 struct iommu_softc *sc = t->_cookie;
664 struct vm_page *m;
665 paddr_t pa;
666 bus_addr_t dva;
667 bus_size_t sgsize;
668 struct pglist *mlist;
669 int pagesz = PAGE_SIZE;
670 int error;
671
672 map->dm_nsegs = 0;
673
674 /* Allocate IOMMU resources */
675 if ((error = iommu_dvma_alloc(sc, map, segs[0]._ds_va, size,
676 flags, &dva, &sgsize)) != 0)
677 return (error);
678
679 /*
680 * Note DVMA address in case bus_dmamem_map() is called later.
681 * It can then insure cache coherency by choosing a KVA that
682 * is aligned to `ds_addr'.
683 */
684 segs[0].ds_addr = dva;
685 segs[0].ds_len = size;
686
687 map->dm_segs[0].ds_addr = dva;
688 map->dm_segs[0].ds_len = size;
689 map->dm_segs[0]._ds_sgsize = sgsize;
690
691 /* Map physical pages into IOMMU */
692 mlist = segs[0]._ds_mlist;
693 for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq.queue)) {
694 if (sgsize == 0)
695 panic("iommu_dmamap_load_raw: size botch");
696 pa = VM_PAGE_TO_PHYS(m);
697 iommu_enter(sc, dva, pa);
698 dva += pagesz;
699 sgsize -= pagesz;
700 }
701
702 map->dm_nsegs = 1;
703 map->dm_mapsize = size;
704
705 return (0);
706 }
707
708 /*
709 * Unload an IOMMU DMA map.
710 */
711 void
iommu_dmamap_unload(bus_dma_tag_t t,bus_dmamap_t map)712 iommu_dmamap_unload(bus_dma_tag_t t, bus_dmamap_t map)
713 {
714 struct iommu_softc *sc = t->_cookie;
715 bus_dma_segment_t *segs = map->dm_segs;
716 int nsegs = map->dm_nsegs;
717 bus_addr_t dva;
718 bus_size_t len;
719 int i;
720
721 for (i = 0; i < nsegs; i++) {
722 dva = segs[i].ds_addr & -PAGE_SIZE;
723 len = segs[i]._ds_sgsize;
724
725 iommu_remove(sc, dva, len);
726 vmem_xfree(sc->sc_dvmamap, dva, len);
727 }
728
729 /* Mark the mappings as invalid. */
730 map->dm_mapsize = 0;
731 map->dm_nsegs = 0;
732 }
733
734 /*
735 * DMA map synchronization.
736 */
737 void
iommu_dmamap_sync(bus_dma_tag_t t,bus_dmamap_t map,bus_addr_t offset,bus_size_t len,int ops)738 iommu_dmamap_sync(bus_dma_tag_t t, bus_dmamap_t map,
739 bus_addr_t offset, bus_size_t len, int ops)
740 {
741
742 /*
743 * XXX Should flush CPU write buffers.
744 */
745 }
746
747 /*
748 * Map DMA-safe memory.
749 */
750 int
iommu_dmamem_map(bus_dma_tag_t t,bus_dma_segment_t * segs,int nsegs,size_t size,void ** kvap,int flags)751 iommu_dmamem_map(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs,
752 size_t size, void **kvap, int flags)
753 {
754 struct iommu_softc *sc = t->_cookie;
755 struct vm_page *m;
756 vaddr_t va;
757 bus_addr_t addr;
758 struct pglist *mlist;
759 int cbit;
760 u_long align;
761 int pagesz = PAGE_SIZE;
762
763 if (nsegs != 1)
764 panic("iommu_dmamem_map: nsegs = %d", nsegs);
765
766 cbit = sc->sc_cachecoherent ? 0 : PMAP_NC;
767 align = dvma_cachealign ? dvma_cachealign : pagesz;
768
769 size = round_page(size);
770
771 /*
772 * In case the segment has already been loaded by
773 * iommu_dmamap_load_raw(), find a region of kernel virtual
774 * addresses that can accommodate our alignment requirements.
775 */
776 va = _bus_dma_valloc_skewed(size, 0, align,
777 segs[0].ds_addr & (align - 1));
778 if (va == 0)
779 return (ENOMEM);
780
781 segs[0]._ds_va = va;
782 *kvap = (void *)va;
783
784 /*
785 * Map the pages allocated in _bus_dmamem_alloc() to the
786 * kernel virtual address space.
787 */
788 mlist = segs[0]._ds_mlist;
789 for (m = TAILQ_FIRST(mlist); m != NULL; m = TAILQ_NEXT(m,pageq.queue)) {
790
791 if (size == 0)
792 panic("iommu_dmamem_map: size botch");
793
794 addr = VM_PAGE_TO_PHYS(m);
795 pmap_kenter_pa(va, addr | cbit,
796 VM_PROT_READ | VM_PROT_WRITE, 0);
797 #if 0
798 if (flags & BUS_DMA_COHERENT)
799 /* XXX */;
800 #endif
801 va += pagesz;
802 size -= pagesz;
803 }
804 pmap_update(pmap_kernel());
805
806 return (0);
807 }
808
809 void
iommu_dmamem_unmap(bus_dma_tag_t t,void * kva,size_t size)810 iommu_dmamem_unmap(bus_dma_tag_t t, void *kva, size_t size)
811 {
812
813 #ifdef DIAGNOSTIC
814 if ((u_long)kva & PAGE_MASK)
815 panic("iommu_dmamem_unmap");
816 #endif
817
818 size = round_page(size);
819 pmap_kremove((vaddr_t)kva, size);
820 pmap_update(pmap_kernel());
821 uvm_unmap(kernel_map, (vaddr_t)kva, (vaddr_t)kva + size);
822 }
823
824
825 /*
826 * mmap(2)'ing DMA-safe memory.
827 */
828 paddr_t
iommu_dmamem_mmap(bus_dma_tag_t t,bus_dma_segment_t * segs,int nsegs,off_t off,int prot,int flags)829 iommu_dmamem_mmap(bus_dma_tag_t t, bus_dma_segment_t *segs, int nsegs,
830 off_t off, int prot, int flags)
831 {
832
833 panic("_bus_dmamem_mmap: not implemented");
834 }
835