1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
23 * Use is subject to license terms.
24 */
25
26 /*
27 * PCI nexus DVMA and DMA core routines:
28 * dma_map/dma_bind_handle implementation
29 * bypass and peer-to-peer support
30 * fast track DVMA space allocation
31 * runtime DVMA debug
32 */
33 #include <sys/types.h>
34 #include <sys/kmem.h>
35 #include <sys/async.h>
36 #include <sys/sysmacros.h>
37 #include <sys/sunddi.h>
38 #include <sys/machsystm.h> /* lddphys() */
39 #include <sys/ddi_impldefs.h>
40 #include <vm/hat.h>
41 #include <sys/pci/pci_obj.h>
42
43 /*LINTLIBRARY*/
44
45 static void
pci_sc_pg_inv(dev_info_t * dip,sc_t * sc_p,ddi_dma_impl_t * mp,off_t off,size_t len)46 pci_sc_pg_inv(dev_info_t *dip, sc_t *sc_p, ddi_dma_impl_t *mp, off_t off,
47 size_t len)
48 {
49 dvma_addr_t dvma_addr, pg_off;
50 volatile uint64_t *invl_va = sc_p->sc_invl_reg;
51
52 if (!len)
53 len = mp->dmai_size;
54
55 pg_off = mp->dmai_offset; /* start min */
56 dvma_addr = MAX(off, pg_off); /* lo */
57 pg_off += mp->dmai_size; /* end max */
58 pg_off = MIN(off + len, pg_off); /* hi */
59 if (dvma_addr >= pg_off) { /* lo >= hi ? */
60 DEBUG4(DBG_SC, dip, "%x+%x out of window [%x,%x)\n",
61 off, len, mp->dmai_offset,
62 mp->dmai_offset + mp->dmai_size);
63 return;
64 }
65
66 len = pg_off - dvma_addr; /* sz = hi - lo */
67 dvma_addr += mp->dmai_mapping; /* start addr */
68 pg_off = dvma_addr & IOMMU_PAGE_OFFSET; /* offset in 1st pg */
69 len = IOMMU_BTOPR(len + pg_off); /* # of pages */
70 dvma_addr ^= pg_off;
71
72 DEBUG2(DBG_SC, dip, "addr=%x+%x pages: \n", dvma_addr, len);
73 for (; len; len--, dvma_addr += IOMMU_PAGE_SIZE) {
74 DEBUG1(DBG_SC|DBG_CONT, dip, " %x", dvma_addr);
75 *invl_va = (uint64_t)dvma_addr;
76 }
77 DEBUG0(DBG_SC|DBG_CONT, dip, "\n");
78 }
79
80 static void
pci_dma_sync_flag_wait(ddi_dma_impl_t * mp,sc_t * sc_p,uint32_t onstack)81 pci_dma_sync_flag_wait(ddi_dma_impl_t *mp, sc_t *sc_p, uint32_t onstack)
82 {
83 hrtime_t start_time;
84 uint64_t loops = 0;
85 uint64_t sync_flag_pa = SYNC_BUF_PA(mp);
86 uint64_t sync_reg_pa = sc_p->sc_sync_reg_pa;
87 uint8_t stack_buf[128];
88
89 stack_buf[0] = DDI_SUCCESS;
90
91 /* check for handle specific sync flag */
92 if (sync_flag_pa)
93 goto start;
94
95 sync_flag_pa = sc_p->sc_sync_flag_pa;
96
97 if (onstack) {
98 sync_flag_pa = va_to_pa(stack_buf);
99 sync_flag_pa += PCI_SYNC_FLAG_SIZE;
100 sync_flag_pa >>= PCI_SYNC_FLAG_SZSHIFT;
101 sync_flag_pa <<= PCI_SYNC_FLAG_SZSHIFT;
102 goto start;
103 }
104 stack_buf[0] |= PCI_SYNC_FLAG_LOCKED;
105 mutex_enter(&sc_p->sc_sync_mutex);
106 start:
107 ASSERT(!(sync_flag_pa & PCI_SYNC_FLAG_SIZE - 1));
108 stdphys(sync_flag_pa, 0); /* reset sync flag to 0 */
109 /* membar #LoadStore|#StoreStore */
110 stdphysio(sync_reg_pa, sync_flag_pa);
111 start_time = gethrtime();
112
113 for (; gethrtime() - start_time < pci_sync_buf_timeout; loops++)
114 if (lddphys(sync_flag_pa))
115 goto done;
116
117 if (!lddphys(sync_flag_pa))
118 stack_buf[0] |= PCI_SYNC_FLAG_FAILED;
119 done:
120 DEBUG3(DBG_SC|DBG_CONT, 0, "flag wait loops=%lu ticks=%lu status=%x\n",
121 loops, gethrtime() - start_time, stack_buf[0]);
122
123 if (stack_buf[0] & PCI_SYNC_FLAG_LOCKED)
124 mutex_exit(&sc_p->sc_sync_mutex);
125
126 if (stack_buf[0] & PCI_SYNC_FLAG_FAILED)
127 cmn_err(CE_PANIC, "%p pci dma sync %lx %lx timeout!",
128 mp, sync_flag_pa, loops);
129 }
130
131 /*
132 * Cache RW Before During After
133 *
134 * STREAMING read no/no pg/no ctx,pg/no
135 * STREAMING write no/no pg/yes ctx,pg/yes
136 * CONSISTENT read no/no yes,no/no yes,no/no
137 * CONSISTENT write no/no yes,yes/yes yes,yes/yes
138 *
139 * STREAMING read ctx,pg/no
140 * STREAMING write ctx,pg/yes
141 * CONSISTENT read yes,no/no
142 * CONSISTENT write yes,yes/yes
143 */
144 int
pci_dma_sync(dev_info_t * dip,dev_info_t * rdip,ddi_dma_handle_t handle,off_t off,size_t len,uint32_t sync_flag)145 pci_dma_sync(dev_info_t *dip, dev_info_t *rdip, ddi_dma_handle_t handle,
146 off_t off, size_t len, uint32_t sync_flag)
147 {
148 ddi_dma_impl_t *mp = (ddi_dma_impl_t *)handle;
149 int ret = ddi_get_instance(dip);
150 pci_t *pci_p = get_pci_soft_state(ret);
151 pbm_t *pbm_p = pci_p->pci_pbm_p;
152 uint32_t dev_flag = mp->dmai_rflags;
153 sc_t *sc_p;
154
155 DEBUG4(DBG_DMA_SYNC, dip, "%s%d flags=%x,%x\n", ddi_driver_name(rdip),
156 ddi_get_instance(rdip), dev_flag, sync_flag);
157 DEBUG4(DBG_SC, dip, "dmai_mapping=%x, dmai_sz=%x off=%x len=%x\n",
158 mp->dmai_mapping, mp->dmai_size, off, len);
159 DEBUG2(DBG_SC, dip, "mp=%p, ctx=%x\n", mp, MP2CTX(mp));
160
161 if (!(mp->dmai_flags & DMAI_FLAGS_INUSE)) {
162 cmn_err(CE_WARN, "Unbound dma handle %p from %s%d", mp,
163 ddi_driver_name(rdip), ddi_get_instance(rdip));
164 return (DDI_FAILURE);
165 }
166
167 if (mp->dmai_flags & DMAI_FLAGS_NOSYNC)
168 return (DDI_SUCCESS);
169
170 if (!(dev_flag & DDI_DMA_CONSISTENT))
171 goto streaming;
172
173 if (sync_flag & PCI_DMA_SYNC_EXT) {
174 if (sync_flag & (PCI_DMA_SYNC_BEFORE | PCI_DMA_SYNC_POST) ||
175 !(sync_flag & PCI_DMA_SYNC_WRITE))
176 return (DDI_SUCCESS);
177 } else {
178 if (!(dev_flag & DDI_DMA_READ) ||
179 ((sync_flag & PCI_DMA_SYNC_DDI_FLAGS) ==
180 DDI_DMA_SYNC_FORDEV))
181 return (DDI_SUCCESS);
182 }
183
184 pci_pbm_dma_sync(pbm_p, pbm_p->pbm_sync_ino);
185 return (DDI_SUCCESS);
186
187 streaming:
188 ASSERT(pci_stream_buf_exists && (pci_stream_buf_enable & 1 << ret));
189 sc_p = pci_p->pci_sc_p;
190 ret = DDI_FAILURE;
191
192 if (sync_flag & PCI_DMA_SYNC_EXT)
193 goto ext;
194
195 if (mp->dmai_flags & DMAI_FLAGS_CONTEXT && pci_sc_use_contexts)
196 ret = pci_sc_ctx_inv(dip, sc_p, mp);
197 if (ret)
198 pci_sc_pg_inv(dip, sc_p, mp, off, len);
199
200 if ((dev_flag & DDI_DMA_READ) &&
201 ((sync_flag & PCI_DMA_SYNC_DDI_FLAGS) != DDI_DMA_SYNC_FORDEV))
202 goto wait;
203
204 return (DDI_SUCCESS);
205 ext:
206 if (sync_flag & PCI_DMA_SYNC_BEFORE)
207 return (DDI_SUCCESS);
208 if (sync_flag & PCI_DMA_SYNC_BAR)
209 goto wait_check;
210 if (sync_flag & PCI_DMA_SYNC_AFTER &&
211 mp->dmai_flags & DMAI_FLAGS_CONTEXT && pci_sc_use_contexts)
212 ret = pci_sc_ctx_inv(dip, sc_p, mp);
213 if (ret)
214 pci_sc_pg_inv(dip, sc_p, mp, off, len);
215 wait_check:
216 if (sync_flag & PCI_DMA_SYNC_POST || !(sync_flag & PCI_DMA_SYNC_WRITE))
217 return (DDI_SUCCESS);
218 wait:
219 pci_dma_sync_flag_wait(mp, sc_p, sync_flag & PCI_DMA_SYNC_PRIVATE);
220 return (DDI_SUCCESS);
221 }
222
223 int
pci_dma_handle_clean(dev_info_t * rdip,ddi_dma_handle_t h)224 pci_dma_handle_clean(dev_info_t *rdip, ddi_dma_handle_t h)
225 {
226 ddi_dma_impl_t *mp = (ddi_dma_impl_t *)h;
227 if ((mp->dmai_flags & DMAI_FLAGS_INUSE) == 0)
228 return (DDI_FAILURE);
229 mp->dmai_rflags |= DMP_NOSYNC;
230 mp->dmai_flags |= DMAI_FLAGS_NOSYNC;
231 return (DDI_SUCCESS);
232 }
233
234 /*
235 * pci_dma_allocmp - Allocate a pci dma implementation structure
236 *
237 * An extra ddi_dma_attr structure is bundled with the usual ddi_dma_impl
238 * to hold unmodified device limits. The ddi_dma_attr inside the
239 * ddi_dma_impl structure is augumented with system limits to enhance
240 * DVMA performance at runtime. The unaugumented device limits saved
241 * right after (accessed through the DEV_ATTR macro) is used
242 * strictly for peer-to-peer transfers which do not obey system limits.
243 *
244 * return: DDI_SUCCESS DDI_DMA_NORESOURCES
245 */
246 ddi_dma_impl_t *
pci_dma_allocmp(dev_info_t * dip,dev_info_t * rdip,int (* waitfp)(caddr_t),caddr_t arg)247 pci_dma_allocmp(dev_info_t *dip, dev_info_t *rdip, int (*waitfp)(caddr_t),
248 caddr_t arg)
249 {
250 ddi_dma_impl_t *mp;
251 int sleep = (waitfp == DDI_DMA_SLEEP) ? KM_SLEEP : KM_NOSLEEP;
252
253 /* Caution: we don't use zalloc to enhance performance! */
254 if ((mp = kmem_alloc(sizeof (pci_dma_hdl_t), sleep)) == 0) {
255 DEBUG0(DBG_DMA_MAP, dip, "can't alloc dma_handle\n");
256 if (waitfp != DDI_DMA_DONTWAIT) {
257 DEBUG0(DBG_DMA_MAP, dip, "alloc_mp kmem cb\n");
258 ddi_set_callback(waitfp, arg, &pci_kmem_clid);
259 }
260 return (mp);
261 }
262
263 mp->dmai_rdip = rdip;
264 mp->dmai_flags = 0;
265 mp->dmai_pfnlst = NULL;
266 mp->dmai_winlst = NULL;
267
268 /*
269 * kmem_alloc debug: the following fields are not zero-ed
270 * mp->dmai_mapping = 0;
271 * mp->dmai_size = 0;
272 * mp->dmai_offset = 0;
273 * mp->dmai_minxfer = 0;
274 * mp->dmai_burstsizes = 0;
275 * mp->dmai_ndvmapages = 0;
276 * mp->dmai_pool/roffset = 0;
277 * mp->dmai_rflags = 0;
278 * mp->dmai_inuse/flags
279 * mp->dmai_nwin = 0;
280 * mp->dmai_winsize = 0;
281 * mp->dmai_nexus_private/tte = 0;
282 * mp->dmai_iopte/pfnlst
283 * mp->dmai_sbi/pfn0 = 0;
284 * mp->dmai_minfo/winlst/fdvma
285 * mp->dmai_rdip
286 * bzero(&mp->dmai_object, sizeof (ddi_dma_obj_t));
287 * mp->dmai_cookie = 0;
288 */
289
290 mp->dmai_attr.dma_attr_version = (uint_t)DMA_ATTR_VERSION;
291 mp->dmai_attr.dma_attr_flags = (uint_t)0;
292 mp->dmai_fault = 0;
293 mp->dmai_fault_check = NULL;
294 mp->dmai_fault_notify = NULL;
295
296 mp->dmai_error.err_ena = 0;
297 mp->dmai_error.err_status = DDI_FM_OK;
298 mp->dmai_error.err_expected = DDI_FM_ERR_UNEXPECTED;
299 mp->dmai_error.err_ontrap = NULL;
300 mp->dmai_error.err_fep = NULL;
301 mp->dmai_error.err_cf = NULL;
302 ndi_fmc_insert(rdip, DMA_HANDLE, mp, NULL);
303
304 SYNC_BUF_PA(mp) = 0ull;
305 return (mp);
306 }
307
308 void
pci_dma_freemp(ddi_dma_impl_t * mp)309 pci_dma_freemp(ddi_dma_impl_t *mp)
310 {
311 ndi_fmc_remove(mp->dmai_rdip, DMA_HANDLE, mp);
312 if (mp->dmai_ndvmapages > 1)
313 pci_dma_freepfn(mp);
314 if (mp->dmai_winlst)
315 pci_dma_freewin(mp);
316 kmem_free(mp, sizeof (pci_dma_hdl_t));
317 }
318
319 void
pci_dma_freepfn(ddi_dma_impl_t * mp)320 pci_dma_freepfn(ddi_dma_impl_t *mp)
321 {
322 void *addr = mp->dmai_pfnlst;
323 ASSERT(!PCI_DMA_CANRELOC(mp));
324 if (addr) {
325 size_t npages = mp->dmai_ndvmapages;
326 if (npages > 1)
327 kmem_free(addr, npages * sizeof (iopfn_t));
328 mp->dmai_pfnlst = NULL;
329 }
330 mp->dmai_ndvmapages = 0;
331 }
332
333 /*
334 * pci_dma_lmts2hdl - alloate a ddi_dma_impl_t, validate practical limits
335 * and convert dmareq->dmar_limits to mp->dmai_attr
336 *
337 * ddi_dma_impl_t member modified input
338 * ------------------------------------------------------------------------
339 * mp->dmai_minxfer - dev
340 * mp->dmai_burstsizes - dev
341 * mp->dmai_flags - no limit? peer-to-peer only?
342 *
343 * ddi_dma_attr member modified input
344 * ------------------------------------------------------------------------
345 * mp->dmai_attr.dma_attr_addr_lo - dev lo, sys lo
346 * mp->dmai_attr.dma_attr_addr_hi - dev hi, sys hi
347 * mp->dmai_attr.dma_attr_count_max - dev count max, dev/sys lo/hi delta
348 * mp->dmai_attr.dma_attr_seg - 0 (no nocross restriction)
349 * mp->dmai_attr.dma_attr_align - 1 (no alignment restriction)
350 *
351 * The dlim_dmaspeed member of dmareq->dmar_limits is ignored.
352 */
353 ddi_dma_impl_t *
pci_dma_lmts2hdl(dev_info_t * dip,dev_info_t * rdip,iommu_t * iommu_p,ddi_dma_req_t * dmareq)354 pci_dma_lmts2hdl(dev_info_t *dip, dev_info_t *rdip, iommu_t *iommu_p,
355 ddi_dma_req_t *dmareq)
356 {
357 ddi_dma_impl_t *mp;
358 ddi_dma_attr_t *attr_p;
359 uint64_t syslo = iommu_p->iommu_dvma_base;
360 uint64_t syshi = iommu_p->iommu_dvma_end;
361 uint64_t fasthi = iommu_p->iommu_dvma_fast_end;
362 ddi_dma_lim_t *lim_p = dmareq->dmar_limits;
363 uint32_t count_max = lim_p->dlim_cntr_max;
364 uint64_t lo = lim_p->dlim_addr_lo;
365 uint64_t hi = lim_p->dlim_addr_hi;
366 if (hi <= lo) {
367 DEBUG0(DBG_DMA_MAP, dip, "Bad limits\n");
368 return ((ddi_dma_impl_t *)DDI_DMA_NOMAPPING);
369 }
370 if (!count_max)
371 count_max--;
372
373 if (!(mp = pci_dma_allocmp(dip, rdip, dmareq->dmar_fp,
374 dmareq->dmar_arg)))
375 return (NULL);
376
377 /* store original dev input at the 2nd ddi_dma_attr */
378 attr_p = DEV_ATTR(mp);
379 SET_DMAATTR(attr_p, lo, hi, -1, count_max);
380 SET_DMAALIGN(attr_p, 1);
381
382 lo = MAX(lo, syslo);
383 hi = MIN(hi, syshi);
384 if (hi <= lo)
385 mp->dmai_flags |= DMAI_FLAGS_PEER_ONLY;
386 count_max = MIN(count_max, hi - lo);
387
388 if (DEV_NOSYSLIMIT(lo, hi, syslo, fasthi, 1))
389 mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT |
390 DMAI_FLAGS_NOSYSLIMIT;
391 else {
392 if (DEV_NOFASTLIMIT(lo, hi, syslo, syshi, 1))
393 mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT;
394 }
395 if (PCI_DMA_NOCTX(rdip))
396 mp->dmai_flags |= DMAI_FLAGS_NOCTX;
397
398 /* store augumented dev input to mp->dmai_attr */
399 mp->dmai_minxfer = lim_p->dlim_minxfer;
400 mp->dmai_burstsizes = lim_p->dlim_burstsizes;
401 attr_p = &mp->dmai_attr;
402 SET_DMAATTR(attr_p, lo, hi, -1, count_max);
403 SET_DMAALIGN(attr_p, 1);
404 return (mp);
405 }
406
407 /*
408 * pci_dma_attr2hdl
409 *
410 * This routine is called from the alloc handle entry point to sanity check the
411 * dma attribute structure.
412 *
413 * use by: pci_dma_allochdl()
414 *
415 * return value:
416 *
417 * DDI_SUCCESS - on success
418 * DDI_DMA_BADATTR - attribute has invalid version number
419 * or address limits exclude dvma space
420 */
421 int
pci_dma_attr2hdl(pci_t * pci_p,ddi_dma_impl_t * mp)422 pci_dma_attr2hdl(pci_t *pci_p, ddi_dma_impl_t *mp)
423 {
424 iommu_t *iommu_p = pci_p->pci_iommu_p;
425 uint64_t syslo, syshi;
426 ddi_dma_attr_t *attrp = DEV_ATTR(mp);
427 uint64_t hi = attrp->dma_attr_addr_hi;
428 uint64_t lo = attrp->dma_attr_addr_lo;
429 uint64_t align = attrp->dma_attr_align;
430 uint64_t nocross = attrp->dma_attr_seg;
431 uint64_t count_max = attrp->dma_attr_count_max;
432
433 DEBUG3(DBG_DMA_ALLOCH, pci_p->pci_dip, "attrp=%p cntr_max=%x.%08x\n",
434 attrp, HI32(count_max), LO32(count_max));
435 DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "hi=%x.%08x lo=%x.%08x\n",
436 HI32(hi), LO32(hi), HI32(lo), LO32(lo));
437 DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "seg=%x.%08x align=%x.%08x\n",
438 HI32(nocross), LO32(nocross), HI32(align), LO32(align));
439
440 if (!nocross)
441 nocross--;
442 if (attrp->dma_attr_flags & DDI_DMA_FORCE_PHYSICAL) { /* BYPASS */
443
444 DEBUG0(DBG_DMA_ALLOCH, pci_p->pci_dip, "bypass mode\n");
445 /* if tomatillo ver <= 2.3 don't allow bypass */
446 if (tomatillo_disallow_bypass)
447 return (DDI_DMA_BADATTR);
448
449 mp->dmai_flags |= DMAI_FLAGS_BYPASSREQ;
450 if (nocross != UINT64_MAX)
451 return (DDI_DMA_BADATTR);
452 if (align && (align > IOMMU_PAGE_SIZE))
453 return (DDI_DMA_BADATTR);
454 align = 1; /* align on 1 page boundary */
455 syslo = iommu_p->iommu_dma_bypass_base;
456 syshi = iommu_p->iommu_dma_bypass_end;
457
458 } else { /* IOMMU_XLATE or PEER_TO_PEER */
459 align = MAX(align, IOMMU_PAGE_SIZE) - 1;
460 if ((align & nocross) != align) {
461 dev_info_t *rdip = mp->dmai_rdip;
462 cmn_err(CE_WARN, "%s%d dma_attr_seg not aligned",
463 NAMEINST(rdip));
464 return (DDI_DMA_BADATTR);
465 }
466 align = IOMMU_BTOP(align + 1);
467 syslo = iommu_p->iommu_dvma_base;
468 syshi = iommu_p->iommu_dvma_end;
469 }
470 if (hi <= lo) {
471 dev_info_t *rdip = mp->dmai_rdip;
472 cmn_err(CE_WARN, "%s%d limits out of range", NAMEINST(rdip));
473 return (DDI_DMA_BADATTR);
474 }
475 lo = MAX(lo, syslo);
476 hi = MIN(hi, syshi);
477 if (!count_max)
478 count_max--;
479
480 DEBUG4(DBG_DMA_ALLOCH, pci_p->pci_dip, "hi=%x.%08x, lo=%x.%08x\n",
481 HI32(hi), LO32(hi), HI32(lo), LO32(lo));
482 if (hi <= lo) { /* peer transfers cannot have alignment & nocross */
483 dev_info_t *rdip = mp->dmai_rdip;
484 cmn_err(CE_WARN, "%s%d peer only dev %p", NAMEINST(rdip), mp);
485 if ((nocross < UINT32_MAX) || (align > 1)) {
486 cmn_err(CE_WARN, "%s%d peer only device bad attr",
487 NAMEINST(rdip));
488 return (DDI_DMA_BADATTR);
489 }
490 mp->dmai_flags |= DMAI_FLAGS_PEER_ONLY;
491 } else /* set practical counter_max value */
492 count_max = MIN(count_max, hi - lo);
493
494 if (DEV_NOSYSLIMIT(lo, hi, syslo, syshi, align))
495 mp->dmai_flags |= DMAI_FLAGS_NOSYSLIMIT |
496 DMAI_FLAGS_NOFASTLIMIT;
497 else {
498 syshi = iommu_p->iommu_dvma_fast_end;
499 if (DEV_NOFASTLIMIT(lo, hi, syslo, syshi, align))
500 mp->dmai_flags |= DMAI_FLAGS_NOFASTLIMIT;
501 }
502 if (PCI_DMA_NOCTX(mp->dmai_rdip))
503 mp->dmai_flags |= DMAI_FLAGS_NOCTX;
504
505 mp->dmai_minxfer = attrp->dma_attr_minxfer;
506 mp->dmai_burstsizes = attrp->dma_attr_burstsizes;
507 attrp = &mp->dmai_attr;
508 SET_DMAATTR(attrp, lo, hi, nocross, count_max);
509 return (DDI_SUCCESS);
510 }
511
512 /*
513 * set up consistent dma flags according to hardware capability
514 */
515 uint32_t
pci_dma_consist_check(uint32_t req_flags,pbm_t * pbm_p)516 pci_dma_consist_check(uint32_t req_flags, pbm_t *pbm_p)
517 {
518 if (!pci_stream_buf_enable || !pci_stream_buf_exists)
519 req_flags |= DDI_DMA_CONSISTENT;
520 if (req_flags & DDI_DMA_CONSISTENT && !pbm_p->pbm_sync_reg_pa)
521 req_flags |= DMP_NOSYNC;
522 return (req_flags);
523 }
524
525 #define TGT_PFN_INBETWEEN(pfn, bgn, end) ((pfn >= bgn) && (pfn <= end))
526
527 /*
528 * pci_dma_type - determine which of the three types DMA (peer-to-peer,
529 * iommu bypass, or iommu translate) we are asked to do.
530 * Also checks pfn0 and rejects any non-peer-to-peer
531 * requests for peer-only devices.
532 *
533 * return values:
534 * DDI_DMA_NOMAPPING - can't get valid pfn0, or bad dma type
535 * DDI_SUCCESS
536 *
537 * dma handle members affected (set on exit):
538 * mp->dmai_object - dmareq->dmar_object
539 * mp->dmai_rflags - consistent?, nosync?, dmareq->dmar_flags
540 * mp->dmai_flags - DMA type
541 * mp->dmai_pfn0 - 1st page pfn (if va/size pair and not shadow)
542 * mp->dmai_roffset - initialized to starting IOMMU page offset
543 * mp->dmai_ndvmapages - # of total IOMMU pages of entire object
544 * mp->pdh_sync_buf_pa - dma sync buffer PA is DMA flow is supported
545 */
546 int
pci_dma_type(pci_t * pci_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)547 pci_dma_type(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
548 {
549 dev_info_t *dip = pci_p->pci_dip;
550 ddi_dma_obj_t *dobj_p = &dmareq->dmar_object;
551 pbm_t *pbm_p = pci_p->pci_pbm_p;
552 page_t **pplist;
553 struct as *as_p;
554 uint32_t offset;
555 caddr_t vaddr;
556 pfn_t pfn0;
557
558 mp->dmai_rflags = pci_dma_consist_check(dmareq->dmar_flags, pbm_p);
559 mp->dmai_flags |= mp->dmai_rflags & DMP_NOSYNC ? DMAI_FLAGS_NOSYNC : 0;
560
561 switch (dobj_p->dmao_type) {
562 case DMA_OTYP_BUFVADDR:
563 case DMA_OTYP_VADDR: {
564 vaddr = dobj_p->dmao_obj.virt_obj.v_addr;
565 pplist = dobj_p->dmao_obj.virt_obj.v_priv;
566 as_p = dobj_p->dmao_obj.virt_obj.v_as;
567 if (as_p == NULL)
568 as_p = &kas;
569
570 DEBUG2(DBG_DMA_MAP, dip, "vaddr=%p pplist=%p\n", vaddr, pplist);
571 offset = (ulong_t)vaddr & IOMMU_PAGE_OFFSET;
572
573 if (pplist) { /* shadow list */
574 mp->dmai_flags |= DMAI_FLAGS_PGPFN;
575 ASSERT(PAGE_LOCKED(*pplist));
576 pfn0 = page_pptonum(*pplist);
577 } else if (pci_dvma_remap_enabled && as_p == &kas &&
578 dobj_p->dmao_type != DMA_OTYP_BUFVADDR) {
579 int (*waitfp)(caddr_t) = dmareq->dmar_fp;
580 uint_t flags = ((waitfp == DDI_DMA_SLEEP)?
581 HAC_SLEEP : HAC_NOSLEEP) | HAC_PAGELOCK;
582 int ret;
583
584 ret = hat_add_callback(pci_dvma_cbid, vaddr,
585 IOMMU_PAGE_SIZE - offset, flags, mp, &pfn0,
586 MP_HAT_CB_COOKIE_PTR(mp, 0));
587
588 if (pfn0 == PFN_INVALID && ret == ENOMEM) {
589 ASSERT(waitfp != DDI_DMA_SLEEP);
590 if (waitfp != DDI_DMA_DONTWAIT) {
591 ddi_set_callback(waitfp,
592 dmareq->dmar_arg,
593 &pci_kmem_clid);
594 return (DDI_DMA_NORESOURCES);
595 }
596 }
597 mp->dmai_flags |= DMAI_FLAGS_RELOC;
598 } else
599 pfn0 = hat_getpfnum(as_p->a_hat, vaddr);
600 }
601 break;
602
603 case DMA_OTYP_PAGES:
604 offset = dobj_p->dmao_obj.pp_obj.pp_offset;
605 mp->dmai_flags |= DMAI_FLAGS_PGPFN;
606 pfn0 = page_pptonum(dobj_p->dmao_obj.pp_obj.pp_pp);
607 ASSERT(PAGE_LOCKED(dobj_p->dmao_obj.pp_obj.pp_pp));
608 break;
609
610 case DMA_OTYP_PADDR:
611 default:
612 cmn_err(CE_WARN, "%s%d requested unsupported dma type %x",
613 NAMEINST(mp->dmai_rdip), dobj_p->dmao_type);
614 return (DDI_DMA_NOMAPPING);
615 }
616 if (pfn0 == PFN_INVALID) {
617 cmn_err(CE_WARN, "%s%d: invalid pfn0 for DMA object %p",
618 NAMEINST(dip), dobj_p);
619 return (DDI_DMA_NOMAPPING);
620 }
621 if (TGT_PFN_INBETWEEN(pfn0, pbm_p->pbm_base_pfn, pbm_p->pbm_last_pfn)) {
622 mp->dmai_flags |= DMAI_FLAGS_PEER_TO_PEER;
623 goto done; /* leave bypass and dvma flag as 0 */
624 }
625 if (PCI_DMA_ISPEERONLY(mp)) {
626 dev_info_t *rdip = mp->dmai_rdip;
627 cmn_err(CE_WARN, "Bad peer-to-peer req %s%d", NAMEINST(rdip));
628 return (DDI_DMA_NOMAPPING);
629 }
630 mp->dmai_flags |= (mp->dmai_flags & DMAI_FLAGS_BYPASSREQ) ?
631 DMAI_FLAGS_BYPASS : DMAI_FLAGS_DVMA;
632 done:
633 mp->dmai_object = *dobj_p; /* whole object */
634 mp->dmai_pfn0 = (void *)pfn0; /* cache pfn0 */
635 mp->dmai_roffset = offset; /* win0 pg0 offset */
636 mp->dmai_ndvmapages = IOMMU_BTOPR(offset + mp->dmai_object.dmao_size);
637
638 return (DDI_SUCCESS);
639 }
640
641 /*
642 * pci_dma_pgpfn - set up pfnlst array according to pages
643 * VA/size pair: <shadow IO, bypass, peer-to-peer>, or OTYP_PAGES
644 */
645 /*ARGSUSED*/
646 static int
pci_dma_pgpfn(pci_t * pci_p,ddi_dma_impl_t * mp,uint_t npages)647 pci_dma_pgpfn(pci_t *pci_p, ddi_dma_impl_t *mp, uint_t npages)
648 {
649 int i;
650 #ifdef DEBUG
651 dev_info_t *dip = pci_p->pci_dip;
652 #endif
653 switch (mp->dmai_object.dmao_type) {
654 case DMA_OTYP_BUFVADDR:
655 case DMA_OTYP_VADDR: {
656 page_t **pplist = mp->dmai_object.dmao_obj.virt_obj.v_priv;
657 DEBUG2(DBG_DMA_MAP, dip, "shadow pplist=%p, %x pages, pfns=",
658 pplist, npages);
659 for (i = 1; i < npages; i++) {
660 iopfn_t pfn = page_pptonum(pplist[i]);
661 ASSERT(PAGE_LOCKED(pplist[i]));
662 PCI_SET_MP_PFN1(mp, i, pfn);
663 DEBUG1(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn);
664 }
665 DEBUG0(DBG_DMA_MAP|DBG_CONT, dip, "\n");
666 }
667 break;
668
669 case DMA_OTYP_PAGES: {
670 page_t *pp = mp->dmai_object.dmao_obj.pp_obj.pp_pp->p_next;
671 DEBUG1(DBG_DMA_MAP, dip, "pp=%p pfns=", pp);
672 for (i = 1; i < npages; i++, pp = pp->p_next) {
673 iopfn_t pfn = page_pptonum(pp);
674 ASSERT(PAGE_LOCKED(pp));
675 PCI_SET_MP_PFN1(mp, i, pfn);
676 DEBUG1(DBG_DMA_MAP|DBG_CONT, dip, "%x ", pfn);
677 }
678 DEBUG0(DBG_DMA_MAP|DBG_CONT, dip, "\n");
679 }
680 break;
681
682 default: /* check is already done by pci_dma_type */
683 ASSERT(0);
684 break;
685 }
686 return (DDI_SUCCESS);
687 }
688
689 /*
690 * pci_dma_vapfn - set up pfnlst array according to VA
691 * VA/size pair: <normal, bypass, peer-to-peer>
692 * pfn0 is skipped as it is already done.
693 * In this case, the cached pfn0 is used to fill pfnlst[0]
694 */
695 static int
pci_dma_vapfn(pci_t * pci_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp,uint_t npages)696 pci_dma_vapfn(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp,
697 uint_t npages)
698 {
699 dev_info_t *dip = pci_p->pci_dip;
700 int i;
701 caddr_t vaddr = (caddr_t)mp->dmai_object.dmao_obj.virt_obj.v_as;
702 struct hat *hat_p = vaddr ? ((struct as *)vaddr)->a_hat : kas.a_hat;
703 caddr_t sva;
704 int needcb = 0;
705
706 sva = (caddr_t)(((uintptr_t)mp->dmai_object.dmao_obj.virt_obj.v_addr +
707 IOMMU_PAGE_SIZE) & IOMMU_PAGE_MASK);
708
709 if (pci_dvma_remap_enabled && hat_p == kas.a_hat &&
710 mp->dmai_object.dmao_type != DMA_OTYP_BUFVADDR)
711 needcb = 1;
712
713 for (vaddr = sva, i = 1; i < npages; i++, vaddr += IOMMU_PAGE_SIZE) {
714 pfn_t pfn;
715
716 if (needcb) {
717 int (*waitfp)(caddr_t) = dmareq->dmar_fp;
718 uint_t flags = ((waitfp == DDI_DMA_SLEEP)?
719 HAC_SLEEP : HAC_NOSLEEP) | HAC_PAGELOCK;
720 int ret;
721
722 ret = hat_add_callback(pci_dvma_cbid, vaddr,
723 IOMMU_PAGE_SIZE, flags, mp, &pfn,
724 MP_HAT_CB_COOKIE_PTR(mp, i));
725
726 if (pfn == PFN_INVALID && ret == ENOMEM) {
727 ASSERT(waitfp != DDI_DMA_SLEEP);
728 if (waitfp != DDI_DMA_DONTWAIT)
729 ddi_set_callback(waitfp,
730 dmareq->dmar_arg, &pci_kmem_clid);
731 return (DDI_DMA_NORESOURCES);
732 }
733 } else
734 pfn = hat_getpfnum(hat_p, vaddr);
735 if (pfn == PFN_INVALID)
736 goto err_badpfn;
737 PCI_SET_MP_PFN1(mp, i, (iopfn_t)pfn);
738 DEBUG3(DBG_DMA_MAP, dip, "pci_dma_vapfn: mp=%p pfnlst[%x]=%x\n",
739 mp, i, (iopfn_t)pfn);
740 }
741 return (DDI_SUCCESS);
742 err_badpfn:
743 cmn_err(CE_WARN, "%s%d: bad page frame vaddr=%p", NAMEINST(dip), vaddr);
744 return (DDI_DMA_NOMAPPING);
745 }
746
747 /*
748 * pci_dma_pfn - Fills pfn list for all pages being DMA-ed.
749 *
750 * dependencies:
751 * mp->dmai_ndvmapages - set to total # of dma pages
752 *
753 * return value:
754 * DDI_SUCCESS
755 * DDI_DMA_NOMAPPING
756 */
757 int
pci_dma_pfn(pci_t * pci_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)758 pci_dma_pfn(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
759 {
760 uint32_t npages = mp->dmai_ndvmapages;
761 int (*waitfp)(caddr_t) = dmareq->dmar_fp;
762 int i, ret, peer = PCI_DMA_ISPTP(mp);
763
764 pbm_t *pbm_p = pci_p->pci_pbm_p;
765 iopfn_t pfn_base = pbm_p->pbm_base_pfn;
766 iopfn_t pfn_last = pbm_p->pbm_last_pfn;
767 iopfn_t pfn_adj = peer ? pfn_base : 0;
768
769 DEBUG2(DBG_DMA_MAP, pci_p->pci_dip, "pci_dma_pfn: mp=%p pfn0=%x\n",
770 mp, MP_PFN0(mp) - pfn_adj);
771 /* 1 page: no array alloc/fill, no mixed mode check */
772 if (npages == 1) {
773 PCI_SET_MP_PFN(mp, 0, MP_PFN0(mp) - pfn_adj);
774 return (DDI_SUCCESS);
775 }
776 /* allocate pfn array */
777 if (!(mp->dmai_pfnlst = kmem_alloc(npages * sizeof (iopfn_t),
778 waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP))) {
779 if (waitfp != DDI_DMA_DONTWAIT)
780 ddi_set_callback(waitfp, dmareq->dmar_arg,
781 &pci_kmem_clid);
782 return (DDI_DMA_NORESOURCES);
783 }
784 /* fill pfn array */
785 PCI_SET_MP_PFN(mp, 0, MP_PFN0(mp) - pfn_adj); /* pfnlst[0] */
786 if ((ret = PCI_DMA_ISPGPFN(mp) ? pci_dma_pgpfn(pci_p, mp, npages) :
787 pci_dma_vapfn(pci_p, dmareq, mp, npages)) != DDI_SUCCESS)
788 goto err;
789
790 /* skip pfn0, check mixed mode and adjust peer to peer pfn */
791 for (i = 1; i < npages; i++) {
792 iopfn_t pfn = PCI_GET_MP_PFN1(mp, i);
793 if (peer ^ TGT_PFN_INBETWEEN(pfn, pfn_base, pfn_last)) {
794 cmn_err(CE_WARN, "%s%d mixed mode DMA %lx %lx",
795 NAMEINST(mp->dmai_rdip), MP_PFN0(mp), pfn);
796 ret = DDI_DMA_NOMAPPING; /* mixed mode */
797 goto err;
798 }
799 DEBUG3(DBG_DMA_MAP, pci_p->pci_dip,
800 "pci_dma_pfn: pfnlst[%x]=%x-%x\n", i, pfn, pfn_adj);
801 if (pfn_adj)
802 PCI_SET_MP_PFN1(mp, i, pfn - pfn_adj);
803 }
804 return (DDI_SUCCESS);
805 err:
806 pci_dvma_unregister_callbacks(pci_p, mp);
807 pci_dma_freepfn(mp);
808 return (ret);
809 }
810
811 /*
812 * pci_dvma_win() - trim requested DVMA size down to window size
813 * The 1st window starts from offset and ends at page-aligned boundary.
814 * From the 2nd window on, each window starts and ends at page-aligned
815 * boundary except the last window ends at wherever requested.
816 *
817 * accesses the following mp-> members:
818 * mp->dmai_attr.dma_attr_count_max
819 * mp->dmai_attr.dma_attr_seg
820 * mp->dmai_roffset - start offset of 1st window
821 * mp->dmai_rflags (redzone)
822 * mp->dmai_ndvmapages (for 1 page fast path)
823 *
824 * sets the following mp-> members:
825 * mp->dmai_size - xfer size, != winsize if 1st/last win (not fixed)
826 * mp->dmai_winsize - window size (no redzone), n * page size (fixed)
827 * mp->dmai_nwin - # of DMA windows of entire object (fixed)
828 * mp->dmai_rflags - remove partial flag if nwin == 1 (fixed)
829 * mp->dmai_winlst - NULL, window objects not used for DVMA (fixed)
830 *
831 * fixed - not changed across different DMA windows
832 */
833 /*ARGSUSED*/
834 int
pci_dvma_win(pci_t * pci_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)835 pci_dvma_win(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
836 {
837 uint32_t redzone_sz = HAS_REDZONE(mp) ? IOMMU_PAGE_SIZE : 0;
838 size_t obj_sz = mp->dmai_object.dmao_size;
839 size_t xfer_sz;
840 ulong_t pg_off;
841
842 if ((mp->dmai_ndvmapages == 1) && !redzone_sz) {
843 mp->dmai_rflags &= ~DDI_DMA_PARTIAL;
844 mp->dmai_size = obj_sz;
845 mp->dmai_winsize = IOMMU_PAGE_SIZE;
846 mp->dmai_nwin = 1;
847 goto done;
848 }
849
850 pg_off = mp->dmai_roffset;
851 xfer_sz = obj_sz + redzone_sz;
852
853 /* include redzone in nocross check */
854 {
855 uint64_t nocross = mp->dmai_attr.dma_attr_seg;
856 if (xfer_sz + pg_off - 1 > nocross)
857 xfer_sz = nocross - pg_off + 1;
858 if (redzone_sz && (xfer_sz <= redzone_sz)) {
859 DEBUG5(DBG_DMA_MAP, pci_p->pci_dip,
860 "nocross too small %lx(%lx)+%lx+%x < %" PRIx64 "\n",
861 xfer_sz, obj_sz, pg_off, redzone_sz, nocross);
862 return (DDI_DMA_TOOBIG);
863 }
864 }
865 xfer_sz -= redzone_sz; /* restore transfer size */
866 /* check counter max */
867 {
868 uint32_t count_max = mp->dmai_attr.dma_attr_count_max;
869 if (xfer_sz - 1 > count_max)
870 xfer_sz = count_max + 1;
871 }
872 if (xfer_sz >= obj_sz) {
873 mp->dmai_rflags &= ~DDI_DMA_PARTIAL;
874 mp->dmai_size = xfer_sz;
875 mp->dmai_winsize = P2ROUNDUP(xfer_sz + pg_off, IOMMU_PAGE_SIZE);
876 mp->dmai_nwin = 1;
877 goto done;
878 }
879 if (!(dmareq->dmar_flags & DDI_DMA_PARTIAL)) {
880 DEBUG4(DBG_DMA_MAP, pci_p->pci_dip,
881 "too big: %lx+%lx+%x > %lx\n",
882 obj_sz, pg_off, redzone_sz, xfer_sz);
883 return (DDI_DMA_TOOBIG);
884 }
885
886 xfer_sz = IOMMU_PTOB(IOMMU_BTOP(xfer_sz + pg_off)); /* page align */
887 mp->dmai_size = xfer_sz - pg_off; /* 1st window xferrable size */
888 mp->dmai_winsize = xfer_sz; /* redzone not in winsize */
889 mp->dmai_nwin = (obj_sz + pg_off + xfer_sz - 1) / xfer_sz;
890 done:
891 mp->dmai_winlst = NULL;
892 dump_dma_handle(DBG_DMA_MAP, pci_p->pci_dip, mp);
893 return (DDI_SUCCESS);
894 }
895
896 /*
897 * fast track cache entry to iommu context, inserts 3 0 bits between
898 * upper 6-bits and lower 3-bits of the 9-bit cache entry
899 */
900 #define IOMMU_FCE_TO_CTX(i) (((i) << 3) | ((i) & 0x7) | 0x38)
901
902 /*
903 * pci_dvma_map_fast - attempts to map fast trackable DVMA
904 */
905 int
pci_dvma_map_fast(iommu_t * iommu_p,ddi_dma_impl_t * mp)906 pci_dvma_map_fast(iommu_t *iommu_p, ddi_dma_impl_t *mp)
907 {
908 uint_t clustsz = pci_dvma_page_cache_clustsz;
909 uint_t entries = pci_dvma_page_cache_entries;
910 uint64_t *tte_addr;
911 uint64_t tte = GET_TTE_TEMPLATE(mp);
912 int i = iommu_p->iommu_dvma_addr_scan_start;
913 uint8_t *lock_addr = iommu_p->iommu_dvma_cache_locks + i;
914 iopfn_t *pfn_addr;
915 dvma_addr_t dvma_pg;
916 size_t npages = IOMMU_BTOP(mp->dmai_winsize);
917 #ifdef DEBUG
918 dev_info_t *dip = mp->dmai_rdip;
919 #endif
920 extern uint8_t ldstub(uint8_t *);
921 ASSERT(IOMMU_PTOB(npages) == mp->dmai_winsize);
922 ASSERT(npages + HAS_REDZONE(mp) <= clustsz);
923
924 for (; i < entries && ldstub(lock_addr); i++, lock_addr++)
925 ;
926 if (i >= entries) {
927 lock_addr = iommu_p->iommu_dvma_cache_locks;
928 i = 0;
929 for (; i < entries && ldstub(lock_addr); i++, lock_addr++)
930 ;
931 if (i >= entries) {
932 #ifdef PCI_DMA_PROF
933 pci_dvmaft_exhaust++;
934 #endif
935 return (DDI_DMA_NORESOURCES);
936 }
937 }
938 iommu_p->iommu_dvma_addr_scan_start = (i + 1) & (entries - 1);
939 if (PCI_DMA_USECTX(mp)) {
940 dvma_context_t ctx = IOMMU_FCE_TO_CTX(i);
941 tte |= IOMMU_CTX2TTE(ctx);
942 mp->dmai_flags |= DMAI_FLAGS_CONTEXT;
943 DEBUG1(DBG_DMA_MAP, dip, "fast: ctx=0x%x\n", ctx);
944 }
945 i *= clustsz;
946 tte_addr = iommu_p->iommu_tsb_vaddr + i;
947 dvma_pg = iommu_p->dvma_base_pg + i;
948 #ifdef DEBUG
949 for (i = 0; i < clustsz; i++)
950 ASSERT(TTE_IS_INVALID(tte_addr[i]));
951 #endif
952 *tte_addr = tte | IOMMU_PTOB(MP_PFN0(mp)); /* map page 0 */
953 DEBUG5(DBG_DMA_MAP, dip, "fast %p:dvma_pg=%x tte0(%p)=%08x.%08x\n", mp,
954 dvma_pg, tte_addr, HI32(*tte_addr), LO32(*tte_addr));
955 if (npages == 1)
956 goto tte_done;
957 pfn_addr = PCI_GET_MP_PFN1_ADDR(mp); /* short iommu_map_pages() */
958 for (tte_addr++, i = 1; i < npages; i++, tte_addr++, pfn_addr++) {
959 *tte_addr = tte | IOMMU_PTOB(*pfn_addr);
960 DEBUG5(DBG_DMA_MAP, dip, "fast %p:tte(%p, %p)=%08x.%08x\n", mp,
961 tte_addr, pfn_addr, HI32(*tte_addr), LO32(*tte_addr));
962 }
963 tte_done:
964 #ifdef PCI_DMA_PROF
965 pci_dvmaft_success++;
966 #endif
967 mp->dmai_mapping = mp->dmai_roffset | IOMMU_PTOB(dvma_pg);
968 mp->dmai_offset = 0;
969 mp->dmai_flags |= DMAI_FLAGS_FASTTRACK;
970 PCI_SAVE_MP_TTE(mp, tte); /* save TTE template for unmapping */
971 if (DVMA_DBG_ON(iommu_p))
972 pci_dvma_alloc_debug(iommu_p, (char *)mp->dmai_mapping,
973 mp->dmai_size, mp);
974 return (DDI_SUCCESS);
975 }
976
977 /*
978 * pci_dvma_map: map non-fasttrack DMA
979 * Use quantum cache if single page DMA.
980 */
981 int
pci_dvma_map(ddi_dma_impl_t * mp,ddi_dma_req_t * dmareq,iommu_t * iommu_p)982 pci_dvma_map(ddi_dma_impl_t *mp, ddi_dma_req_t *dmareq, iommu_t *iommu_p)
983 {
984 uint_t npages = PCI_DMA_WINNPGS(mp);
985 dvma_addr_t dvma_pg, dvma_pg_index;
986 void *dvma_addr;
987 uint64_t tte = GET_TTE_TEMPLATE(mp);
988 int sleep = dmareq->dmar_fp == DDI_DMA_SLEEP ? VM_SLEEP : VM_NOSLEEP;
989 #ifdef DEBUG
990 dev_info_t *dip = mp->dmai_rdip;
991 #endif
992 /*
993 * allocate dvma space resource and map in the first window.
994 * (vmem_t *vmp, size_t size,
995 * size_t align, size_t phase, size_t nocross,
996 * void *minaddr, void *maxaddr, int vmflag)
997 */
998 if ((npages == 1) && !HAS_REDZONE(mp) && HAS_NOSYSLIMIT(mp)) {
999 dvma_addr = vmem_alloc(iommu_p->iommu_dvma_map,
1000 IOMMU_PAGE_SIZE, sleep);
1001 mp->dmai_flags |= DMAI_FLAGS_VMEMCACHE;
1002 #ifdef PCI_DMA_PROF
1003 pci_dvma_vmem_alloc++;
1004 #endif
1005 } else {
1006 dvma_addr = vmem_xalloc(iommu_p->iommu_dvma_map,
1007 IOMMU_PTOB(npages + HAS_REDZONE(mp)),
1008 MAX(mp->dmai_attr.dma_attr_align, IOMMU_PAGE_SIZE),
1009 0,
1010 mp->dmai_attr.dma_attr_seg + 1,
1011 (void *)mp->dmai_attr.dma_attr_addr_lo,
1012 (void *)(mp->dmai_attr.dma_attr_addr_hi + 1),
1013 sleep);
1014 #ifdef PCI_DMA_PROF
1015 pci_dvma_vmem_xalloc++;
1016 #endif
1017 }
1018 dvma_pg = IOMMU_BTOP((ulong_t)dvma_addr);
1019 dvma_pg_index = dvma_pg - iommu_p->dvma_base_pg;
1020 DEBUG2(DBG_DMA_MAP, dip, "fallback dvma_pages: dvma_pg=%x index=%x\n",
1021 dvma_pg, dvma_pg_index);
1022 if (dvma_pg == 0)
1023 goto noresource;
1024
1025 /* allocate DVMA context */
1026 if ((npages >= pci_context_minpages) && PCI_DMA_USECTX(mp)) {
1027 dvma_context_t ctx;
1028 if (ctx = pci_iommu_get_dvma_context(iommu_p, dvma_pg_index)) {
1029 tte |= IOMMU_CTX2TTE(ctx);
1030 mp->dmai_flags |= DMAI_FLAGS_CONTEXT;
1031 }
1032 }
1033 mp->dmai_mapping = mp->dmai_roffset | IOMMU_PTOB(dvma_pg);
1034 mp->dmai_offset = 0;
1035 PCI_SAVE_MP_TTE(mp, tte); /* mp->dmai_tte = tte */
1036 iommu_map_pages(iommu_p, mp, dvma_pg, npages, 0);
1037 return (DDI_SUCCESS);
1038 noresource:
1039 if (dmareq->dmar_fp != DDI_DMA_DONTWAIT) {
1040 DEBUG0(DBG_DMA_MAP, dip, "dvma_pg 0 - set callback\n");
1041 ddi_set_callback(dmareq->dmar_fp, dmareq->dmar_arg,
1042 &iommu_p->iommu_dvma_clid);
1043 }
1044 DEBUG0(DBG_DMA_MAP, dip, "vmem_xalloc - DDI_DMA_NORESOURCES\n");
1045 return (DDI_DMA_NORESOURCES);
1046 }
1047
1048 void
pci_dvma_unmap(iommu_t * iommu_p,ddi_dma_impl_t * mp)1049 pci_dvma_unmap(iommu_t *iommu_p, ddi_dma_impl_t *mp)
1050 {
1051 size_t npages;
1052 dvma_addr_t dvma_addr = (dvma_addr_t)mp->dmai_mapping;
1053 dvma_addr_t dvma_pg = IOMMU_BTOP(dvma_addr);
1054 dvma_addr = IOMMU_PTOB(dvma_pg);
1055
1056 if (mp->dmai_flags & DMAI_FLAGS_FASTTRACK) {
1057 iopfn_t index = dvma_pg - iommu_p->dvma_base_pg;
1058 ASSERT(index % pci_dvma_page_cache_clustsz == 0);
1059 index /= pci_dvma_page_cache_clustsz;
1060 ASSERT(index < pci_dvma_page_cache_entries);
1061 iommu_p->iommu_dvma_cache_locks[index] = 0;
1062 #ifdef PCI_DMA_PROF
1063 pci_dvmaft_free++;
1064 #endif
1065 return;
1066 }
1067 npages = IOMMU_BTOP(mp->dmai_winsize) + HAS_REDZONE(mp);
1068 pci_vmem_free(iommu_p, mp, (void *)dvma_addr, npages);
1069
1070 if (mp->dmai_flags & DMAI_FLAGS_CONTEXT)
1071 pci_iommu_free_dvma_context(iommu_p, MP2CTX(mp));
1072 }
1073
1074 void
pci_dma_sync_unmap(dev_info_t * dip,dev_info_t * rdip,ddi_dma_impl_t * mp)1075 pci_dma_sync_unmap(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp)
1076 {
1077 pci_t *pci_p = get_pci_soft_state(ddi_get_instance(dip));
1078 iommu_t *iommu_p = pci_p->pci_iommu_p;
1079 uint64_t sync_buf_save = SYNC_BUF_PA(mp);
1080 uint32_t fast_track = mp->dmai_flags & DMAI_FLAGS_FASTTRACK;
1081
1082 if (fast_track) {
1083 dvma_addr_t dvma_pg = IOMMU_BTOP(mp->dmai_mapping);
1084
1085 SYNC_BUF_PA(mp) = IOMMU_PAGE_TTEPA(iommu_p, dvma_pg);
1086 ASSERT(!(SYNC_BUF_PA(mp) & PCI_SYNC_FLAG_SIZE - 1));
1087 }
1088
1089 if (pci_dvma_sync_before_unmap) {
1090 pci_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 0, 0,
1091 DDI_DMA_SYNC_FORCPU);
1092 iommu_unmap_window(iommu_p, mp);
1093 } else {
1094 iommu_unmap_window(iommu_p, mp);
1095 pci_dma_sync(dip, rdip, (ddi_dma_handle_t)mp, 0, 0,
1096 DDI_DMA_SYNC_FORCPU);
1097 }
1098
1099 if (fast_track)
1100 SYNC_BUF_PA(mp) = sync_buf_save;
1101 }
1102
1103 /*
1104 * DVMA mappings may have multiple windows, but each window always have
1105 * one segment.
1106 */
1107 int
pci_dvma_ctl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_impl_t * mp,enum ddi_dma_ctlops cmd,off_t * offp,size_t * lenp,caddr_t * objp,uint_t cache_flags)1108 pci_dvma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp,
1109 enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp,
1110 uint_t cache_flags)
1111 {
1112 switch (cmd) {
1113 case DDI_DMA_SYNC:
1114 return (pci_dma_sync(dip, rdip, (ddi_dma_handle_t)mp,
1115 *offp, *lenp, cache_flags));
1116
1117 case DDI_DMA_HTOC: {
1118 int ret;
1119 off_t wo_off, off = *offp; /* wo_off: wnd's obj offset */
1120 uint_t win_size = mp->dmai_winsize;
1121 ddi_dma_cookie_t *cp = (ddi_dma_cookie_t *)objp;
1122
1123 if (off >= mp->dmai_object.dmao_size) {
1124 cmn_err(CE_WARN, "%s%d invalid dma_htoc offset %lx",
1125 NAMEINST(mp->dmai_rdip), off);
1126 return (DDI_FAILURE);
1127 }
1128 off += mp->dmai_roffset;
1129 ret = pci_dma_win(dip, rdip, (ddi_dma_handle_t)mp,
1130 off / win_size, &wo_off, NULL, cp, NULL); /* lenp == NULL */
1131 if (ret)
1132 return (ret);
1133 DEBUG4(DBG_DMA_CTL, dip, "HTOC:cookie=%x+%lx off=%lx,%lx\n",
1134 cp->dmac_address, cp->dmac_size, off, *offp);
1135
1136 /* adjust cookie addr/len if we are not on window boundary */
1137 ASSERT((off % win_size) == (off -
1138 (PCI_DMA_CURWIN(mp) ? mp->dmai_roffset : 0) - wo_off));
1139 off = PCI_DMA_CURWIN(mp) ? off % win_size : *offp;
1140 ASSERT(cp->dmac_size > off);
1141 cp->dmac_laddress += off;
1142 cp->dmac_size -= off;
1143 DEBUG5(DBG_DMA_CTL, dip,
1144 "HTOC:mp=%p cookie=%x+%lx off=%lx,%lx\n",
1145 mp, cp->dmac_address, cp->dmac_size, off, wo_off);
1146 }
1147 return (DDI_SUCCESS);
1148
1149 case DDI_DMA_REPWIN:
1150 *offp = mp->dmai_offset;
1151 *lenp = mp->dmai_size;
1152 return (DDI_SUCCESS);
1153
1154 case DDI_DMA_MOVWIN: {
1155 off_t off = *offp;
1156 if (off >= mp->dmai_object.dmao_size)
1157 return (DDI_FAILURE);
1158 off += mp->dmai_roffset;
1159 return (pci_dma_win(dip, rdip, (ddi_dma_handle_t)mp,
1160 off / mp->dmai_winsize, offp, lenp,
1161 (ddi_dma_cookie_t *)objp, NULL));
1162 }
1163
1164 case DDI_DMA_NEXTWIN: {
1165 window_t win = PCI_DMA_CURWIN(mp);
1166 if (offp) {
1167 if (*(window_t *)offp != win) { /* window not active */
1168 *(window_t *)objp = win; /* return cur win */
1169 return (DDI_DMA_STALE);
1170 }
1171 win++;
1172 } else /* map win 0 */
1173 win = 0;
1174 if (win >= mp->dmai_nwin) {
1175 *(window_t *)objp = win - 1;
1176 return (DDI_DMA_DONE);
1177 }
1178 if (pci_dma_win(dip, rdip, (ddi_dma_handle_t)mp,
1179 win, 0, 0, 0, 0)) {
1180 *(window_t *)objp = win - 1;
1181 return (DDI_FAILURE);
1182 }
1183 *(window_t *)objp = win;
1184 }
1185 return (DDI_SUCCESS);
1186
1187 case DDI_DMA_NEXTSEG:
1188 if (*(window_t *)offp != PCI_DMA_CURWIN(mp))
1189 return (DDI_DMA_STALE);
1190 if (lenp) /* only 1 seg allowed */
1191 return (DDI_DMA_DONE);
1192 /* return mp as seg 0 */
1193 *(ddi_dma_seg_t *)objp = (ddi_dma_seg_t)mp;
1194 return (DDI_SUCCESS);
1195
1196 case DDI_DMA_SEGTOC:
1197 MAKE_DMA_COOKIE((ddi_dma_cookie_t *)objp, mp->dmai_mapping,
1198 mp->dmai_size);
1199 *offp = mp->dmai_offset;
1200 *lenp = mp->dmai_size;
1201 return (DDI_SUCCESS);
1202
1203 case DDI_DMA_COFF: {
1204 ddi_dma_cookie_t *cp = (ddi_dma_cookie_t *)offp;
1205 if (cp->dmac_address < mp->dmai_mapping ||
1206 (cp->dmac_address + cp->dmac_size) >
1207 (mp->dmai_mapping + mp->dmai_size))
1208 return (DDI_FAILURE);
1209 *objp = (caddr_t)(cp->dmac_address - mp->dmai_mapping +
1210 mp->dmai_offset);
1211 }
1212 return (DDI_SUCCESS);
1213
1214 case DDI_DMA_REMAP:
1215 if (pci_dvma_remap_enabled)
1216 return (pci_dvma_remap(dip, rdip, mp, *offp, *lenp));
1217 return (DDI_FAILURE);
1218
1219 default:
1220 DEBUG3(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n",
1221 cmd, ddi_driver_name(rdip), ddi_get_instance(rdip));
1222 break;
1223 }
1224 return (DDI_FAILURE);
1225 }
1226
1227 void
pci_dma_freewin(ddi_dma_impl_t * mp)1228 pci_dma_freewin(ddi_dma_impl_t *mp)
1229 {
1230 pci_dma_win_t *win_p = mp->dmai_winlst, *win2_p;
1231 for (win2_p = win_p; win_p; win2_p = win_p) {
1232 win_p = win2_p->win_next;
1233 kmem_free(win2_p, sizeof (pci_dma_win_t) +
1234 sizeof (ddi_dma_cookie_t) * win2_p->win_ncookies);
1235 }
1236 mp->dmai_nwin = 0;
1237 mp->dmai_winlst = NULL;
1238 }
1239
1240 /*
1241 * pci_dma_newwin - create a dma window object and cookies
1242 *
1243 * After the initial scan in pci_dma_physwin(), which identifies
1244 * a portion of the pfn array that belongs to a dma window,
1245 * we are called to allocate and initialize representing memory
1246 * resources. We know from the 1st scan the number of cookies
1247 * or dma segment in this window so we can allocate a contiguous
1248 * memory array for the dma cookies (The implementation of
1249 * ddi_dma_nextcookie(9f) dictates dma cookies be contiguous).
1250 *
1251 * A second round scan is done on the pfn array to identify
1252 * each dma segment and initialize its corresponding dma cookie.
1253 * We don't need to do all the safety checking and we know they
1254 * all belong to the same dma window.
1255 *
1256 * Input: cookie_no - # of cookies identified by the 1st scan
1257 * start_idx - subscript of the pfn array for the starting pfn
1258 * end_idx - subscript of the last pfn in dma window
1259 * win_pp - pointer to win_next member of previous window
1260 * Return: DDI_SUCCESS - with **win_pp as newly created window object
1261 * DDI_DMA_NORESROUCE - caller frees all previous window objs
1262 * Note: Each cookie and window size are all initialized on page
1263 * boundary. This is not true for the 1st cookie of the 1st
1264 * window and the last cookie of the last window.
1265 * We fix that later in upper layer which has access to size
1266 * and offset info.
1267 *
1268 */
1269 static int
pci_dma_newwin(ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp,uint32_t cookie_no,uint32_t start_idx,uint32_t end_idx,pci_dma_win_t ** win_pp,uint64_t count_max,uint64_t bypass_prefix)1270 pci_dma_newwin(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, uint32_t cookie_no,
1271 uint32_t start_idx, uint32_t end_idx, pci_dma_win_t **win_pp,
1272 uint64_t count_max, uint64_t bypass_prefix)
1273 {
1274 int (*waitfp)(caddr_t) = dmareq->dmar_fp;
1275 ddi_dma_cookie_t *cookie_p;
1276 uint32_t pfn_no = 1;
1277 iopfn_t pfn = PCI_GET_MP_PFN(mp, start_idx);
1278 iopfn_t prev_pfn = pfn;
1279 uint64_t seg_pfn0 = pfn;
1280 size_t sz = cookie_no * sizeof (ddi_dma_cookie_t);
1281 pci_dma_win_t *win_p = kmem_alloc(sizeof (pci_dma_win_t) + sz,
1282 waitfp == DDI_DMA_SLEEP ? KM_SLEEP : KM_NOSLEEP);
1283 if (!win_p)
1284 goto noresource;
1285
1286 win_p->win_next = NULL;
1287 win_p->win_ncookies = cookie_no;
1288 win_p->win_curseg = 0; /* start from segment 0 */
1289 win_p->win_size = IOMMU_PTOB(end_idx - start_idx + 1);
1290 /* win_p->win_offset is left uninitialized */
1291
1292 cookie_p = (ddi_dma_cookie_t *)(win_p + 1);
1293 start_idx++;
1294 for (; start_idx <= end_idx; start_idx++, prev_pfn = pfn, pfn_no++) {
1295 pfn = PCI_GET_MP_PFN1(mp, start_idx);
1296 if ((pfn == prev_pfn + 1) &&
1297 (IOMMU_PTOB(pfn_no + 1) - 1 <= count_max))
1298 continue;
1299
1300 /* close up the cookie up to (including) prev_pfn */
1301 MAKE_DMA_COOKIE(cookie_p, IOMMU_PTOB(seg_pfn0) | bypass_prefix,
1302 IOMMU_PTOB(pfn_no));
1303 DEBUG2(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages)\n",
1304 IOMMU_PTOB(seg_pfn0) | bypass_prefix, pfn_no);
1305
1306 cookie_p++; /* advance to next available cookie cell */
1307 pfn_no = 0;
1308 seg_pfn0 = pfn; /* start a new segment from current pfn */
1309 }
1310 MAKE_DMA_COOKIE(cookie_p, IOMMU_PTOB(seg_pfn0) | bypass_prefix,
1311 IOMMU_PTOB(pfn_no));
1312 DEBUG3(DBG_BYPASS, mp->dmai_rdip, "cookie %p (%x pages) of total %x\n",
1313 IOMMU_PTOB(seg_pfn0) | bypass_prefix, pfn_no, cookie_no);
1314 #ifdef DEBUG
1315 cookie_p++;
1316 ASSERT((cookie_p - (ddi_dma_cookie_t *)(win_p + 1)) == cookie_no);
1317 #endif
1318 *win_pp = win_p;
1319 return (DDI_SUCCESS);
1320 noresource:
1321 if (waitfp != DDI_DMA_DONTWAIT)
1322 ddi_set_callback(waitfp, dmareq->dmar_arg, &pci_kmem_clid);
1323 return (DDI_DMA_NORESOURCES);
1324 }
1325
1326 /*
1327 * pci_dma_adjust - adjust 1st and last cookie and window sizes
1328 * remove initial dma page offset from 1st cookie and window size
1329 * remove last dma page remainder from last cookie and window size
1330 * fill win_offset of each dma window according to just fixed up
1331 * each window sizes
1332 * pci_dma_win_t members modified:
1333 * win_p->win_offset - this window's offset within entire DMA object
1334 * win_p->win_size - xferrable size (in bytes) for this window
1335 *
1336 * ddi_dma_impl_t members modified:
1337 * mp->dmai_size - 1st window xferrable size
1338 * mp->dmai_offset - 0, which is the dma offset of the 1st window
1339 *
1340 * ddi_dma_cookie_t members modified:
1341 * cookie_p->dmac_size - 1st and last cookie remove offset or remainder
1342 * cookie_p->dmac_laddress - 1st cookie add page offset
1343 */
1344 static void
pci_dma_adjust(ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp,pci_dma_win_t * win_p)1345 pci_dma_adjust(ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp, pci_dma_win_t *win_p)
1346 {
1347 ddi_dma_cookie_t *cookie_p = (ddi_dma_cookie_t *)(win_p + 1);
1348 size_t pg_offset = mp->dmai_roffset;
1349 size_t win_offset = 0;
1350
1351 cookie_p->dmac_size -= pg_offset;
1352 cookie_p->dmac_laddress |= pg_offset;
1353 win_p->win_size -= pg_offset;
1354 DEBUG1(DBG_BYPASS, mp->dmai_rdip, "pg0 adjust %lx\n", pg_offset);
1355
1356 mp->dmai_size = win_p->win_size;
1357 mp->dmai_offset = 0;
1358
1359 pg_offset += mp->dmai_object.dmao_size;
1360 pg_offset &= IOMMU_PAGE_OFFSET;
1361 if (pg_offset)
1362 pg_offset = IOMMU_PAGE_SIZE - pg_offset;
1363 DEBUG1(DBG_BYPASS, mp->dmai_rdip, "last pg adjust %lx\n", pg_offset);
1364
1365 for (; win_p->win_next; win_p = win_p->win_next) {
1366 DEBUG1(DBG_BYPASS, mp->dmai_rdip, "win off %p\n", win_offset);
1367 win_p->win_offset = win_offset;
1368 win_offset += win_p->win_size;
1369 }
1370 /* last window */
1371 win_p->win_offset = win_offset;
1372 cookie_p = (ddi_dma_cookie_t *)(win_p + 1);
1373 cookie_p[win_p->win_ncookies - 1].dmac_size -= pg_offset;
1374 win_p->win_size -= pg_offset;
1375 ASSERT((win_offset + win_p->win_size) == mp->dmai_object.dmao_size);
1376 }
1377
1378 /*
1379 * pci_dma_physwin() - carve up dma windows using physical addresses.
1380 * Called to handle iommu bypass and pci peer-to-peer transfers.
1381 * Calls pci_dma_newwin() to allocate window objects.
1382 *
1383 * Dependency: mp->dmai_pfnlst points to an array of pfns
1384 *
1385 * 1. Each dma window is represented by a pci_dma_win_t object.
1386 * The object will be casted to ddi_dma_win_t and returned
1387 * to leaf driver through the DDI interface.
1388 * 2. Each dma window can have several dma segments with each
1389 * segment representing a physically contiguous either memory
1390 * space (if we are doing an iommu bypass transfer) or pci address
1391 * space (if we are doing a peer-to-peer transfer).
1392 * 3. Each segment has a DMA cookie to program the DMA engine.
1393 * The cookies within each DMA window must be located in a
1394 * contiguous array per ddi_dma_nextcookie(9f).
1395 * 4. The number of DMA segments within each DMA window cannot exceed
1396 * mp->dmai_attr.dma_attr_sgllen. If the transfer size is
1397 * too large to fit in the sgllen, the rest needs to be
1398 * relocated to the next dma window.
1399 * 5. Peer-to-peer DMA segment follows device hi, lo, count_max,
1400 * and nocross restrictions while bypass DMA follows the set of
1401 * restrictions with system limits factored in.
1402 *
1403 * Return:
1404 * mp->dmai_winlst - points to a link list of pci_dma_win_t objects.
1405 * Each pci_dma_win_t object on the link list contains
1406 * infomation such as its window size (# of pages),
1407 * starting offset (also see Restriction), an array of
1408 * DMA cookies, and # of cookies in the array.
1409 * mp->dmai_pfnlst - NULL, the pfn list is freed to conserve memory.
1410 * mp->dmai_nwin - # of total DMA windows on mp->dmai_winlst.
1411 * mp->dmai_mapping - starting cookie address
1412 * mp->dmai_rflags - consistent, nosync, no redzone
1413 * mp->dmai_cookie - start of cookie table of the 1st DMA window
1414 *
1415 * Restriction:
1416 * Each pci_dma_win_t object can theoratically start from any offset
1417 * since the iommu is not involved. However, this implementation
1418 * always make windows start from page aligned offset (except
1419 * the 1st window, which follows the requested offset) due to the
1420 * fact that we are handed a pfn list. This does require device's
1421 * count_max and attr_seg to be at least IOMMU_PAGE_SIZE aligned.
1422 */
1423 int
pci_dma_physwin(pci_t * pci_p,ddi_dma_req_t * dmareq,ddi_dma_impl_t * mp)1424 pci_dma_physwin(pci_t *pci_p, ddi_dma_req_t *dmareq, ddi_dma_impl_t *mp)
1425 {
1426 uint_t npages = mp->dmai_ndvmapages;
1427 int ret, sgllen = mp->dmai_attr.dma_attr_sgllen;
1428 iopfn_t pfn_lo, pfn_hi, prev_pfn, bypass_pfn;
1429 iopfn_t pfn = PCI_GET_MP_PFN(mp, 0);
1430 uint32_t i, win_no = 0, pfn_no = 1, win_pfn0_index = 0, cookie_no = 0;
1431 uint64_t count_max, bypass = PCI_DMA_BYPASS_PREFIX(mp, pfn);
1432 pci_dma_win_t **win_pp = (pci_dma_win_t **)&mp->dmai_winlst;
1433 ddi_dma_cookie_t *cookie0_p;
1434
1435 if (PCI_DMA_ISPTP(mp)) { /* ignore sys limits for peer-to-peer */
1436 ddi_dma_attr_t *dev_attr_p = DEV_ATTR(mp);
1437 iopfn_t pfn_base = pci_p->pci_pbm_p->pbm_base_pfn;
1438 iopfn_t pfn_last = pci_p->pci_pbm_p->pbm_last_pfn - pfn_base;
1439 uint64_t nocross = dev_attr_p->dma_attr_seg;
1440 if (nocross && (nocross < UINT32_MAX))
1441 return (DDI_DMA_NOMAPPING);
1442 if (dev_attr_p->dma_attr_align > IOMMU_PAGE_SIZE)
1443 return (DDI_DMA_NOMAPPING);
1444 pfn_lo = IOMMU_BTOP(dev_attr_p->dma_attr_addr_lo);
1445 pfn_hi = IOMMU_BTOP(dev_attr_p->dma_attr_addr_hi);
1446 pfn_hi = MIN(pfn_hi, pfn_last);
1447 if ((pfn_lo > pfn_hi) || (pfn < pfn_lo))
1448 return (DDI_DMA_NOMAPPING);
1449 count_max = dev_attr_p->dma_attr_count_max;
1450 count_max = MIN(count_max, nocross);
1451 /*
1452 * the following count_max trim is not done because we are
1453 * making sure pfn_lo <= pfn <= pfn_hi inside the loop
1454 * count_max=MIN(count_max, IOMMU_PTOB(pfn_hi - pfn_lo + 1)-1);
1455 */
1456 } else { /* bypass hi/lo/count_max have been processed by attr2hdl() */
1457 count_max = mp->dmai_attr.dma_attr_count_max;
1458 pfn_lo = IOMMU_BTOP(mp->dmai_attr.dma_attr_addr_lo);
1459 pfn_hi = IOMMU_BTOP(mp->dmai_attr.dma_attr_addr_hi);
1460 }
1461
1462 bypass_pfn = IOMMU_BTOP(bypass);
1463
1464 for (prev_pfn = (bypass_pfn | pfn), i = 1; i < npages;
1465 i++, prev_pfn = pfn, pfn_no++) {
1466 pfn = bypass_pfn | PCI_GET_MP_PFN1(mp, i);
1467 if ((pfn == prev_pfn + 1) &&
1468 (IOMMU_PTOB(pfn_no + 1) - 1 <= count_max))
1469 continue;
1470 if ((pfn < pfn_lo) || (prev_pfn > pfn_hi)) {
1471 ret = DDI_DMA_NOMAPPING;
1472 goto err;
1473 }
1474 cookie_no++;
1475 pfn_no = 0;
1476 if (cookie_no < sgllen)
1477 continue;
1478
1479 DEBUG3(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n",
1480 win_pfn0_index, i - 1, cookie_no);
1481 if (ret = pci_dma_newwin(dmareq, mp, cookie_no,
1482 win_pfn0_index, i - 1, win_pp, count_max, bypass))
1483 goto err;
1484
1485 win_pp = &(*win_pp)->win_next; /* win_pp = *(win_pp) */
1486 win_no++;
1487 win_pfn0_index = i;
1488 cookie_no = 0;
1489 }
1490 if (pfn > pfn_hi) {
1491 ret = DDI_DMA_NOMAPPING;
1492 goto err;
1493 }
1494 cookie_no++;
1495 DEBUG3(DBG_BYPASS, mp->dmai_rdip, "newwin pfn[%x-%x] %x cks\n",
1496 win_pfn0_index, i - 1, cookie_no);
1497 if (ret = pci_dma_newwin(dmareq, mp, cookie_no, win_pfn0_index,
1498 i - 1, win_pp, count_max, bypass))
1499 goto err;
1500 win_no++;
1501 pci_dma_adjust(dmareq, mp, mp->dmai_winlst);
1502 mp->dmai_nwin = win_no;
1503 mp->dmai_rflags |= DDI_DMA_CONSISTENT;
1504 if (!pci_p->pci_pbm_p->pbm_sync_reg_pa) {
1505 mp->dmai_rflags |= DMP_NOSYNC;
1506 mp->dmai_flags |= DMAI_FLAGS_NOSYNC;
1507 }
1508 mp->dmai_rflags &= ~DDI_DMA_REDZONE;
1509 cookie0_p = (ddi_dma_cookie_t *)(WINLST(mp) + 1);
1510 mp->dmai_cookie = WINLST(mp)->win_ncookies > 1 ? cookie0_p + 1 : 0;
1511 mp->dmai_mapping = cookie0_p->dmac_laddress;
1512
1513 pci_dma_freepfn(mp);
1514 return (DDI_DMA_MAPPED);
1515 err:
1516 pci_dma_freewin(mp);
1517 return (ret);
1518 }
1519
1520 /*ARGSUSED*/
1521 int
pci_dma_ctl(dev_info_t * dip,dev_info_t * rdip,ddi_dma_impl_t * mp,enum ddi_dma_ctlops cmd,off_t * offp,size_t * lenp,caddr_t * objp,uint_t cache_flags)1522 pci_dma_ctl(dev_info_t *dip, dev_info_t *rdip, ddi_dma_impl_t *mp,
1523 enum ddi_dma_ctlops cmd, off_t *offp, size_t *lenp, caddr_t *objp,
1524 uint_t cache_flags)
1525 {
1526 switch (cmd) {
1527 case DDI_DMA_SYNC: /* XXX */
1528 return (DDI_SUCCESS);
1529
1530 case DDI_DMA_HTOC: {
1531 off_t off = *offp;
1532 ddi_dma_cookie_t *loop_cp, *cp;
1533 pci_dma_win_t *win_p = mp->dmai_winlst;
1534
1535 if (off >= mp->dmai_object.dmao_size)
1536 return (DDI_FAILURE);
1537
1538 /* locate window */
1539 while (win_p->win_offset + win_p->win_size <= off)
1540 win_p = win_p->win_next;
1541
1542 loop_cp = cp = (ddi_dma_cookie_t *)(win_p + 1);
1543 mp->dmai_offset = win_p->win_offset;
1544 mp->dmai_size = win_p->win_size;
1545 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 start addr */
1546
1547 /* adjust cookie addr/len if we are not on cookie boundary */
1548 off -= win_p->win_offset; /* offset within window */
1549 for (; off >= loop_cp->dmac_size; loop_cp++)
1550 off -= loop_cp->dmac_size; /* offset within cookie */
1551
1552 mp->dmai_cookie = loop_cp + 1;
1553 win_p->win_curseg = loop_cp - cp;
1554 cp = (ddi_dma_cookie_t *)objp;
1555 MAKE_DMA_COOKIE(cp, loop_cp->dmac_laddress + off,
1556 loop_cp->dmac_size - off);
1557
1558 DEBUG2(DBG_DMA_CTL, dip,
1559 "HTOC: cookie - dmac_laddress=%p dmac_size=%x\n",
1560 cp->dmac_laddress, cp->dmac_size);
1561 }
1562 return (DDI_SUCCESS);
1563
1564 case DDI_DMA_REPWIN:
1565 *offp = mp->dmai_offset;
1566 *lenp = mp->dmai_size;
1567 return (DDI_SUCCESS);
1568
1569 case DDI_DMA_MOVWIN: {
1570 off_t off = *offp;
1571 ddi_dma_cookie_t *cp;
1572 pci_dma_win_t *win_p = mp->dmai_winlst;
1573
1574 if (off >= mp->dmai_object.dmao_size)
1575 return (DDI_FAILURE);
1576
1577 /* locate window */
1578 while (win_p->win_offset + win_p->win_size <= off)
1579 win_p = win_p->win_next;
1580
1581 cp = (ddi_dma_cookie_t *)(win_p + 1);
1582 mp->dmai_offset = win_p->win_offset;
1583 mp->dmai_size = win_p->win_size;
1584 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 star addr */
1585 mp->dmai_cookie = cp + 1;
1586 win_p->win_curseg = 0;
1587
1588 *(ddi_dma_cookie_t *)objp = *cp;
1589 *offp = win_p->win_offset;
1590 *lenp = win_p->win_size;
1591 DEBUG2(DBG_DMA_CTL, dip,
1592 "HTOC: cookie - dmac_laddress=%p dmac_size=%x\n",
1593 cp->dmac_laddress, cp->dmac_size);
1594 }
1595 return (DDI_SUCCESS);
1596
1597 case DDI_DMA_NEXTWIN: {
1598 pci_dma_win_t *win_p = *(pci_dma_win_t **)offp;
1599 pci_dma_win_t **nw_pp = (pci_dma_win_t **)objp;
1600 ddi_dma_cookie_t *cp;
1601 if (!win_p) {
1602 *nw_pp = mp->dmai_winlst;
1603 return (DDI_SUCCESS);
1604 }
1605
1606 if (win_p->win_offset != mp->dmai_offset)
1607 return (DDI_DMA_STALE);
1608 if (!win_p->win_next)
1609 return (DDI_DMA_DONE);
1610 win_p = win_p->win_next;
1611 cp = (ddi_dma_cookie_t *)(win_p + 1);
1612 mp->dmai_offset = win_p->win_offset;
1613 mp->dmai_size = win_p->win_size;
1614 mp->dmai_mapping = cp->dmac_laddress; /* cookie0 star addr */
1615 mp->dmai_cookie = cp + 1;
1616 win_p->win_curseg = 0;
1617 *nw_pp = win_p;
1618 }
1619 return (DDI_SUCCESS);
1620
1621 case DDI_DMA_NEXTSEG: {
1622 pci_dma_win_t *w_p = *(pci_dma_win_t **)offp;
1623 if (w_p->win_offset != mp->dmai_offset)
1624 return (DDI_DMA_STALE);
1625 if (w_p->win_curseg + 1 >= w_p->win_ncookies)
1626 return (DDI_DMA_DONE);
1627 w_p->win_curseg++;
1628 }
1629 *(ddi_dma_seg_t *)objp = (ddi_dma_seg_t)mp;
1630 return (DDI_SUCCESS);
1631
1632 case DDI_DMA_SEGTOC: {
1633 pci_dma_win_t *win_p = mp->dmai_winlst;
1634 off_t off = mp->dmai_offset;
1635 ddi_dma_cookie_t *cp;
1636 int i;
1637
1638 /* locate active window */
1639 for (; win_p->win_offset != off; win_p = win_p->win_next)
1640 ;
1641 cp = (ddi_dma_cookie_t *)(win_p + 1);
1642 for (i = 0; i < win_p->win_curseg; i++, cp++)
1643 off += cp->dmac_size;
1644 *offp = off;
1645 *lenp = cp->dmac_size;
1646 *(ddi_dma_cookie_t *)objp = *cp; /* copy cookie */
1647 }
1648 return (DDI_SUCCESS);
1649
1650 case DDI_DMA_COFF: {
1651 pci_dma_win_t *win_p;
1652 ddi_dma_cookie_t *cp;
1653 uint64_t addr, key = ((ddi_dma_cookie_t *)offp)->dmac_laddress;
1654 size_t win_off;
1655
1656 for (win_p = mp->dmai_winlst; win_p; win_p = win_p->win_next) {
1657 int i;
1658 win_off = 0;
1659 cp = (ddi_dma_cookie_t *)(win_p + 1);
1660 for (i = 0; i < win_p->win_ncookies; i++, cp++) {
1661 size_t sz = cp->dmac_size;
1662
1663 addr = cp->dmac_laddress;
1664 if ((addr <= key) && (addr + sz >= key))
1665 goto found;
1666 win_off += sz;
1667 }
1668 }
1669 return (DDI_FAILURE);
1670 found:
1671 *objp = (caddr_t)(win_p->win_offset + win_off + (key - addr));
1672 return (DDI_SUCCESS);
1673 }
1674
1675 case DDI_DMA_REMAP:
1676 return (DDI_FAILURE);
1677
1678 default:
1679 DEBUG3(DBG_DMA_CTL, dip, "unknown command (%x): rdip=%s%d\n",
1680 cmd, ddi_driver_name(rdip), ddi_get_instance(rdip));
1681 break;
1682 }
1683 return (DDI_FAILURE);
1684 }
1685
1686 static void
pci_dvma_debug_init(iommu_t * iommu_p)1687 pci_dvma_debug_init(iommu_t *iommu_p)
1688 {
1689 size_t sz = sizeof (struct dvma_rec) * pci_dvma_debug_rec;
1690 ASSERT(MUTEX_HELD(&iommu_p->dvma_debug_lock));
1691 cmn_err(CE_NOTE, "PCI DVMA %p stat ON", iommu_p);
1692
1693 iommu_p->dvma_alloc_rec = kmem_zalloc(sz, KM_SLEEP);
1694 iommu_p->dvma_free_rec = kmem_zalloc(sz, KM_SLEEP);
1695
1696 iommu_p->dvma_active_list = NULL;
1697 iommu_p->dvma_alloc_rec_index = 0;
1698 iommu_p->dvma_free_rec_index = 0;
1699 iommu_p->dvma_active_count = 0;
1700 }
1701
1702 void
pci_dvma_debug_fini(iommu_t * iommu_p)1703 pci_dvma_debug_fini(iommu_t *iommu_p)
1704 {
1705 struct dvma_rec *prev, *ptr;
1706 size_t sz = sizeof (struct dvma_rec) * pci_dvma_debug_rec;
1707 uint64_t mask = ~(1ull << iommu_p->iommu_inst);
1708 cmn_err(CE_NOTE, "PCI DVMA %p stat OFF", iommu_p);
1709
1710 kmem_free(iommu_p->dvma_alloc_rec, sz);
1711 kmem_free(iommu_p->dvma_free_rec, sz);
1712 iommu_p->dvma_alloc_rec = iommu_p->dvma_free_rec = NULL;
1713
1714 prev = iommu_p->dvma_active_list;
1715 if (!prev)
1716 return;
1717 for (ptr = prev->next; ptr; prev = ptr, ptr = ptr->next)
1718 kmem_free(prev, sizeof (struct dvma_rec));
1719 kmem_free(prev, sizeof (struct dvma_rec));
1720
1721 iommu_p->dvma_active_list = NULL;
1722 iommu_p->dvma_alloc_rec_index = 0;
1723 iommu_p->dvma_free_rec_index = 0;
1724 iommu_p->dvma_active_count = 0;
1725
1726 pci_dvma_debug_on &= mask;
1727 pci_dvma_debug_off &= mask;
1728 }
1729
1730 void
pci_dvma_alloc_debug(iommu_t * iommu_p,char * address,uint_t len,ddi_dma_impl_t * mp)1731 pci_dvma_alloc_debug(iommu_t *iommu_p, char *address, uint_t len,
1732 ddi_dma_impl_t *mp)
1733 {
1734 struct dvma_rec *ptr;
1735 mutex_enter(&iommu_p->dvma_debug_lock);
1736
1737 if (!iommu_p->dvma_alloc_rec)
1738 pci_dvma_debug_init(iommu_p);
1739 if (DVMA_DBG_OFF(iommu_p)) {
1740 pci_dvma_debug_fini(iommu_p);
1741 goto done;
1742 }
1743
1744 ptr = &iommu_p->dvma_alloc_rec[iommu_p->dvma_alloc_rec_index];
1745 ptr->dvma_addr = address;
1746 ptr->len = len;
1747 ptr->mp = mp;
1748 if (++iommu_p->dvma_alloc_rec_index == pci_dvma_debug_rec)
1749 iommu_p->dvma_alloc_rec_index = 0;
1750
1751 ptr = kmem_alloc(sizeof (struct dvma_rec), KM_SLEEP);
1752 ptr->dvma_addr = address;
1753 ptr->len = len;
1754 ptr->mp = mp;
1755
1756 ptr->next = iommu_p->dvma_active_list;
1757 iommu_p->dvma_active_list = ptr;
1758 iommu_p->dvma_active_count++;
1759 done:
1760 mutex_exit(&iommu_p->dvma_debug_lock);
1761 }
1762
1763 void
pci_dvma_free_debug(iommu_t * iommu_p,char * address,uint_t len,ddi_dma_impl_t * mp)1764 pci_dvma_free_debug(iommu_t *iommu_p, char *address, uint_t len,
1765 ddi_dma_impl_t *mp)
1766 {
1767 struct dvma_rec *ptr, *ptr_save;
1768 mutex_enter(&iommu_p->dvma_debug_lock);
1769
1770 if (!iommu_p->dvma_alloc_rec)
1771 pci_dvma_debug_init(iommu_p);
1772 if (DVMA_DBG_OFF(iommu_p)) {
1773 pci_dvma_debug_fini(iommu_p);
1774 goto done;
1775 }
1776
1777 ptr = &iommu_p->dvma_free_rec[iommu_p->dvma_free_rec_index];
1778 ptr->dvma_addr = address;
1779 ptr->len = len;
1780 ptr->mp = mp;
1781 if (++iommu_p->dvma_free_rec_index == pci_dvma_debug_rec)
1782 iommu_p->dvma_free_rec_index = 0;
1783
1784 ptr_save = iommu_p->dvma_active_list;
1785 for (ptr = ptr_save; ptr; ptr = ptr->next) {
1786 if ((ptr->dvma_addr == address) && (ptr->len = len))
1787 break;
1788 ptr_save = ptr;
1789 }
1790 if (!ptr) {
1791 cmn_err(CE_WARN, "bad dvma free addr=%lx len=%x",
1792 (long)address, len);
1793 goto done;
1794 }
1795 if (ptr == iommu_p->dvma_active_list)
1796 iommu_p->dvma_active_list = ptr->next;
1797 else
1798 ptr_save->next = ptr->next;
1799 kmem_free(ptr, sizeof (struct dvma_rec));
1800 iommu_p->dvma_active_count--;
1801 done:
1802 mutex_exit(&iommu_p->dvma_debug_lock);
1803 }
1804
1805 #ifdef DEBUG
1806 void
dump_dma_handle(uint64_t flag,dev_info_t * dip,ddi_dma_impl_t * hp)1807 dump_dma_handle(uint64_t flag, dev_info_t *dip, ddi_dma_impl_t *hp)
1808 {
1809 DEBUG4(flag, dip, "mp(%p): flags=%x mapping=%lx xfer_size=%x\n",
1810 hp, hp->dmai_inuse, hp->dmai_mapping, hp->dmai_size);
1811 DEBUG4(flag|DBG_CONT, dip, "\tnpages=%x roffset=%x rflags=%x nwin=%x\n",
1812 hp->dmai_ndvmapages, hp->dmai_roffset, hp->dmai_rflags,
1813 hp->dmai_nwin);
1814 DEBUG4(flag|DBG_CONT, dip, "\twinsize=%x tte=%p pfnlst=%p pfn0=%p\n",
1815 hp->dmai_winsize, hp->dmai_tte, hp->dmai_pfnlst, hp->dmai_pfn0);
1816 DEBUG4(flag|DBG_CONT, dip, "\twinlst=%x obj=%p attr=%p ckp=%p\n",
1817 hp->dmai_winlst, &hp->dmai_object, &hp->dmai_attr,
1818 hp->dmai_cookie);
1819 }
1820 #endif
1821
1822 void
pci_vmem_do_free(iommu_t * iommu_p,void * base_addr,size_t npages,int vmemcache)1823 pci_vmem_do_free(iommu_t *iommu_p, void *base_addr, size_t npages,
1824 int vmemcache)
1825 {
1826 vmem_t *map_p = iommu_p->iommu_dvma_map;
1827
1828 if (vmemcache) {
1829 vmem_free(map_p, base_addr, IOMMU_PAGE_SIZE);
1830 #ifdef PCI_DMA_PROF
1831 pci_dvma_vmem_free++;
1832 #endif
1833 return;
1834 }
1835
1836 vmem_xfree(map_p, base_addr, IOMMU_PTOB(npages));
1837 #ifdef PCI_DMA_PROF
1838 pci_dvma_vmem_xfree++;
1839 #endif
1840 }
1841