1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 * Use is subject to license terms.
25 */
26
27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
28 /* All Rights Reserved */
29
30 /*
31 * University Copyright- Copyright (c) 1982, 1986, 1988
32 * The Regents of the University of California
33 * All Rights Reserved
34 *
35 * University Acknowledgment- Portions of this document are derived from
36 * software developed by the University of California, Berkeley, and its
37 * contributors.
38 */
39
40 /*
41 * VM - segment of a mapped device.
42 *
43 * This segment driver is used when mapping character special devices.
44 */
45
46 #include <sys/types.h>
47 #include <sys/t_lock.h>
48 #include <sys/sysmacros.h>
49 #include <sys/vtrace.h>
50 #include <sys/systm.h>
51 #include <sys/vmsystm.h>
52 #include <sys/mman.h>
53 #include <sys/errno.h>
54 #include <sys/kmem.h>
55 #include <sys/cmn_err.h>
56 #include <sys/vnode.h>
57 #include <sys/proc.h>
58 #include <sys/conf.h>
59 #include <sys/debug.h>
60 #include <sys/ddidevmap.h>
61 #include <sys/ddi_implfuncs.h>
62 #include <sys/lgrp.h>
63
64 #include <vm/page.h>
65 #include <vm/hat.h>
66 #include <vm/as.h>
67 #include <vm/seg.h>
68 #include <vm/seg_dev.h>
69 #include <vm/seg_kp.h>
70 #include <vm/seg_kmem.h>
71 #include <vm/vpage.h>
72
73 #include <sys/sunddi.h>
74 #include <sys/esunddi.h>
75 #include <sys/fs/snode.h>
76
77
78 #if DEBUG
79 int segdev_debug;
80 #define DEBUGF(level, args) { if (segdev_debug >= (level)) cmn_err args; }
81 #else
82 #define DEBUGF(level, args)
83 #endif
84
85 /* Default timeout for devmap context management */
86 #define CTX_TIMEOUT_VALUE 0
87
88 #define HOLD_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
89 { mutex_enter(&dhp->dh_lock); }
90
91 #define RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
92 { mutex_exit(&dhp->dh_lock); }
93
94 #define round_down_p2(a, s) ((a) & ~((s) - 1))
95 #define round_up_p2(a, s) (((a) + (s) - 1) & ~((s) - 1))
96
97 /*
98 * VA_PA_ALIGNED checks to see if both VA and PA are on pgsize boundary
99 * VA_PA_PGSIZE_ALIGNED check to see if VA is aligned with PA w.r.t. pgsize
100 */
101 #define VA_PA_ALIGNED(uvaddr, paddr, pgsize) \
102 (((uvaddr | paddr) & (pgsize - 1)) == 0)
103 #define VA_PA_PGSIZE_ALIGNED(uvaddr, paddr, pgsize) \
104 (((uvaddr ^ paddr) & (pgsize - 1)) == 0)
105
106 #define vpgtob(n) ((n) * sizeof (struct vpage)) /* For brevity */
107
108 #define VTOCVP(vp) (VTOS(vp)->s_commonvp) /* we "know" it's an snode */
109
110 static struct devmap_ctx *devmapctx_list = NULL;
111 static struct devmap_softlock *devmap_slist = NULL;
112
113 /*
114 * mutex, vnode and page for the page of zeros we use for the trash mappings.
115 * One trash page is allocated on the first ddi_umem_setup call that uses it
116 * XXX Eventually, we may want to combine this with what segnf does when all
117 * hat layers implement HAT_NOFAULT.
118 *
119 * The trash page is used when the backing store for a userland mapping is
120 * removed but the application semantics do not take kindly to a SIGBUS.
121 * In that scenario, the applications pages are mapped to some dummy page
122 * which returns garbage on read and writes go into a common place.
123 * (Perfect for NO_FAULT semantics)
124 * The device driver is responsible to communicating to the app with some
125 * other mechanism that such remapping has happened and the app should take
126 * corrective action.
127 * We can also use an anonymous memory page as there is no requirement to
128 * keep the page locked, however this complicates the fault code. RFE.
129 */
130 static struct vnode trashvp;
131 static struct page *trashpp;
132
133 /* Non-pageable kernel memory is allocated from the umem_np_arena. */
134 static vmem_t *umem_np_arena;
135
136 /* Set the cookie to a value we know will never be a valid umem_cookie */
137 #define DEVMAP_DEVMEM_COOKIE ((ddi_umem_cookie_t)0x1)
138
139 /*
140 * Macros to check if type of devmap handle
141 */
142 #define cookie_is_devmem(c) \
143 ((c) == (struct ddi_umem_cookie *)DEVMAP_DEVMEM_COOKIE)
144
145 #define cookie_is_pmem(c) \
146 ((c) == (struct ddi_umem_cookie *)DEVMAP_PMEM_COOKIE)
147
148 #define cookie_is_kpmem(c) (!cookie_is_devmem(c) && !cookie_is_pmem(c) &&\
149 ((c)->type == KMEM_PAGEABLE))
150
151 #define dhp_is_devmem(dhp) \
152 (cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
153
154 #define dhp_is_pmem(dhp) \
155 (cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
156
157 #define dhp_is_kpmem(dhp) \
158 (cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
159
160 /*
161 * Private seg op routines.
162 */
163 static int segdev_dup(struct seg *, struct seg *);
164 static int segdev_unmap(struct seg *, caddr_t, size_t);
165 static void segdev_free(struct seg *);
166 static faultcode_t segdev_fault(struct hat *, struct seg *, caddr_t, size_t,
167 enum fault_type, enum seg_rw);
168 static faultcode_t segdev_faulta(struct seg *, caddr_t);
169 static int segdev_setprot(struct seg *, caddr_t, size_t, uint_t);
170 static int segdev_checkprot(struct seg *, caddr_t, size_t, uint_t);
171 static void segdev_badop(void);
172 static int segdev_sync(struct seg *, caddr_t, size_t, int, uint_t);
173 static size_t segdev_incore(struct seg *, caddr_t, size_t, char *);
174 static int segdev_lockop(struct seg *, caddr_t, size_t, int, int,
175 ulong_t *, size_t);
176 static int segdev_getprot(struct seg *, caddr_t, size_t, uint_t *);
177 static u_offset_t segdev_getoffset(struct seg *, caddr_t);
178 static int segdev_gettype(struct seg *, caddr_t);
179 static int segdev_getvp(struct seg *, caddr_t, struct vnode **);
180 static int segdev_advise(struct seg *, caddr_t, size_t, uint_t);
181 static void segdev_dump(struct seg *);
182 static int segdev_pagelock(struct seg *, caddr_t, size_t,
183 struct page ***, enum lock_type, enum seg_rw);
184 static int segdev_setpagesize(struct seg *, caddr_t, size_t, uint_t);
185 static int segdev_getmemid(struct seg *, caddr_t, memid_t *);
186 static lgrp_mem_policy_info_t *segdev_getpolicy(struct seg *, caddr_t);
187 static int segdev_capable(struct seg *, segcapability_t);
188
189 /*
190 * XXX this struct is used by rootnex_map_fault to identify
191 * the segment it has been passed. So if you make it
192 * "static" you'll need to fix rootnex_map_fault.
193 */
194 struct seg_ops segdev_ops = {
195 segdev_dup,
196 segdev_unmap,
197 segdev_free,
198 segdev_fault,
199 segdev_faulta,
200 segdev_setprot,
201 segdev_checkprot,
202 (int (*)())segdev_badop, /* kluster */
203 (size_t (*)(struct seg *))NULL, /* swapout */
204 segdev_sync, /* sync */
205 segdev_incore,
206 segdev_lockop, /* lockop */
207 segdev_getprot,
208 segdev_getoffset,
209 segdev_gettype,
210 segdev_getvp,
211 segdev_advise,
212 segdev_dump,
213 segdev_pagelock,
214 segdev_setpagesize,
215 segdev_getmemid,
216 segdev_getpolicy,
217 segdev_capable,
218 };
219
220 /*
221 * Private segdev support routines
222 */
223 static struct segdev_data *sdp_alloc(void);
224
225 static void segdev_softunlock(struct hat *, struct seg *, caddr_t,
226 size_t, enum seg_rw);
227
228 static faultcode_t segdev_faultpage(struct hat *, struct seg *, caddr_t,
229 struct vpage *, enum fault_type, enum seg_rw, devmap_handle_t *);
230
231 static faultcode_t segdev_faultpages(struct hat *, struct seg *, caddr_t,
232 size_t, enum fault_type, enum seg_rw, devmap_handle_t *);
233
234 static struct devmap_ctx *devmap_ctxinit(dev_t, ulong_t);
235 static struct devmap_softlock *devmap_softlock_init(dev_t, ulong_t);
236 static void devmap_softlock_rele(devmap_handle_t *);
237 static void devmap_ctx_rele(devmap_handle_t *);
238
239 static void devmap_ctxto(void *);
240
241 static devmap_handle_t *devmap_find_handle(devmap_handle_t *dhp_head,
242 caddr_t addr);
243
244 static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
245 ulong_t *opfn, ulong_t *pagesize);
246
247 static void free_devmap_handle(devmap_handle_t *dhp);
248
249 static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
250 struct seg *newseg);
251
252 static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp);
253
254 static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len);
255
256 static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr);
257
258 static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
259 offset_t off, size_t len, uint_t flags);
260
261 static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len,
262 caddr_t addr, size_t *llen, caddr_t *laddr);
263
264 static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len);
265
266 static void *devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag);
267 static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size);
268
269 static void *devmap_umem_alloc_np(size_t size, size_t flags);
270 static void devmap_umem_free_np(void *addr, size_t size);
271
272 /*
273 * routines to lock and unlock underlying segkp segment for
274 * KMEM_PAGEABLE type cookies.
275 */
276 static faultcode_t acquire_kpmem_lock(struct ddi_umem_cookie *, size_t);
277 static void release_kpmem_lock(struct ddi_umem_cookie *, size_t);
278
279 /*
280 * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
281 * drivers with devmap_access callbacks
282 */
283 static int devmap_softlock_enter(struct devmap_softlock *, size_t,
284 enum fault_type);
285 static void devmap_softlock_exit(struct devmap_softlock *, size_t,
286 enum fault_type);
287
288 static kmutex_t devmapctx_lock;
289
290 static kmutex_t devmap_slock;
291
292 /*
293 * Initialize the thread callbacks and thread private data.
294 */
295 static struct devmap_ctx *
devmap_ctxinit(dev_t dev,ulong_t id)296 devmap_ctxinit(dev_t dev, ulong_t id)
297 {
298 struct devmap_ctx *devctx;
299 struct devmap_ctx *tmp;
300 dev_info_t *dip;
301
302 tmp = kmem_zalloc(sizeof (struct devmap_ctx), KM_SLEEP);
303
304 mutex_enter(&devmapctx_lock);
305
306 dip = e_ddi_hold_devi_by_dev(dev, 0);
307 ASSERT(dip != NULL);
308 ddi_release_devi(dip);
309
310 for (devctx = devmapctx_list; devctx != NULL; devctx = devctx->next)
311 if ((devctx->dip == dip) && (devctx->id == id))
312 break;
313
314 if (devctx == NULL) {
315 devctx = tmp;
316 devctx->dip = dip;
317 devctx->id = id;
318 mutex_init(&devctx->lock, NULL, MUTEX_DEFAULT, NULL);
319 cv_init(&devctx->cv, NULL, CV_DEFAULT, NULL);
320 devctx->next = devmapctx_list;
321 devmapctx_list = devctx;
322 } else
323 kmem_free(tmp, sizeof (struct devmap_ctx));
324
325 mutex_enter(&devctx->lock);
326 devctx->refcnt++;
327 mutex_exit(&devctx->lock);
328 mutex_exit(&devmapctx_lock);
329
330 return (devctx);
331 }
332
333 /*
334 * Timeout callback called if a CPU has not given up the device context
335 * within dhp->dh_timeout_length ticks
336 */
337 static void
devmap_ctxto(void * data)338 devmap_ctxto(void *data)
339 {
340 struct devmap_ctx *devctx = data;
341
342 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_CTXTO,
343 "devmap_ctxto:timeout expired, devctx=%p", (void *)devctx);
344 mutex_enter(&devctx->lock);
345 /*
346 * Set oncpu = 0 so the next mapping trying to get the device context
347 * can.
348 */
349 devctx->oncpu = 0;
350 devctx->timeout = 0;
351 cv_signal(&devctx->cv);
352 mutex_exit(&devctx->lock);
353 }
354
355 /*
356 * Create a device segment.
357 */
358 int
segdev_create(struct seg * seg,void * argsp)359 segdev_create(struct seg *seg, void *argsp)
360 {
361 struct segdev_data *sdp;
362 struct segdev_crargs *a = (struct segdev_crargs *)argsp;
363 devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data;
364 int error;
365
366 /*
367 * Since the address space is "write" locked, we
368 * don't need the segment lock to protect "segdev" data.
369 */
370 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
371
372 hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP);
373
374 sdp = sdp_alloc();
375
376 sdp->mapfunc = a->mapfunc;
377 sdp->offset = a->offset;
378 sdp->prot = a->prot;
379 sdp->maxprot = a->maxprot;
380 sdp->type = a->type;
381 sdp->pageprot = 0;
382 sdp->softlockcnt = 0;
383 sdp->vpage = NULL;
384
385 if (sdp->mapfunc == NULL)
386 sdp->devmap_data = dhp;
387 else
388 sdp->devmap_data = dhp = NULL;
389
390 sdp->hat_flags = a->hat_flags;
391 sdp->hat_attr = a->hat_attr;
392
393 /*
394 * Currently, hat_flags supports only HAT_LOAD_NOCONSIST
395 */
396 ASSERT(!(sdp->hat_flags & ~HAT_LOAD_NOCONSIST));
397
398 /*
399 * Hold shadow vnode -- segdev only deals with
400 * character (VCHR) devices. We use the common
401 * vp to hang pages on.
402 */
403 sdp->vp = specfind(a->dev, VCHR);
404 ASSERT(sdp->vp != NULL);
405
406 seg->s_ops = &segdev_ops;
407 seg->s_data = sdp;
408
409 while (dhp != NULL) {
410 dhp->dh_seg = seg;
411 dhp = dhp->dh_next;
412 }
413
414 /*
415 * Inform the vnode of the new mapping.
416 */
417 /*
418 * It is ok to use pass sdp->maxprot to ADDMAP rather than to use
419 * dhp specific maxprot because spec_addmap does not use maxprot.
420 */
421 error = VOP_ADDMAP(VTOCVP(sdp->vp), sdp->offset,
422 seg->s_as, seg->s_base, seg->s_size,
423 sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
424
425 if (error != 0) {
426 sdp->devmap_data = NULL;
427 hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size,
428 HAT_UNLOAD_UNMAP);
429 } else {
430 /*
431 * Mappings of /dev/null don't count towards the VSZ of a
432 * process. Mappings of /dev/null have no mapping type.
433 */
434 if ((SEGOP_GETTYPE(seg, (seg)->s_base) & (MAP_SHARED |
435 MAP_PRIVATE)) == 0) {
436 seg->s_as->a_resvsize -= seg->s_size;
437 }
438 }
439
440 return (error);
441 }
442
443 static struct segdev_data *
sdp_alloc(void)444 sdp_alloc(void)
445 {
446 struct segdev_data *sdp;
447
448 sdp = kmem_zalloc(sizeof (struct segdev_data), KM_SLEEP);
449 rw_init(&sdp->lock, NULL, RW_DEFAULT, NULL);
450
451 return (sdp);
452 }
453
454 /*
455 * Duplicate seg and return new segment in newseg.
456 */
457 static int
segdev_dup(struct seg * seg,struct seg * newseg)458 segdev_dup(struct seg *seg, struct seg *newseg)
459 {
460 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
461 struct segdev_data *newsdp;
462 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
463 size_t npages;
464 int ret;
465
466 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DUP,
467 "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg);
468
469 DEBUGF(3, (CE_CONT, "segdev_dup: dhp %p seg %p\n",
470 (void *)dhp, (void *)seg));
471
472 /*
473 * Since the address space is "write" locked, we
474 * don't need the segment lock to protect "segdev" data.
475 */
476 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
477
478 newsdp = sdp_alloc();
479
480 newseg->s_ops = seg->s_ops;
481 newseg->s_data = (void *)newsdp;
482
483 VN_HOLD(sdp->vp);
484 newsdp->vp = sdp->vp;
485 newsdp->mapfunc = sdp->mapfunc;
486 newsdp->offset = sdp->offset;
487 newsdp->pageprot = sdp->pageprot;
488 newsdp->prot = sdp->prot;
489 newsdp->maxprot = sdp->maxprot;
490 newsdp->type = sdp->type;
491 newsdp->hat_attr = sdp->hat_attr;
492 newsdp->hat_flags = sdp->hat_flags;
493 newsdp->softlockcnt = 0;
494
495 /*
496 * Initialize per page data if the segment we are
497 * dup'ing has per page information.
498 */
499 npages = seg_pages(newseg);
500
501 if (sdp->vpage != NULL) {
502 size_t nbytes = vpgtob(npages);
503
504 newsdp->vpage = kmem_zalloc(nbytes, KM_SLEEP);
505 bcopy(sdp->vpage, newsdp->vpage, nbytes);
506 } else
507 newsdp->vpage = NULL;
508
509 /*
510 * duplicate devmap handles
511 */
512 if (dhp != NULL) {
513 ret = devmap_handle_dup(dhp,
514 (devmap_handle_t **)&newsdp->devmap_data, newseg);
515 if (ret != 0) {
516 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DUP_CK1,
517 "segdev_dup:ret1 ret=%x, dhp=%p seg=%p",
518 ret, (void *)dhp, (void *)seg);
519 DEBUGF(1, (CE_CONT,
520 "segdev_dup: ret %x dhp %p seg %p\n",
521 ret, (void *)dhp, (void *)seg));
522 return (ret);
523 }
524 }
525
526 /*
527 * Inform the common vnode of the new mapping.
528 */
529 return (VOP_ADDMAP(VTOCVP(newsdp->vp),
530 newsdp->offset, newseg->s_as,
531 newseg->s_base, newseg->s_size, newsdp->prot,
532 newsdp->maxprot, sdp->type, CRED(), NULL));
533 }
534
535 /*
536 * duplicate devmap handles
537 */
538 static int
devmap_handle_dup(devmap_handle_t * dhp,devmap_handle_t ** new_dhp,struct seg * newseg)539 devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
540 struct seg *newseg)
541 {
542 devmap_handle_t *newdhp_save = NULL;
543 devmap_handle_t *newdhp = NULL;
544 struct devmap_callback_ctl *callbackops;
545
546 while (dhp != NULL) {
547 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
548
549 /* Need to lock the original dhp while copying if REMAP */
550 HOLD_DHP_LOCK(dhp);
551 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
552 RELE_DHP_LOCK(dhp);
553 newdhp->dh_seg = newseg;
554 newdhp->dh_next = NULL;
555 if (newdhp_save != NULL)
556 newdhp_save->dh_next = newdhp;
557 else
558 *new_dhp = newdhp;
559 newdhp_save = newdhp;
560
561 callbackops = &newdhp->dh_callbackops;
562
563 if (dhp->dh_softlock != NULL)
564 newdhp->dh_softlock = devmap_softlock_init(
565 newdhp->dh_dev,
566 (ulong_t)callbackops->devmap_access);
567 if (dhp->dh_ctx != NULL)
568 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
569 (ulong_t)callbackops->devmap_access);
570
571 /*
572 * Initialize dh_lock if we want to do remap.
573 */
574 if (newdhp->dh_flags & DEVMAP_ALLOW_REMAP) {
575 mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
576 newdhp->dh_flags |= DEVMAP_LOCK_INITED;
577 }
578
579 if (callbackops->devmap_dup != NULL) {
580 int ret;
581
582 /*
583 * Call the dup callback so that the driver can
584 * duplicate its private data.
585 */
586 ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp,
587 (devmap_cookie_t *)newdhp, &newdhp->dh_pvtp);
588
589 if (ret != 0) {
590 /*
591 * We want to free up this segment as the driver
592 * has indicated that we can't dup it. But we
593 * don't want to call the drivers, devmap_unmap,
594 * callback function as the driver does not
595 * think this segment exists. The caller of
596 * devmap_dup will call seg_free on newseg
597 * as it was the caller that allocated the
598 * segment.
599 */
600 DEBUGF(1, (CE_CONT, "devmap_handle_dup ERROR: "
601 "newdhp %p dhp %p\n", (void *)newdhp,
602 (void *)dhp));
603 callbackops->devmap_unmap = NULL;
604 return (ret);
605 }
606 }
607
608 dhp = dhp->dh_next;
609 }
610
611 return (0);
612 }
613
614 /*
615 * Split a segment at addr for length len.
616 */
617 /*ARGSUSED*/
618 static int
segdev_unmap(struct seg * seg,caddr_t addr,size_t len)619 segdev_unmap(struct seg *seg, caddr_t addr, size_t len)
620 {
621 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
622 register struct segdev_data *nsdp;
623 register struct seg *nseg;
624 register size_t opages; /* old segment size in pages */
625 register size_t npages; /* new segment size in pages */
626 register size_t dpages; /* pages being deleted (unmapped) */
627 register size_t nbytes;
628 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
629 devmap_handle_t *dhpp;
630 devmap_handle_t *newdhp;
631 struct devmap_callback_ctl *callbackops;
632 caddr_t nbase;
633 offset_t off;
634 ulong_t nsize;
635 size_t mlen, sz;
636
637 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP,
638 "segdev_unmap:start dhp=%p, seg=%p addr=%p len=%lx",
639 (void *)dhp, (void *)seg, (void *)addr, len);
640
641 DEBUGF(3, (CE_CONT, "segdev_unmap: dhp %p seg %p addr %p len %lx\n",
642 (void *)dhp, (void *)seg, (void *)addr, len));
643
644 /*
645 * Since the address space is "write" locked, we
646 * don't need the segment lock to protect "segdev" data.
647 */
648 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
649
650 if ((sz = sdp->softlockcnt) > 0) {
651 /*
652 * Fail the unmap if pages are SOFTLOCKed through this mapping.
653 * softlockcnt is protected from change by the as write lock.
654 */
655 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK1,
656 "segdev_unmap:error softlockcnt = %ld", sz);
657 DEBUGF(1, (CE_CONT, "segdev_unmap: softlockcnt %ld\n", sz));
658 return (EAGAIN);
659 }
660
661 /*
662 * Check for bad sizes
663 */
664 if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
665 (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET))
666 panic("segdev_unmap");
667
668 if (dhp != NULL) {
669 devmap_handle_t *tdhp;
670 /*
671 * If large page size was used in hat_devload(),
672 * the same page size must be used in hat_unload().
673 */
674 dhpp = tdhp = devmap_find_handle(dhp, addr);
675 while (tdhp != NULL) {
676 if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
677 break;
678 }
679 tdhp = tdhp->dh_next;
680 }
681 if (tdhp != NULL) { /* found a dhp using large pages */
682 size_t slen = len;
683 size_t mlen;
684 size_t soff;
685
686 soff = (ulong_t)(addr - dhpp->dh_uvaddr);
687 while (slen != 0) {
688 mlen = MIN(slen, (dhpp->dh_len - soff));
689 hat_unload(seg->s_as->a_hat, dhpp->dh_uvaddr,
690 dhpp->dh_len, HAT_UNLOAD_UNMAP);
691 dhpp = dhpp->dh_next;
692 ASSERT(slen >= mlen);
693 slen -= mlen;
694 soff = 0;
695 }
696 } else
697 hat_unload(seg->s_as->a_hat, addr, len,
698 HAT_UNLOAD_UNMAP);
699 } else {
700 /*
701 * Unload any hardware translations in the range
702 * to be taken out.
703 */
704 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
705 }
706
707 /*
708 * get the user offset which will used in the driver callbacks
709 */
710 off = sdp->offset + (offset_t)(addr - seg->s_base);
711
712 /*
713 * Inform the vnode of the unmapping.
714 */
715 ASSERT(sdp->vp != NULL);
716 (void) VOP_DELMAP(VTOCVP(sdp->vp), off, seg->s_as, addr, len,
717 sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
718
719 /*
720 * Check for entire segment
721 */
722 if (addr == seg->s_base && len == seg->s_size) {
723 seg_free(seg);
724 return (0);
725 }
726
727 opages = seg_pages(seg);
728 dpages = btop(len);
729 npages = opages - dpages;
730
731 /*
732 * Check for beginning of segment
733 */
734 if (addr == seg->s_base) {
735 if (sdp->vpage != NULL) {
736 register struct vpage *ovpage;
737
738 ovpage = sdp->vpage; /* keep pointer to vpage */
739
740 nbytes = vpgtob(npages);
741 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
742 bcopy(&ovpage[dpages], sdp->vpage, nbytes);
743
744 /* free up old vpage */
745 kmem_free(ovpage, vpgtob(opages));
746 }
747
748 /*
749 * free devmap handles from the beginning of the mapping.
750 */
751 if (dhp != NULL)
752 devmap_handle_unmap_head(dhp, len);
753
754 sdp->offset += (offset_t)len;
755
756 seg->s_base += len;
757 seg->s_size -= len;
758
759 return (0);
760 }
761
762 /*
763 * Check for end of segment
764 */
765 if (addr + len == seg->s_base + seg->s_size) {
766 if (sdp->vpage != NULL) {
767 register struct vpage *ovpage;
768
769 ovpage = sdp->vpage; /* keep pointer to vpage */
770
771 nbytes = vpgtob(npages);
772 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
773 bcopy(ovpage, sdp->vpage, nbytes);
774
775 /* free up old vpage */
776 kmem_free(ovpage, vpgtob(opages));
777 }
778 seg->s_size -= len;
779
780 /*
781 * free devmap handles from addr to the end of the mapping.
782 */
783 if (dhp != NULL)
784 devmap_handle_unmap_tail(dhp, addr);
785
786 return (0);
787 }
788
789 /*
790 * The section to go is in the middle of the segment,
791 * have to make it into two segments. nseg is made for
792 * the high end while seg is cut down at the low end.
793 */
794 nbase = addr + len; /* new seg base */
795 nsize = (seg->s_base + seg->s_size) - nbase; /* new seg size */
796 seg->s_size = addr - seg->s_base; /* shrink old seg */
797 nseg = seg_alloc(seg->s_as, nbase, nsize);
798 if (nseg == NULL)
799 panic("segdev_unmap seg_alloc");
800
801 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK2,
802 "segdev_unmap: seg=%p nseg=%p", (void *)seg, (void *)nseg);
803 DEBUGF(3, (CE_CONT, "segdev_unmap: segdev_dup seg %p nseg %p\n",
804 (void *)seg, (void *)nseg));
805 nsdp = sdp_alloc();
806
807 nseg->s_ops = seg->s_ops;
808 nseg->s_data = (void *)nsdp;
809
810 VN_HOLD(sdp->vp);
811 nsdp->mapfunc = sdp->mapfunc;
812 nsdp->offset = sdp->offset + (offset_t)(nseg->s_base - seg->s_base);
813 nsdp->vp = sdp->vp;
814 nsdp->pageprot = sdp->pageprot;
815 nsdp->prot = sdp->prot;
816 nsdp->maxprot = sdp->maxprot;
817 nsdp->type = sdp->type;
818 nsdp->hat_attr = sdp->hat_attr;
819 nsdp->hat_flags = sdp->hat_flags;
820 nsdp->softlockcnt = 0;
821
822 /*
823 * Initialize per page data if the segment we are
824 * dup'ing has per page information.
825 */
826 if (sdp->vpage != NULL) {
827 /* need to split vpage into two arrays */
828 register size_t nnbytes;
829 register size_t nnpages;
830 register struct vpage *ovpage;
831
832 ovpage = sdp->vpage; /* keep pointer to vpage */
833
834 npages = seg_pages(seg); /* seg has shrunk */
835 nbytes = vpgtob(npages);
836 nnpages = seg_pages(nseg);
837 nnbytes = vpgtob(nnpages);
838
839 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
840 bcopy(ovpage, sdp->vpage, nbytes);
841
842 nsdp->vpage = kmem_alloc(nnbytes, KM_SLEEP);
843 bcopy(&ovpage[npages + dpages], nsdp->vpage, nnbytes);
844
845 /* free up old vpage */
846 kmem_free(ovpage, vpgtob(opages));
847 } else
848 nsdp->vpage = NULL;
849
850 /*
851 * unmap dhps.
852 */
853 if (dhp == NULL) {
854 nsdp->devmap_data = NULL;
855 return (0);
856 }
857 while (dhp != NULL) {
858 callbackops = &dhp->dh_callbackops;
859 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK3,
860 "segdev_unmap: dhp=%p addr=%p", dhp, addr);
861 DEBUGF(3, (CE_CONT, "unmap: dhp %p addr %p uvaddr %p len %lx\n",
862 (void *)dhp, (void *)addr,
863 (void *)dhp->dh_uvaddr, dhp->dh_len));
864
865 if (addr == (dhp->dh_uvaddr + dhp->dh_len)) {
866 dhpp = dhp->dh_next;
867 dhp->dh_next = NULL;
868 dhp = dhpp;
869 } else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) {
870 dhp = dhp->dh_next;
871 } else if (addr > dhp->dh_uvaddr &&
872 (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) {
873 /*
874 * <addr, addr+len> is enclosed by dhp.
875 * create a newdhp that begins at addr+len and
876 * ends at dhp->dh_uvaddr+dhp->dh_len.
877 */
878 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
879 HOLD_DHP_LOCK(dhp);
880 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
881 RELE_DHP_LOCK(dhp);
882 newdhp->dh_seg = nseg;
883 newdhp->dh_next = dhp->dh_next;
884 if (dhp->dh_softlock != NULL)
885 newdhp->dh_softlock = devmap_softlock_init(
886 newdhp->dh_dev,
887 (ulong_t)callbackops->devmap_access);
888 if (dhp->dh_ctx != NULL)
889 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
890 (ulong_t)callbackops->devmap_access);
891 if (newdhp->dh_flags & DEVMAP_LOCK_INITED) {
892 mutex_init(&newdhp->dh_lock,
893 NULL, MUTEX_DEFAULT, NULL);
894 }
895 if (callbackops->devmap_unmap != NULL)
896 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
897 off, len, dhp, &dhp->dh_pvtp,
898 newdhp, &newdhp->dh_pvtp);
899 mlen = len + (addr - dhp->dh_uvaddr);
900 devmap_handle_reduce_len(newdhp, mlen);
901 nsdp->devmap_data = newdhp;
902 /* XX Changing len should recalculate LARGE flag */
903 dhp->dh_len = addr - dhp->dh_uvaddr;
904 dhpp = dhp->dh_next;
905 dhp->dh_next = NULL;
906 dhp = dhpp;
907 } else if ((addr > dhp->dh_uvaddr) &&
908 ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) {
909 mlen = dhp->dh_len + dhp->dh_uvaddr - addr;
910 /*
911 * <addr, addr+len> spans over dhps.
912 */
913 if (callbackops->devmap_unmap != NULL)
914 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
915 off, mlen, (devmap_cookie_t *)dhp,
916 &dhp->dh_pvtp, NULL, NULL);
917 /* XX Changing len should recalculate LARGE flag */
918 dhp->dh_len = addr - dhp->dh_uvaddr;
919 dhpp = dhp->dh_next;
920 dhp->dh_next = NULL;
921 dhp = dhpp;
922 nsdp->devmap_data = dhp;
923 } else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) {
924 /*
925 * dhp is enclosed by <addr, addr+len>.
926 */
927 dhp->dh_seg = nseg;
928 nsdp->devmap_data = dhp;
929 dhp = devmap_handle_unmap(dhp);
930 nsdp->devmap_data = dhp; /* XX redundant? */
931 } else if (((addr + len) > dhp->dh_uvaddr) &&
932 ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) {
933 mlen = addr + len - dhp->dh_uvaddr;
934 if (callbackops->devmap_unmap != NULL)
935 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
936 dhp->dh_uoff, mlen, NULL,
937 NULL, dhp, &dhp->dh_pvtp);
938 devmap_handle_reduce_len(dhp, mlen);
939 nsdp->devmap_data = dhp;
940 dhp->dh_seg = nseg;
941 dhp = dhp->dh_next;
942 } else {
943 dhp->dh_seg = nseg;
944 dhp = dhp->dh_next;
945 }
946 }
947 return (0);
948 }
949
950 /*
951 * Utility function handles reducing the length of a devmap handle during unmap
952 * Note that is only used for unmapping the front portion of the handler,
953 * i.e., we are bumping up the offset/pfn etc up by len
954 * Do not use if reducing length at the tail.
955 */
956 static void
devmap_handle_reduce_len(devmap_handle_t * dhp,size_t len)957 devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len)
958 {
959 struct ddi_umem_cookie *cp;
960 struct devmap_pmem_cookie *pcp;
961 /*
962 * adjust devmap handle fields
963 */
964 ASSERT(len < dhp->dh_len);
965
966 /* Make sure only page-aligned changes are done */
967 ASSERT((len & PAGEOFFSET) == 0);
968
969 dhp->dh_len -= len;
970 dhp->dh_uoff += (offset_t)len;
971 dhp->dh_roff += (offset_t)len;
972 dhp->dh_uvaddr += len;
973 /* Need to grab dhp lock if REMAP */
974 HOLD_DHP_LOCK(dhp);
975 cp = dhp->dh_cookie;
976 if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) {
977 if (cookie_is_devmem(cp)) {
978 dhp->dh_pfn += btop(len);
979 } else if (cookie_is_pmem(cp)) {
980 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
981 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
982 dhp->dh_roff < ptob(pcp->dp_npages));
983 } else {
984 ASSERT(dhp->dh_roff < cp->size);
985 ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
986 dhp->dh_cvaddr < (cp->cvaddr + cp->size));
987 ASSERT((dhp->dh_cvaddr + len) <=
988 (cp->cvaddr + cp->size));
989
990 dhp->dh_cvaddr += len;
991 }
992 }
993 /* XXX - Should recalculate the DEVMAP_FLAG_LARGE after changes */
994 RELE_DHP_LOCK(dhp);
995 }
996
997 /*
998 * Free devmap handle, dhp.
999 * Return the next devmap handle on the linked list.
1000 */
1001 static devmap_handle_t *
devmap_handle_unmap(devmap_handle_t * dhp)1002 devmap_handle_unmap(devmap_handle_t *dhp)
1003 {
1004 struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops;
1005 struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data;
1006 devmap_handle_t *dhpp = (devmap_handle_t *)sdp->devmap_data;
1007
1008 ASSERT(dhp != NULL);
1009
1010 /*
1011 * before we free up dhp, call the driver's devmap_unmap entry point
1012 * to free resources allocated for this dhp.
1013 */
1014 if (callbackops->devmap_unmap != NULL) {
1015 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff,
1016 dhp->dh_len, NULL, NULL, NULL, NULL);
1017 }
1018
1019 if (dhpp == dhp) { /* releasing first dhp, change sdp data */
1020 sdp->devmap_data = dhp->dh_next;
1021 } else {
1022 while (dhpp->dh_next != dhp) {
1023 dhpp = dhpp->dh_next;
1024 }
1025 dhpp->dh_next = dhp->dh_next;
1026 }
1027 dhpp = dhp->dh_next; /* return value is next dhp in chain */
1028
1029 if (dhp->dh_softlock != NULL)
1030 devmap_softlock_rele(dhp);
1031
1032 if (dhp->dh_ctx != NULL)
1033 devmap_ctx_rele(dhp);
1034
1035 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1036 mutex_destroy(&dhp->dh_lock);
1037 }
1038 kmem_free(dhp, sizeof (devmap_handle_t));
1039
1040 return (dhpp);
1041 }
1042
1043 /*
1044 * Free complete devmap handles from dhp for len bytes
1045 * dhp can be either the first handle or a subsequent handle
1046 */
1047 static void
devmap_handle_unmap_head(devmap_handle_t * dhp,size_t len)1048 devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len)
1049 {
1050 struct devmap_callback_ctl *callbackops;
1051
1052 /*
1053 * free the devmap handles covered by len.
1054 */
1055 while (len >= dhp->dh_len) {
1056 len -= dhp->dh_len;
1057 dhp = devmap_handle_unmap(dhp);
1058 }
1059 if (len != 0) { /* partial unmap at head of first remaining dhp */
1060 callbackops = &dhp->dh_callbackops;
1061
1062 /*
1063 * Call the unmap callback so the drivers can make
1064 * adjustment on its private data.
1065 */
1066 if (callbackops->devmap_unmap != NULL)
1067 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
1068 dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp);
1069 devmap_handle_reduce_len(dhp, len);
1070 }
1071 }
1072
1073 /*
1074 * Free devmap handles to truncate the mapping after addr
1075 * RFE: Simpler to pass in dhp pointing at correct dhp (avoid find again)
1076 * Also could then use the routine in middle unmap case too
1077 */
1078 static void
devmap_handle_unmap_tail(devmap_handle_t * dhp,caddr_t addr)1079 devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr)
1080 {
1081 register struct seg *seg = dhp->dh_seg;
1082 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1083 register devmap_handle_t *dhph = (devmap_handle_t *)sdp->devmap_data;
1084 struct devmap_callback_ctl *callbackops;
1085 register devmap_handle_t *dhpp;
1086 size_t maplen;
1087 ulong_t off;
1088 size_t len;
1089
1090 maplen = (size_t)(addr - dhp->dh_uvaddr);
1091 dhph = devmap_find_handle(dhph, addr);
1092
1093 while (dhph != NULL) {
1094 if (maplen == 0) {
1095 dhph = devmap_handle_unmap(dhph);
1096 } else {
1097 callbackops = &dhph->dh_callbackops;
1098 len = dhph->dh_len - maplen;
1099 off = (ulong_t)sdp->offset + (addr - seg->s_base);
1100 /*
1101 * Call the unmap callback so the driver
1102 * can make adjustments on its private data.
1103 */
1104 if (callbackops->devmap_unmap != NULL)
1105 (*callbackops->devmap_unmap)(dhph,
1106 dhph->dh_pvtp, off, len,
1107 (devmap_cookie_t *)dhph,
1108 &dhph->dh_pvtp, NULL, NULL);
1109 /* XXX Reducing len needs to recalculate LARGE flag */
1110 dhph->dh_len = maplen;
1111 maplen = 0;
1112 dhpp = dhph->dh_next;
1113 dhph->dh_next = NULL;
1114 dhph = dhpp;
1115 }
1116 } /* end while */
1117 }
1118
1119 /*
1120 * Free a segment.
1121 */
1122 static void
segdev_free(struct seg * seg)1123 segdev_free(struct seg *seg)
1124 {
1125 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1126 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
1127
1128 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FREE,
1129 "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg);
1130 DEBUGF(3, (CE_CONT, "segdev_free: dhp %p seg %p\n",
1131 (void *)dhp, (void *)seg));
1132
1133 /*
1134 * Since the address space is "write" locked, we
1135 * don't need the segment lock to protect "segdev" data.
1136 */
1137 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
1138
1139 while (dhp != NULL)
1140 dhp = devmap_handle_unmap(dhp);
1141
1142 VN_RELE(sdp->vp);
1143 if (sdp->vpage != NULL)
1144 kmem_free(sdp->vpage, vpgtob(seg_pages(seg)));
1145
1146 rw_destroy(&sdp->lock);
1147 kmem_free(sdp, sizeof (*sdp));
1148 }
1149
1150 static void
free_devmap_handle(devmap_handle_t * dhp)1151 free_devmap_handle(devmap_handle_t *dhp)
1152 {
1153 register devmap_handle_t *dhpp;
1154
1155 /*
1156 * free up devmap handle
1157 */
1158 while (dhp != NULL) {
1159 dhpp = dhp->dh_next;
1160 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1161 mutex_destroy(&dhp->dh_lock);
1162 }
1163
1164 if (dhp->dh_softlock != NULL)
1165 devmap_softlock_rele(dhp);
1166
1167 if (dhp->dh_ctx != NULL)
1168 devmap_ctx_rele(dhp);
1169
1170 kmem_free(dhp, sizeof (devmap_handle_t));
1171 dhp = dhpp;
1172 }
1173 }
1174
1175 /*
1176 * routines to lock and unlock underlying segkp segment for
1177 * KMEM_PAGEABLE type cookies.
1178 * segkp only allows a single pending F_SOFTLOCK
1179 * we keep track of number of locks in the cookie so we can
1180 * have multiple pending faults and manage the calls to segkp.
1181 * RFE: if segkp supports either pagelock or can support multiple
1182 * calls to F_SOFTLOCK, then these routines can go away.
1183 * If pagelock, segdev_faultpage can fault on a page by page basis
1184 * and simplifies the code quite a bit.
1185 * if multiple calls allowed but not partial ranges, then need for
1186 * cookie->lock and locked count goes away, code can call as_fault directly
1187 */
1188 static faultcode_t
acquire_kpmem_lock(struct ddi_umem_cookie * cookie,size_t npages)1189 acquire_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1190 {
1191 int err = 0;
1192 ASSERT(cookie_is_kpmem(cookie));
1193 /*
1194 * Fault in pages in segkp with F_SOFTLOCK.
1195 * We want to hold the lock until all pages have been loaded.
1196 * segkp only allows single caller to hold SOFTLOCK, so cookie
1197 * holds a count so we dont call into segkp multiple times
1198 */
1199 mutex_enter(&cookie->lock);
1200
1201 /*
1202 * Check for overflow in locked field
1203 */
1204 if ((UINT32_MAX - cookie->locked) < npages) {
1205 err = FC_MAKE_ERR(ENOMEM);
1206 } else if (cookie->locked == 0) {
1207 /* First time locking */
1208 err = as_fault(kas.a_hat, &kas, cookie->cvaddr,
1209 cookie->size, F_SOFTLOCK, PROT_READ|PROT_WRITE);
1210 }
1211 if (!err) {
1212 cookie->locked += npages;
1213 }
1214 mutex_exit(&cookie->lock);
1215 return (err);
1216 }
1217
1218 static void
release_kpmem_lock(struct ddi_umem_cookie * cookie,size_t npages)1219 release_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1220 {
1221 mutex_enter(&cookie->lock);
1222 ASSERT(cookie_is_kpmem(cookie));
1223 ASSERT(cookie->locked >= npages);
1224 cookie->locked -= (uint_t)npages;
1225 if (cookie->locked == 0) {
1226 /* Last unlock */
1227 if (as_fault(kas.a_hat, &kas, cookie->cvaddr,
1228 cookie->size, F_SOFTUNLOCK, PROT_READ|PROT_WRITE))
1229 panic("segdev releasing kpmem lock %p", (void *)cookie);
1230 }
1231 mutex_exit(&cookie->lock);
1232 }
1233
1234 /*
1235 * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
1236 * drivers with devmap_access callbacks
1237 * slock->softlocked basically works like a rw lock
1238 * -ve counts => F_SOFTLOCK in progress
1239 * +ve counts => F_INVAL/F_PROT in progress
1240 * We allow only one F_SOFTLOCK at a time
1241 * but can have multiple pending F_INVAL/F_PROT calls
1242 *
1243 * This routine waits using cv_wait_sig so killing processes is more graceful
1244 * Returns EINTR if coming out of this routine due to a signal, 0 otherwise
1245 */
devmap_softlock_enter(struct devmap_softlock * slock,size_t npages,enum fault_type type)1246 static int devmap_softlock_enter(
1247 struct devmap_softlock *slock,
1248 size_t npages,
1249 enum fault_type type)
1250 {
1251 if (npages == 0)
1252 return (0);
1253 mutex_enter(&(slock->lock));
1254 switch (type) {
1255 case F_SOFTLOCK :
1256 while (slock->softlocked) {
1257 if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1258 /* signalled */
1259 mutex_exit(&(slock->lock));
1260 return (EINTR);
1261 }
1262 }
1263 slock->softlocked -= npages; /* -ve count => locked */
1264 break;
1265 case F_INVAL :
1266 case F_PROT :
1267 while (slock->softlocked < 0)
1268 if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1269 /* signalled */
1270 mutex_exit(&(slock->lock));
1271 return (EINTR);
1272 }
1273 slock->softlocked += npages; /* +ve count => f_invals */
1274 break;
1275 default:
1276 ASSERT(0);
1277 }
1278 mutex_exit(&(slock->lock));
1279 return (0);
1280 }
1281
devmap_softlock_exit(struct devmap_softlock * slock,size_t npages,enum fault_type type)1282 static void devmap_softlock_exit(
1283 struct devmap_softlock *slock,
1284 size_t npages,
1285 enum fault_type type)
1286 {
1287 if (slock == NULL)
1288 return;
1289 mutex_enter(&(slock->lock));
1290 switch (type) {
1291 case F_SOFTLOCK :
1292 ASSERT(-slock->softlocked >= npages);
1293 slock->softlocked += npages; /* -ve count is softlocked */
1294 if (slock->softlocked == 0)
1295 cv_signal(&slock->cv);
1296 break;
1297 case F_INVAL :
1298 case F_PROT:
1299 ASSERT(slock->softlocked >= npages);
1300 slock->softlocked -= npages;
1301 if (slock->softlocked == 0)
1302 cv_signal(&slock->cv);
1303 break;
1304 default:
1305 ASSERT(0);
1306 }
1307 mutex_exit(&(slock->lock));
1308 }
1309
1310 /*
1311 * Do a F_SOFTUNLOCK call over the range requested.
1312 * The range must have already been F_SOFTLOCK'ed.
1313 * The segment lock should be held, (but not the segment private lock?)
1314 * The softunlock code below does not adjust for large page sizes
1315 * assumes the caller already did any addr/len adjustments for
1316 * pagesize mappings before calling.
1317 */
1318 /*ARGSUSED*/
1319 static void
segdev_softunlock(struct hat * hat,struct seg * seg,caddr_t addr,size_t len,enum seg_rw rw)1320 segdev_softunlock(
1321 struct hat *hat, /* the hat */
1322 struct seg *seg, /* seg_dev of interest */
1323 caddr_t addr, /* base address of range */
1324 size_t len, /* number of bytes */
1325 enum seg_rw rw) /* type of access at fault */
1326 {
1327 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1328 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1329
1330 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SOFTUNLOCK,
1331 "segdev_softunlock:dhp_head=%p sdp=%p addr=%p len=%lx",
1332 dhp_head, sdp, addr, len);
1333 DEBUGF(3, (CE_CONT, "segdev_softunlock: dhp %p lockcnt %lx "
1334 "addr %p len %lx\n",
1335 (void *)dhp_head, sdp->softlockcnt, (void *)addr, len));
1336
1337 hat_unlock(hat, addr, len);
1338
1339 if (dhp_head != NULL) {
1340 devmap_handle_t *dhp;
1341 size_t mlen;
1342 size_t tlen = len;
1343 ulong_t off;
1344
1345 dhp = devmap_find_handle(dhp_head, addr);
1346 ASSERT(dhp != NULL);
1347
1348 off = (ulong_t)(addr - dhp->dh_uvaddr);
1349 while (tlen != 0) {
1350 mlen = MIN(tlen, (dhp->dh_len - off));
1351
1352 /*
1353 * unlock segkp memory, locked during F_SOFTLOCK
1354 */
1355 if (dhp_is_kpmem(dhp)) {
1356 release_kpmem_lock(
1357 (struct ddi_umem_cookie *)dhp->dh_cookie,
1358 btopr(mlen));
1359 }
1360
1361 /*
1362 * Do the softlock accounting for devmap_access
1363 */
1364 if (dhp->dh_callbackops.devmap_access != NULL) {
1365 devmap_softlock_exit(dhp->dh_softlock,
1366 btopr(mlen), F_SOFTLOCK);
1367 }
1368
1369 tlen -= mlen;
1370 dhp = dhp->dh_next;
1371 off = 0;
1372 }
1373 }
1374
1375 mutex_enter(&freemem_lock);
1376 ASSERT(sdp->softlockcnt >= btopr(len));
1377 sdp->softlockcnt -= btopr(len);
1378 mutex_exit(&freemem_lock);
1379 if (sdp->softlockcnt == 0) {
1380 /*
1381 * All SOFTLOCKS are gone. Wakeup any waiting
1382 * unmappers so they can try again to unmap.
1383 * Check for waiters first without the mutex
1384 * held so we don't always grab the mutex on
1385 * softunlocks.
1386 */
1387 if (AS_ISUNMAPWAIT(seg->s_as)) {
1388 mutex_enter(&seg->s_as->a_contents);
1389 if (AS_ISUNMAPWAIT(seg->s_as)) {
1390 AS_CLRUNMAPWAIT(seg->s_as);
1391 cv_broadcast(&seg->s_as->a_cv);
1392 }
1393 mutex_exit(&seg->s_as->a_contents);
1394 }
1395 }
1396
1397 }
1398
1399 /*
1400 * Handle fault for a single page.
1401 * Done in a separate routine so we can handle errors more easily.
1402 * This routine is called only from segdev_faultpages()
1403 * when looping over the range of addresses requested. The segment lock is held.
1404 */
1405 static faultcode_t
segdev_faultpage(struct hat * hat,struct seg * seg,caddr_t addr,struct vpage * vpage,enum fault_type type,enum seg_rw rw,devmap_handle_t * dhp)1406 segdev_faultpage(
1407 struct hat *hat, /* the hat */
1408 struct seg *seg, /* seg_dev of interest */
1409 caddr_t addr, /* address in as */
1410 struct vpage *vpage, /* pointer to vpage for seg, addr */
1411 enum fault_type type, /* type of fault */
1412 enum seg_rw rw, /* type of access at fault */
1413 devmap_handle_t *dhp) /* devmap handle if any for this page */
1414 {
1415 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1416 uint_t prot;
1417 pfn_t pfnum = PFN_INVALID;
1418 u_offset_t offset;
1419 uint_t hat_flags;
1420 dev_info_t *dip;
1421
1422 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE,
1423 "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr);
1424 DEBUGF(8, (CE_CONT, "segdev_faultpage: dhp %p seg %p addr %p \n",
1425 (void *)dhp, (void *)seg, (void *)addr));
1426
1427 /*
1428 * Initialize protection value for this page.
1429 * If we have per page protection values check it now.
1430 */
1431 if (sdp->pageprot) {
1432 uint_t protchk;
1433
1434 switch (rw) {
1435 case S_READ:
1436 protchk = PROT_READ;
1437 break;
1438 case S_WRITE:
1439 protchk = PROT_WRITE;
1440 break;
1441 case S_EXEC:
1442 protchk = PROT_EXEC;
1443 break;
1444 case S_OTHER:
1445 default:
1446 protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1447 break;
1448 }
1449
1450 prot = VPP_PROT(vpage);
1451 if ((prot & protchk) == 0)
1452 return (FC_PROT); /* illegal access type */
1453 } else {
1454 prot = sdp->prot;
1455 /* caller has already done segment level protection check */
1456 }
1457
1458 if (type == F_SOFTLOCK) {
1459 mutex_enter(&freemem_lock);
1460 sdp->softlockcnt++;
1461 mutex_exit(&freemem_lock);
1462 }
1463
1464 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
1465 offset = sdp->offset + (u_offset_t)(addr - seg->s_base);
1466 /*
1467 * In the devmap framework, sdp->mapfunc is set to NULL. we can get
1468 * pfnum from dhp->dh_pfn (at beginning of segment) and offset from
1469 * seg->s_base.
1470 */
1471 if (dhp == NULL) {
1472 /* If segment has devmap_data, then dhp should be non-NULL */
1473 ASSERT(sdp->devmap_data == NULL);
1474 pfnum = (pfn_t)cdev_mmap(sdp->mapfunc, sdp->vp->v_rdev,
1475 (off_t)offset, prot);
1476 prot |= sdp->hat_attr;
1477 } else {
1478 ulong_t off;
1479 struct ddi_umem_cookie *cp;
1480 struct devmap_pmem_cookie *pcp;
1481
1482 /* ensure the dhp passed in contains addr. */
1483 ASSERT(dhp == devmap_find_handle(
1484 (devmap_handle_t *)sdp->devmap_data, addr));
1485
1486 off = addr - dhp->dh_uvaddr;
1487
1488 /*
1489 * This routine assumes that the caller makes sure that the
1490 * fields in dhp used below are unchanged due to remap during
1491 * this call. Caller does HOLD_DHP_LOCK if neeed
1492 */
1493 cp = dhp->dh_cookie;
1494 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1495 pfnum = PFN_INVALID;
1496 } else if (cookie_is_devmem(cp)) {
1497 pfnum = dhp->dh_pfn + btop(off);
1498 } else if (cookie_is_pmem(cp)) {
1499 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
1500 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
1501 dhp->dh_roff < ptob(pcp->dp_npages));
1502 pfnum = page_pptonum(
1503 pcp->dp_pparray[btop(off + dhp->dh_roff)]);
1504 } else {
1505 ASSERT(dhp->dh_roff < cp->size);
1506 ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
1507 dhp->dh_cvaddr < (cp->cvaddr + cp->size));
1508 ASSERT((dhp->dh_cvaddr + off) <=
1509 (cp->cvaddr + cp->size));
1510 ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <=
1511 (cp->cvaddr + cp->size));
1512
1513 switch (cp->type) {
1514 case UMEM_LOCKED :
1515 if (cp->pparray != NULL) {
1516 ASSERT((dhp->dh_roff &
1517 PAGEOFFSET) == 0);
1518 pfnum = page_pptonum(
1519 cp->pparray[btop(off +
1520 dhp->dh_roff)]);
1521 } else {
1522 pfnum = hat_getpfnum(
1523 ((proc_t *)cp->procp)->p_as->a_hat,
1524 cp->cvaddr + off);
1525 }
1526 break;
1527 case UMEM_TRASH :
1528 pfnum = page_pptonum(trashpp);
1529 /*
1530 * We should set hat_flags to HAT_NOFAULT also
1531 * However, not all hat layers implement this
1532 */
1533 break;
1534 case KMEM_PAGEABLE:
1535 case KMEM_NON_PAGEABLE:
1536 pfnum = hat_getpfnum(kas.a_hat,
1537 dhp->dh_cvaddr + off);
1538 break;
1539 default :
1540 pfnum = PFN_INVALID;
1541 break;
1542 }
1543 }
1544 prot |= dhp->dh_hat_attr;
1545 }
1546 if (pfnum == PFN_INVALID) {
1547 return (FC_MAKE_ERR(EFAULT));
1548 }
1549 /* prot should already be OR'ed in with hat_attributes if needed */
1550
1551 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE_CK1,
1552 "segdev_faultpage: pfnum=%lx memory=%x prot=%x flags=%x",
1553 pfnum, pf_is_memory(pfnum), prot, hat_flags);
1554 DEBUGF(9, (CE_CONT, "segdev_faultpage: pfnum %lx memory %x "
1555 "prot %x flags %x\n", pfnum, pf_is_memory(pfnum), prot, hat_flags));
1556
1557 if (pf_is_memory(pfnum) || (dhp != NULL)) {
1558 /*
1559 * It's not _really_ required here to pass sdp->hat_flags
1560 * to hat_devload even though we do it.
1561 * This is because hat figures it out DEVMEM mappings
1562 * are non-consistent, anyway.
1563 */
1564 hat_devload(hat, addr, PAGESIZE, pfnum,
1565 prot, hat_flags | sdp->hat_flags);
1566 return (0);
1567 }
1568
1569 /*
1570 * Fall through to the case where devmap is not used and need to call
1571 * up the device tree to set up the mapping
1572 */
1573
1574 dip = VTOS(VTOCVP(sdp->vp))->s_dip;
1575 ASSERT(dip);
1576
1577 /*
1578 * When calling ddi_map_fault, we do not OR in sdp->hat_attr
1579 * This is because this calls drivers which may not expect
1580 * prot to have any other values than PROT_ALL
1581 * The root nexus driver has a hack to peek into the segment
1582 * structure and then OR in sdp->hat_attr.
1583 * XX In case the bus_ops interfaces are ever revisited
1584 * we need to fix this. prot should include other hat attributes
1585 */
1586 if (ddi_map_fault(dip, hat, seg, addr, NULL, pfnum, prot & PROT_ALL,
1587 (uint_t)(type == F_SOFTLOCK)) != DDI_SUCCESS) {
1588 return (FC_MAKE_ERR(EFAULT));
1589 }
1590 return (0);
1591 }
1592
1593 static faultcode_t
segdev_fault(struct hat * hat,struct seg * seg,caddr_t addr,size_t len,enum fault_type type,enum seg_rw rw)1594 segdev_fault(
1595 struct hat *hat, /* the hat */
1596 struct seg *seg, /* the seg_dev of interest */
1597 caddr_t addr, /* the address of the fault */
1598 size_t len, /* the length of the range */
1599 enum fault_type type, /* type of fault */
1600 enum seg_rw rw) /* type of access at fault */
1601 {
1602 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1603 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1604 devmap_handle_t *dhp;
1605 struct devmap_softlock *slock = NULL;
1606 ulong_t slpage = 0;
1607 ulong_t off;
1608 caddr_t maddr = addr;
1609 int err;
1610 int err_is_faultcode = 0;
1611
1612 TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_FAULT,
1613 "segdev_fault: dhp_head=%p seg=%p addr=%p len=%lx type=%x",
1614 (void *)dhp_head, (void *)seg, (void *)addr, len, type);
1615 DEBUGF(7, (CE_CONT, "segdev_fault: dhp_head %p seg %p "
1616 "addr %p len %lx type %x\n",
1617 (void *)dhp_head, (void *)seg, (void *)addr, len, type));
1618
1619 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
1620
1621 /* Handle non-devmap case */
1622 if (dhp_head == NULL)
1623 return (segdev_faultpages(hat, seg, addr, len, type, rw, NULL));
1624
1625 /* Find devmap handle */
1626 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
1627 return (FC_NOMAP);
1628
1629 /*
1630 * The seg_dev driver does not implement copy-on-write,
1631 * and always loads translations with maximal allowed permissions
1632 * but we got an fault trying to access the device.
1633 * Servicing the fault is not going to result in any better result
1634 * RFE: If we want devmap_access callbacks to be involved in F_PROT
1635 * faults, then the code below is written for that
1636 * Pending resolution of the following:
1637 * - determine if the F_INVAL/F_SOFTLOCK syncing
1638 * is needed for F_PROT also or not. The code below assumes it does
1639 * - If driver sees F_PROT and calls devmap_load with same type,
1640 * then segdev_faultpages will fail with FC_PROT anyway, need to
1641 * change that so calls from devmap_load to segdev_faultpages for
1642 * F_PROT type are retagged to F_INVAL.
1643 * RFE: Today we dont have drivers that use devmap and want to handle
1644 * F_PROT calls. The code in segdev_fault* is written to allow
1645 * this case but is not tested. A driver that needs this capability
1646 * should be able to remove the short-circuit case; resolve the
1647 * above issues and "should" work.
1648 */
1649 if (type == F_PROT) {
1650 return (FC_PROT);
1651 }
1652
1653 /*
1654 * Loop through dhp list calling devmap_access or segdev_faultpages for
1655 * each devmap handle.
1656 * drivers which implement devmap_access can interpose on faults and do
1657 * device-appropriate special actions before calling devmap_load.
1658 */
1659
1660 /*
1661 * Unfortunately, this simple loop has turned out to expose a variety
1662 * of complex problems which results in the following convoluted code.
1663 *
1664 * First, a desire to handle a serialization of F_SOFTLOCK calls
1665 * to the driver within the framework.
1666 * This results in a dh_softlock structure that is on a per device
1667 * (or device instance) basis and serializes devmap_access calls.
1668 * Ideally we would need to do this for underlying
1669 * memory/device regions that are being faulted on
1670 * but that is hard to identify and with REMAP, harder
1671 * Second, a desire to serialize F_INVAL(and F_PROT) calls w.r.t.
1672 * to F_SOFTLOCK calls to the driver.
1673 * These serializations are to simplify the driver programmer model.
1674 * To support these two features, the code first goes through the
1675 * devmap handles and counts the pages (slpage) that are covered
1676 * by devmap_access callbacks.
1677 * This part ends with a devmap_softlock_enter call
1678 * which allows only one F_SOFTLOCK active on a device instance,
1679 * but multiple F_INVAL/F_PROTs can be active except when a
1680 * F_SOFTLOCK is active
1681 *
1682 * Next, we dont short-circuit the fault code upfront to call
1683 * segdev_softunlock for F_SOFTUNLOCK, because we must use
1684 * the same length when we softlock and softunlock.
1685 *
1686 * -Hat layers may not support softunlocking lengths less than the
1687 * original length when there is large page support.
1688 * -kpmem locking is dependent on keeping the lengths same.
1689 * -if drivers handled F_SOFTLOCK, they probably also expect to
1690 * see an F_SOFTUNLOCK of the same length
1691 * Hence, if extending lengths during softlock,
1692 * softunlock has to make the same adjustments and goes through
1693 * the same loop calling segdev_faultpages/segdev_softunlock
1694 * But some of the synchronization and error handling is different
1695 */
1696
1697 if (type != F_SOFTUNLOCK) {
1698 devmap_handle_t *dhpp = dhp;
1699 size_t slen = len;
1700
1701 /*
1702 * Calculate count of pages that are :
1703 * a) within the (potentially extended) fault region
1704 * b) AND covered by devmap handle with devmap_access
1705 */
1706 off = (ulong_t)(addr - dhpp->dh_uvaddr);
1707 while (slen != 0) {
1708 size_t mlen;
1709
1710 /*
1711 * Softlocking on a region that allows remap is
1712 * unsupported due to unresolved locking issues
1713 * XXX: unclear what these are?
1714 * One potential is that if there is a pending
1715 * softlock, then a remap should not be allowed
1716 * until the unlock is done. This is easily
1717 * fixed by returning error in devmap*remap on
1718 * checking the dh->dh_softlock->softlocked value
1719 */
1720 if ((type == F_SOFTLOCK) &&
1721 (dhpp->dh_flags & DEVMAP_ALLOW_REMAP)) {
1722 return (FC_NOSUPPORT);
1723 }
1724
1725 mlen = MIN(slen, (dhpp->dh_len - off));
1726 if (dhpp->dh_callbackops.devmap_access) {
1727 size_t llen;
1728 caddr_t laddr;
1729 /*
1730 * use extended length for large page mappings
1731 */
1732 HOLD_DHP_LOCK(dhpp);
1733 if ((sdp->pageprot == 0) &&
1734 (dhpp->dh_flags & DEVMAP_FLAG_LARGE)) {
1735 devmap_get_large_pgsize(dhpp,
1736 mlen, maddr, &llen, &laddr);
1737 } else {
1738 llen = mlen;
1739 }
1740 RELE_DHP_LOCK(dhpp);
1741
1742 slpage += btopr(llen);
1743 slock = dhpp->dh_softlock;
1744 }
1745 maddr += mlen;
1746 ASSERT(slen >= mlen);
1747 slen -= mlen;
1748 dhpp = dhpp->dh_next;
1749 off = 0;
1750 }
1751 /*
1752 * synchonize with other faulting threads and wait till safe
1753 * devmap_softlock_enter might return due to signal in cv_wait
1754 *
1755 * devmap_softlock_enter has to be called outside of while loop
1756 * to prevent a deadlock if len spans over multiple dhps.
1757 * dh_softlock is based on device instance and if multiple dhps
1758 * use the same device instance, the second dhp's LOCK call
1759 * will hang waiting on the first to complete.
1760 * devmap_setup verifies that slocks in a dhp_chain are same.
1761 * RFE: this deadlock only hold true for F_SOFTLOCK. For
1762 * F_INVAL/F_PROT, since we now allow multiple in parallel,
1763 * we could have done the softlock_enter inside the loop
1764 * and supported multi-dhp mappings with dissimilar devices
1765 */
1766 if (err = devmap_softlock_enter(slock, slpage, type))
1767 return (FC_MAKE_ERR(err));
1768 }
1769
1770 /* reset 'maddr' to the start addr of the range of fault. */
1771 maddr = addr;
1772
1773 /* calculate the offset corresponds to 'addr' in the first dhp. */
1774 off = (ulong_t)(addr - dhp->dh_uvaddr);
1775
1776 /*
1777 * The fault length may span over multiple dhps.
1778 * Loop until the total length is satisfied.
1779 */
1780 while (len != 0) {
1781 size_t llen;
1782 size_t mlen;
1783 caddr_t laddr;
1784
1785 /*
1786 * mlen is the smaller of 'len' and the length
1787 * from addr to the end of mapping defined by dhp.
1788 */
1789 mlen = MIN(len, (dhp->dh_len - off));
1790
1791 HOLD_DHP_LOCK(dhp);
1792 /*
1793 * Pass the extended length and address to devmap_access
1794 * if large pagesize is used for loading address translations.
1795 */
1796 if ((sdp->pageprot == 0) &&
1797 (dhp->dh_flags & DEVMAP_FLAG_LARGE)) {
1798 devmap_get_large_pgsize(dhp, mlen, maddr,
1799 &llen, &laddr);
1800 ASSERT(maddr == addr || laddr == maddr);
1801 } else {
1802 llen = mlen;
1803 laddr = maddr;
1804 }
1805
1806 if (dhp->dh_callbackops.devmap_access != NULL) {
1807 offset_t aoff;
1808
1809 aoff = sdp->offset + (offset_t)(laddr - seg->s_base);
1810
1811 /*
1812 * call driver's devmap_access entry point which will
1813 * call devmap_load/contextmgmt to load the translations
1814 *
1815 * We drop the dhp_lock before calling access so
1816 * drivers can call devmap_*_remap within access
1817 */
1818 RELE_DHP_LOCK(dhp);
1819
1820 err = (*dhp->dh_callbackops.devmap_access)(
1821 dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw);
1822 } else {
1823 /*
1824 * If no devmap_access entry point, then load mappings
1825 * hold dhp_lock across faultpages if REMAP
1826 */
1827 err = segdev_faultpages(hat, seg, laddr, llen,
1828 type, rw, dhp);
1829 err_is_faultcode = 1;
1830 RELE_DHP_LOCK(dhp);
1831 }
1832
1833 if (err) {
1834 if ((type == F_SOFTLOCK) && (maddr > addr)) {
1835 /*
1836 * If not first dhp, use
1837 * segdev_fault(F_SOFTUNLOCK) for prior dhps
1838 * While this is recursion, it is incorrect to
1839 * call just segdev_softunlock
1840 * if we are using either large pages
1841 * or devmap_access. It will be more right
1842 * to go through the same loop as above
1843 * rather than call segdev_softunlock directly
1844 * It will use the right lenghths as well as
1845 * call into the driver devmap_access routines.
1846 */
1847 size_t done = (size_t)(maddr - addr);
1848 (void) segdev_fault(hat, seg, addr, done,
1849 F_SOFTUNLOCK, S_OTHER);
1850 /*
1851 * reduce slpage by number of pages
1852 * released by segdev_softunlock
1853 */
1854 ASSERT(slpage >= btopr(done));
1855 devmap_softlock_exit(slock,
1856 slpage - btopr(done), type);
1857 } else {
1858 devmap_softlock_exit(slock, slpage, type);
1859 }
1860
1861
1862 /*
1863 * Segdev_faultpages() already returns a faultcode,
1864 * hence, result from segdev_faultpages() should be
1865 * returned directly.
1866 */
1867 if (err_is_faultcode)
1868 return (err);
1869 return (FC_MAKE_ERR(err));
1870 }
1871
1872 maddr += mlen;
1873 ASSERT(len >= mlen);
1874 len -= mlen;
1875 dhp = dhp->dh_next;
1876 off = 0;
1877
1878 ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr);
1879 }
1880 /*
1881 * release the softlock count at end of fault
1882 * For F_SOFTLOCk this is done in the later F_SOFTUNLOCK
1883 */
1884 if ((type == F_INVAL) || (type == F_PROT))
1885 devmap_softlock_exit(slock, slpage, type);
1886 return (0);
1887 }
1888
1889 /*
1890 * segdev_faultpages
1891 *
1892 * Used to fault in seg_dev segment pages. Called by segdev_fault or devmap_load
1893 * This routine assumes that the callers makes sure that the fields
1894 * in dhp used below are not changed due to remap during this call.
1895 * Caller does HOLD_DHP_LOCK if neeed
1896 * This routine returns a faultcode_t as a return value for segdev_fault.
1897 */
1898 static faultcode_t
segdev_faultpages(struct hat * hat,struct seg * seg,caddr_t addr,size_t len,enum fault_type type,enum seg_rw rw,devmap_handle_t * dhp)1899 segdev_faultpages(
1900 struct hat *hat, /* the hat */
1901 struct seg *seg, /* the seg_dev of interest */
1902 caddr_t addr, /* the address of the fault */
1903 size_t len, /* the length of the range */
1904 enum fault_type type, /* type of fault */
1905 enum seg_rw rw, /* type of access at fault */
1906 devmap_handle_t *dhp) /* devmap handle */
1907 {
1908 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1909 register caddr_t a;
1910 struct vpage *vpage;
1911 struct ddi_umem_cookie *kpmem_cookie = NULL;
1912 int err;
1913
1914 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGES,
1915 "segdev_faultpages: dhp=%p seg=%p addr=%p len=%lx",
1916 (void *)dhp, (void *)seg, (void *)addr, len);
1917 DEBUGF(5, (CE_CONT, "segdev_faultpages: "
1918 "dhp %p seg %p addr %p len %lx\n",
1919 (void *)dhp, (void *)seg, (void *)addr, len));
1920
1921 /*
1922 * The seg_dev driver does not implement copy-on-write,
1923 * and always loads translations with maximal allowed permissions
1924 * but we got an fault trying to access the device.
1925 * Servicing the fault is not going to result in any better result
1926 * XXX: If we want to allow devmap_access to handle F_PROT calls,
1927 * This code should be removed and let the normal fault handling
1928 * take care of finding the error
1929 */
1930 if (type == F_PROT) {
1931 return (FC_PROT);
1932 }
1933
1934 if (type == F_SOFTUNLOCK) {
1935 segdev_softunlock(hat, seg, addr, len, rw);
1936 return (0);
1937 }
1938
1939 /*
1940 * For kernel pageable memory, fault/lock segkp pages
1941 * We hold this until the completion of this
1942 * fault (INVAL/PROT) or till unlock (SOFTLOCK).
1943 */
1944 if ((dhp != NULL) && dhp_is_kpmem(dhp)) {
1945 kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie;
1946 if (err = acquire_kpmem_lock(kpmem_cookie, btopr(len)))
1947 return (err);
1948 }
1949
1950 /*
1951 * If we have the same protections for the entire segment,
1952 * insure that the access being attempted is legitimate.
1953 */
1954 rw_enter(&sdp->lock, RW_READER);
1955 if (sdp->pageprot == 0) {
1956 uint_t protchk;
1957
1958 switch (rw) {
1959 case S_READ:
1960 protchk = PROT_READ;
1961 break;
1962 case S_WRITE:
1963 protchk = PROT_WRITE;
1964 break;
1965 case S_EXEC:
1966 protchk = PROT_EXEC;
1967 break;
1968 case S_OTHER:
1969 default:
1970 protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1971 break;
1972 }
1973
1974 if ((sdp->prot & protchk) == 0) {
1975 rw_exit(&sdp->lock);
1976 /* undo kpmem locking */
1977 if (kpmem_cookie != NULL) {
1978 release_kpmem_lock(kpmem_cookie, btopr(len));
1979 }
1980 return (FC_PROT); /* illegal access type */
1981 }
1982 }
1983
1984 /*
1985 * we do a single hat_devload for the range if
1986 * - devmap framework (dhp is not NULL),
1987 * - pageprot == 0, i.e., no per-page protection set and
1988 * - is device pages, irrespective of whether we are using large pages
1989 */
1990 if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) {
1991 pfn_t pfnum;
1992 uint_t hat_flags;
1993
1994 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1995 rw_exit(&sdp->lock);
1996 return (FC_NOMAP);
1997 }
1998
1999 if (type == F_SOFTLOCK) {
2000 mutex_enter(&freemem_lock);
2001 sdp->softlockcnt += btopr(len);
2002 mutex_exit(&freemem_lock);
2003 }
2004
2005 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
2006 pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr));
2007 ASSERT(!pf_is_memory(pfnum));
2008
2009 hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr,
2010 hat_flags | sdp->hat_flags);
2011 rw_exit(&sdp->lock);
2012 return (0);
2013 }
2014
2015 /* Handle cases where we have to loop through fault handling per-page */
2016
2017 if (sdp->vpage == NULL)
2018 vpage = NULL;
2019 else
2020 vpage = &sdp->vpage[seg_page(seg, addr)];
2021
2022 /* loop over the address range handling each fault */
2023 for (a = addr; a < addr + len; a += PAGESIZE) {
2024 if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) {
2025 break;
2026 }
2027 if (vpage != NULL)
2028 vpage++;
2029 }
2030 rw_exit(&sdp->lock);
2031 if (err && (type == F_SOFTLOCK)) { /* error handling for F_SOFTLOCK */
2032 size_t done = (size_t)(a - addr); /* pages fault successfully */
2033 if (done > 0) {
2034 /* use softunlock for those pages */
2035 segdev_softunlock(hat, seg, addr, done, S_OTHER);
2036 }
2037 if (kpmem_cookie != NULL) {
2038 /* release kpmem lock for rest of pages */
2039 ASSERT(len >= done);
2040 release_kpmem_lock(kpmem_cookie, btopr(len - done));
2041 }
2042 } else if ((kpmem_cookie != NULL) && (type != F_SOFTLOCK)) {
2043 /* for non-SOFTLOCK cases, release kpmem */
2044 release_kpmem_lock(kpmem_cookie, btopr(len));
2045 }
2046 return (err);
2047 }
2048
2049 /*
2050 * Asynchronous page fault. We simply do nothing since this
2051 * entry point is not supposed to load up the translation.
2052 */
2053 /*ARGSUSED*/
2054 static faultcode_t
segdev_faulta(struct seg * seg,caddr_t addr)2055 segdev_faulta(struct seg *seg, caddr_t addr)
2056 {
2057 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FAULTA,
2058 "segdev_faulta: seg=%p addr=%p", (void *)seg, (void *)addr);
2059 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2060
2061 return (0);
2062 }
2063
2064 static int
segdev_setprot(struct seg * seg,caddr_t addr,size_t len,uint_t prot)2065 segdev_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2066 {
2067 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2068 register devmap_handle_t *dhp;
2069 register struct vpage *vp, *evp;
2070 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
2071 ulong_t off;
2072 size_t mlen, sz;
2073
2074 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT,
2075 "segdev_setprot:start seg=%p addr=%p len=%lx prot=%x",
2076 (void *)seg, (void *)addr, len, prot);
2077 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2078
2079 if ((sz = sdp->softlockcnt) > 0 && dhp_head != NULL) {
2080 /*
2081 * Fail the setprot if pages are SOFTLOCKed through this
2082 * mapping.
2083 * Softlockcnt is protected from change by the as read lock.
2084 */
2085 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT_CK1,
2086 "segdev_setprot:error softlockcnt=%lx", sz);
2087 DEBUGF(1, (CE_CONT, "segdev_setprot: softlockcnt %ld\n", sz));
2088 return (EAGAIN);
2089 }
2090
2091 if (dhp_head != NULL) {
2092 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
2093 return (EINVAL);
2094
2095 /*
2096 * check if violate maxprot.
2097 */
2098 off = (ulong_t)(addr - dhp->dh_uvaddr);
2099 mlen = len;
2100 while (dhp) {
2101 if ((dhp->dh_maxprot & prot) != prot)
2102 return (EACCES); /* violated maxprot */
2103
2104 if (mlen > (dhp->dh_len - off)) {
2105 mlen -= dhp->dh_len - off;
2106 dhp = dhp->dh_next;
2107 off = 0;
2108 } else
2109 break;
2110 }
2111 } else {
2112 if ((sdp->maxprot & prot) != prot)
2113 return (EACCES);
2114 }
2115
2116 rw_enter(&sdp->lock, RW_WRITER);
2117 if (addr == seg->s_base && len == seg->s_size && sdp->pageprot == 0) {
2118 if (sdp->prot == prot) {
2119 rw_exit(&sdp->lock);
2120 return (0); /* all done */
2121 }
2122 sdp->prot = (uchar_t)prot;
2123 } else {
2124 sdp->pageprot = 1;
2125 if (sdp->vpage == NULL) {
2126 /*
2127 * First time through setting per page permissions,
2128 * initialize all the vpage structures to prot
2129 */
2130 sdp->vpage = kmem_zalloc(vpgtob(seg_pages(seg)),
2131 KM_SLEEP);
2132 evp = &sdp->vpage[seg_pages(seg)];
2133 for (vp = sdp->vpage; vp < evp; vp++)
2134 VPP_SETPROT(vp, sdp->prot);
2135 }
2136 /*
2137 * Now go change the needed vpages protections.
2138 */
2139 evp = &sdp->vpage[seg_page(seg, addr + len)];
2140 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++)
2141 VPP_SETPROT(vp, prot);
2142 }
2143 rw_exit(&sdp->lock);
2144
2145 if (dhp_head != NULL) {
2146 devmap_handle_t *tdhp;
2147 /*
2148 * If large page size was used in hat_devload(),
2149 * the same page size must be used in hat_unload().
2150 */
2151 dhp = tdhp = devmap_find_handle(dhp_head, addr);
2152 while (tdhp != NULL) {
2153 if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
2154 break;
2155 }
2156 tdhp = tdhp->dh_next;
2157 }
2158 if (tdhp) {
2159 size_t slen = len;
2160 size_t mlen;
2161 size_t soff;
2162
2163 soff = (ulong_t)(addr - dhp->dh_uvaddr);
2164 while (slen != 0) {
2165 mlen = MIN(slen, (dhp->dh_len - soff));
2166 hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr,
2167 dhp->dh_len, HAT_UNLOAD);
2168 dhp = dhp->dh_next;
2169 ASSERT(slen >= mlen);
2170 slen -= mlen;
2171 soff = 0;
2172 }
2173 return (0);
2174 }
2175 }
2176
2177 if ((prot & ~PROT_USER) == PROT_NONE) {
2178 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD);
2179 } else {
2180 /*
2181 * RFE: the segment should keep track of all attributes
2182 * allowing us to remove the deprecated hat_chgprot
2183 * and use hat_chgattr.
2184 */
2185 hat_chgprot(seg->s_as->a_hat, addr, len, prot);
2186 }
2187
2188 return (0);
2189 }
2190
2191 static int
segdev_checkprot(struct seg * seg,caddr_t addr,size_t len,uint_t prot)2192 segdev_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2193 {
2194 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2195 struct vpage *vp, *evp;
2196
2197 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_CHECKPROT,
2198 "segdev_checkprot:start seg=%p addr=%p len=%lx prot=%x",
2199 (void *)seg, (void *)addr, len, prot);
2200 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2201
2202 /*
2203 * If segment protection can be used, simply check against them
2204 */
2205 rw_enter(&sdp->lock, RW_READER);
2206 if (sdp->pageprot == 0) {
2207 register int err;
2208
2209 err = ((sdp->prot & prot) != prot) ? EACCES : 0;
2210 rw_exit(&sdp->lock);
2211 return (err);
2212 }
2213
2214 /*
2215 * Have to check down to the vpage level
2216 */
2217 evp = &sdp->vpage[seg_page(seg, addr + len)];
2218 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) {
2219 if ((VPP_PROT(vp) & prot) != prot) {
2220 rw_exit(&sdp->lock);
2221 return (EACCES);
2222 }
2223 }
2224 rw_exit(&sdp->lock);
2225 return (0);
2226 }
2227
2228 static int
segdev_getprot(struct seg * seg,caddr_t addr,size_t len,uint_t * protv)2229 segdev_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
2230 {
2231 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2232 size_t pgno;
2233
2234 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_GETPROT,
2235 "segdev_getprot:start seg=%p addr=%p len=%lx protv=%p",
2236 (void *)seg, (void *)addr, len, (void *)protv);
2237 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2238
2239 pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
2240 if (pgno != 0) {
2241 rw_enter(&sdp->lock, RW_READER);
2242 if (sdp->pageprot == 0) {
2243 do {
2244 protv[--pgno] = sdp->prot;
2245 } while (pgno != 0);
2246 } else {
2247 size_t pgoff = seg_page(seg, addr);
2248
2249 do {
2250 pgno--;
2251 protv[pgno] =
2252 VPP_PROT(&sdp->vpage[pgno + pgoff]);
2253 } while (pgno != 0);
2254 }
2255 rw_exit(&sdp->lock);
2256 }
2257 return (0);
2258 }
2259
2260 static u_offset_t
segdev_getoffset(register struct seg * seg,caddr_t addr)2261 segdev_getoffset(register struct seg *seg, caddr_t addr)
2262 {
2263 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2264
2265 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETOFFSET,
2266 "segdev_getoffset:start seg=%p addr=%p", (void *)seg, (void *)addr);
2267
2268 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2269
2270 return ((u_offset_t)sdp->offset + (addr - seg->s_base));
2271 }
2272
2273 /*ARGSUSED*/
2274 static int
segdev_gettype(register struct seg * seg,caddr_t addr)2275 segdev_gettype(register struct seg *seg, caddr_t addr)
2276 {
2277 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2278
2279 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETTYPE,
2280 "segdev_gettype:start seg=%p addr=%p", (void *)seg, (void *)addr);
2281
2282 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2283
2284 return (sdp->type);
2285 }
2286
2287
2288 /*ARGSUSED*/
2289 static int
segdev_getvp(register struct seg * seg,caddr_t addr,struct vnode ** vpp)2290 segdev_getvp(register struct seg *seg, caddr_t addr, struct vnode **vpp)
2291 {
2292 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2293
2294 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETVP,
2295 "segdev_getvp:start seg=%p addr=%p", (void *)seg, (void *)addr);
2296
2297 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2298
2299 /*
2300 * Note that this vp is the common_vp of the device, where the
2301 * pages are hung ..
2302 */
2303 *vpp = VTOCVP(sdp->vp);
2304
2305 return (0);
2306 }
2307
2308 static void
segdev_badop(void)2309 segdev_badop(void)
2310 {
2311 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGDEV_BADOP,
2312 "segdev_badop:start");
2313 panic("segdev_badop");
2314 /*NOTREACHED*/
2315 }
2316
2317 /*
2318 * segdev pages are not in the cache, and thus can't really be controlled.
2319 * Hence, syncs are simply always successful.
2320 */
2321 /*ARGSUSED*/
2322 static int
segdev_sync(struct seg * seg,caddr_t addr,size_t len,int attr,uint_t flags)2323 segdev_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags)
2324 {
2325 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SYNC, "segdev_sync:start");
2326
2327 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2328
2329 return (0);
2330 }
2331
2332 /*
2333 * segdev pages are always "in core".
2334 */
2335 /*ARGSUSED*/
2336 static size_t
segdev_incore(struct seg * seg,caddr_t addr,size_t len,char * vec)2337 segdev_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
2338 {
2339 size_t v = 0;
2340
2341 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_INCORE, "segdev_incore:start");
2342
2343 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2344
2345 for (len = (len + PAGEOFFSET) & PAGEMASK; len; len -= PAGESIZE,
2346 v += PAGESIZE)
2347 *vec++ = 1;
2348 return (v);
2349 }
2350
2351 /*
2352 * segdev pages are not in the cache, and thus can't really be controlled.
2353 * Hence, locks are simply always successful.
2354 */
2355 /*ARGSUSED*/
2356 static int
segdev_lockop(struct seg * seg,caddr_t addr,size_t len,int attr,int op,ulong_t * lockmap,size_t pos)2357 segdev_lockop(struct seg *seg, caddr_t addr,
2358 size_t len, int attr, int op, ulong_t *lockmap, size_t pos)
2359 {
2360 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_LOCKOP, "segdev_lockop:start");
2361
2362 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2363
2364 return (0);
2365 }
2366
2367 /*
2368 * segdev pages are not in the cache, and thus can't really be controlled.
2369 * Hence, advise is simply always successful.
2370 */
2371 /*ARGSUSED*/
2372 static int
segdev_advise(struct seg * seg,caddr_t addr,size_t len,uint_t behav)2373 segdev_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
2374 {
2375 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_ADVISE, "segdev_advise:start");
2376
2377 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2378
2379 return (0);
2380 }
2381
2382 /*
2383 * segdev pages are not dumped, so we just return
2384 */
2385 /*ARGSUSED*/
2386 static void
segdev_dump(struct seg * seg)2387 segdev_dump(struct seg *seg)
2388 {}
2389
2390 /*
2391 * ddi_segmap_setup: Used by drivers who wish specify mapping attributes
2392 * for a segment. Called from a drivers segmap(9E)
2393 * routine.
2394 */
2395 /*ARGSUSED*/
2396 int
ddi_segmap_setup(dev_t dev,off_t offset,struct as * as,caddr_t * addrp,off_t len,uint_t prot,uint_t maxprot,uint_t flags,cred_t * cred,ddi_device_acc_attr_t * accattrp,uint_t rnumber)2397 ddi_segmap_setup(dev_t dev, off_t offset, struct as *as, caddr_t *addrp,
2398 off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cred,
2399 ddi_device_acc_attr_t *accattrp, uint_t rnumber)
2400 {
2401 struct segdev_crargs dev_a;
2402 int (*mapfunc)(dev_t dev, off_t off, int prot);
2403 uint_t hat_attr;
2404 pfn_t pfn;
2405 int error, i;
2406
2407 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP_SETUP,
2408 "ddi_segmap_setup:start");
2409
2410 if ((mapfunc = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap) == nodev)
2411 return (ENODEV);
2412
2413 /*
2414 * Character devices that support the d_mmap
2415 * interface can only be mmap'ed shared.
2416 */
2417 if ((flags & MAP_TYPE) != MAP_SHARED)
2418 return (EINVAL);
2419
2420 /*
2421 * Check that this region is indeed mappable on this platform.
2422 * Use the mapping function.
2423 */
2424 if (ddi_device_mapping_check(dev, accattrp, rnumber, &hat_attr) == -1)
2425 return (ENXIO);
2426
2427 /*
2428 * Check to ensure that the entire range is
2429 * legal and we are not trying to map in
2430 * more than the device will let us.
2431 */
2432 for (i = 0; i < len; i += PAGESIZE) {
2433 if (i == 0) {
2434 /*
2435 * Save the pfn at offset here. This pfn will be
2436 * used later to get user address.
2437 */
2438 if ((pfn = (pfn_t)cdev_mmap(mapfunc, dev, offset,
2439 maxprot)) == PFN_INVALID)
2440 return (ENXIO);
2441 } else {
2442 if (cdev_mmap(mapfunc, dev, offset + i, maxprot) ==
2443 PFN_INVALID)
2444 return (ENXIO);
2445 }
2446 }
2447
2448 as_rangelock(as);
2449 /* Pick an address w/o worrying about any vac alignment constraints. */
2450 error = choose_addr(as, addrp, len, ptob(pfn), ADDR_NOVACALIGN, flags);
2451 if (error != 0) {
2452 as_rangeunlock(as);
2453 return (error);
2454 }
2455
2456 dev_a.mapfunc = mapfunc;
2457 dev_a.dev = dev;
2458 dev_a.offset = (offset_t)offset;
2459 dev_a.type = flags & MAP_TYPE;
2460 dev_a.prot = (uchar_t)prot;
2461 dev_a.maxprot = (uchar_t)maxprot;
2462 dev_a.hat_attr = hat_attr;
2463 dev_a.hat_flags = 0;
2464 dev_a.devmap_data = NULL;
2465
2466 error = as_map(as, *addrp, len, segdev_create, &dev_a);
2467 as_rangeunlock(as);
2468 return (error);
2469
2470 }
2471
2472 /*ARGSUSED*/
2473 static int
segdev_pagelock(struct seg * seg,caddr_t addr,size_t len,struct page *** ppp,enum lock_type type,enum seg_rw rw)2474 segdev_pagelock(struct seg *seg, caddr_t addr, size_t len,
2475 struct page ***ppp, enum lock_type type, enum seg_rw rw)
2476 {
2477 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_PAGELOCK,
2478 "segdev_pagelock:start");
2479 return (ENOTSUP);
2480 }
2481
2482 /*ARGSUSED*/
2483 static int
segdev_setpagesize(struct seg * seg,caddr_t addr,size_t len,uint_t szc)2484 segdev_setpagesize(struct seg *seg, caddr_t addr, size_t len,
2485 uint_t szc)
2486 {
2487 return (ENOTSUP);
2488 }
2489
2490 /*
2491 * devmap_device: Used by devmap framework to establish mapping
2492 * called by devmap_seup(9F) during map setup time.
2493 */
2494 /*ARGSUSED*/
2495 static int
devmap_device(devmap_handle_t * dhp,struct as * as,caddr_t * addr,offset_t off,size_t len,uint_t flags)2496 devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
2497 offset_t off, size_t len, uint_t flags)
2498 {
2499 devmap_handle_t *rdhp, *maxdhp;
2500 struct segdev_crargs dev_a;
2501 int err;
2502 uint_t maxprot = PROT_ALL;
2503 offset_t offset = 0;
2504 pfn_t pfn;
2505 struct devmap_pmem_cookie *pcp;
2506
2507 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVICE,
2508 "devmap_device:start dhp=%p addr=%p off=%llx, len=%lx",
2509 (void *)dhp, (void *)addr, off, len);
2510
2511 DEBUGF(2, (CE_CONT, "devmap_device: dhp %p addr %p off %llx len %lx\n",
2512 (void *)dhp, (void *)addr, off, len));
2513
2514 as_rangelock(as);
2515 if ((flags & MAP_FIXED) == 0) {
2516 offset_t aligned_off;
2517
2518 rdhp = maxdhp = dhp;
2519 while (rdhp != NULL) {
2520 maxdhp = (maxdhp->dh_len > rdhp->dh_len) ?
2521 maxdhp : rdhp;
2522 rdhp = rdhp->dh_next;
2523 maxprot |= dhp->dh_maxprot;
2524 }
2525 offset = maxdhp->dh_uoff - dhp->dh_uoff;
2526
2527 /*
2528 * Use the dhp that has the
2529 * largest len to get user address.
2530 */
2531 /*
2532 * If MAPPING_INVALID, cannot use dh_pfn/dh_cvaddr,
2533 * use 0 which is as good as any other.
2534 */
2535 if (maxdhp->dh_flags & DEVMAP_MAPPING_INVALID) {
2536 aligned_off = (offset_t)0;
2537 } else if (dhp_is_devmem(maxdhp)) {
2538 aligned_off = (offset_t)ptob(maxdhp->dh_pfn) - offset;
2539 } else if (dhp_is_pmem(maxdhp)) {
2540 pcp = (struct devmap_pmem_cookie *)maxdhp->dh_pcookie;
2541 pfn = page_pptonum(
2542 pcp->dp_pparray[btop(maxdhp->dh_roff)]);
2543 aligned_off = (offset_t)ptob(pfn) - offset;
2544 } else {
2545 aligned_off = (offset_t)(uintptr_t)maxdhp->dh_cvaddr -
2546 offset;
2547 }
2548
2549 /*
2550 * Pick an address aligned to dh_cookie.
2551 * for kernel memory/user memory, cookie is cvaddr.
2552 * for device memory, cookie is physical address.
2553 */
2554 map_addr(addr, len, aligned_off, 1, flags);
2555 if (*addr == NULL) {
2556 as_rangeunlock(as);
2557 return (ENOMEM);
2558 }
2559 } else {
2560 /*
2561 * User-specified address; blow away any previous mappings.
2562 */
2563 (void) as_unmap(as, *addr, len);
2564 }
2565
2566 dev_a.mapfunc = NULL;
2567 dev_a.dev = dhp->dh_dev;
2568 dev_a.type = flags & MAP_TYPE;
2569 dev_a.offset = off;
2570 /*
2571 * sdp->maxprot has the least restrict protection of all dhps.
2572 */
2573 dev_a.maxprot = maxprot;
2574 dev_a.prot = dhp->dh_prot;
2575 /*
2576 * devmap uses dhp->dh_hat_attr for hat.
2577 */
2578 dev_a.hat_flags = 0;
2579 dev_a.hat_attr = 0;
2580 dev_a.devmap_data = (void *)dhp;
2581
2582 err = as_map(as, *addr, len, segdev_create, &dev_a);
2583 as_rangeunlock(as);
2584 return (err);
2585 }
2586
2587 int
devmap_do_ctxmgt(devmap_cookie_t dhc,void * pvtp,offset_t off,size_t len,uint_t type,uint_t rw,int (* ctxmgt)(devmap_cookie_t,void *,offset_t,size_t,uint_t,uint_t))2588 devmap_do_ctxmgt(devmap_cookie_t dhc, void *pvtp, offset_t off, size_t len,
2589 uint_t type, uint_t rw, int (*ctxmgt)(devmap_cookie_t, void *, offset_t,
2590 size_t, uint_t, uint_t))
2591 {
2592 register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2593 struct devmap_ctx *devctx;
2594 int do_timeout = 0;
2595 int ret;
2596
2597 #ifdef lint
2598 pvtp = pvtp;
2599 #endif
2600
2601 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT,
2602 "devmap_do_ctxmgt:start dhp=%p off=%llx, len=%lx",
2603 (void *)dhp, off, len);
2604 DEBUGF(7, (CE_CONT, "devmap_do_ctxmgt: dhp %p off %llx len %lx\n",
2605 (void *)dhp, off, len));
2606
2607 if (ctxmgt == NULL)
2608 return (FC_HWERR);
2609
2610 devctx = dhp->dh_ctx;
2611
2612 /*
2613 * If we are on an MP system with more than one cpu running
2614 * and if a thread on some CPU already has the context, wait
2615 * for it to finish if there is a hysteresis timeout.
2616 *
2617 * We call cv_wait() instead of cv_wait_sig() because
2618 * it does not matter much if it returned due to a signal
2619 * or due to a cv_signal() or cv_broadcast(). In either event
2620 * we need to complete the mapping otherwise the processes
2621 * will die with a SEGV.
2622 */
2623 if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) {
2624 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK1,
2625 "devmap_do_ctxmgt:doing hysteresis, devctl %p dhp %p",
2626 devctx, dhp);
2627 do_timeout = 1;
2628 mutex_enter(&devctx->lock);
2629 while (devctx->oncpu)
2630 cv_wait(&devctx->cv, &devctx->lock);
2631 devctx->oncpu = 1;
2632 mutex_exit(&devctx->lock);
2633 }
2634
2635 /*
2636 * Call the contextmgt callback so that the driver can handle
2637 * the fault.
2638 */
2639 ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw);
2640
2641 /*
2642 * If devmap_access() returned -1, then there was a hardware
2643 * error so we need to convert the return value to something
2644 * that trap() will understand. Otherwise, the return value
2645 * is already a fault code generated by devmap_unload()
2646 * or devmap_load().
2647 */
2648 if (ret) {
2649 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK2,
2650 "devmap_do_ctxmgt: ret=%x dhp=%p devctx=%p",
2651 ret, dhp, devctx);
2652 DEBUGF(1, (CE_CONT, "devmap_do_ctxmgt: ret %x dhp %p\n",
2653 ret, (void *)dhp));
2654 if (devctx->oncpu) {
2655 mutex_enter(&devctx->lock);
2656 devctx->oncpu = 0;
2657 cv_signal(&devctx->cv);
2658 mutex_exit(&devctx->lock);
2659 }
2660 return (FC_HWERR);
2661 }
2662
2663 /*
2664 * Setup the timeout if we need to
2665 */
2666 if (do_timeout) {
2667 mutex_enter(&devctx->lock);
2668 if (dhp->dh_timeout_length > 0) {
2669 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK3,
2670 "devmap_do_ctxmgt:timeout set");
2671 devctx->timeout = timeout(devmap_ctxto,
2672 devctx, dhp->dh_timeout_length);
2673 } else {
2674 /*
2675 * We don't want to wait so set oncpu to
2676 * 0 and wake up anyone waiting.
2677 */
2678 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK4,
2679 "devmap_do_ctxmgt:timeout not set");
2680 devctx->oncpu = 0;
2681 cv_signal(&devctx->cv);
2682 }
2683 mutex_exit(&devctx->lock);
2684 }
2685
2686 return (DDI_SUCCESS);
2687 }
2688
2689 /*
2690 * end of mapping
2691 * poff fault_offset |
2692 * base | | |
2693 * | | | |
2694 * V V V V
2695 * +-----------+---------------+-------+---------+-------+
2696 * ^ ^ ^ ^
2697 * |<--- offset--->|<-len->| |
2698 * |<--- dh_len(size of mapping) --->|
2699 * |<-- pg -->|
2700 * -->|rlen|<--
2701 */
2702 static ulong_t
devmap_roundup(devmap_handle_t * dhp,ulong_t offset,size_t len,ulong_t * opfn,ulong_t * pagesize)2703 devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
2704 ulong_t *opfn, ulong_t *pagesize)
2705 {
2706 register int level;
2707 ulong_t pg;
2708 ulong_t poff;
2709 ulong_t base;
2710 caddr_t uvaddr;
2711 long rlen;
2712
2713 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP,
2714 "devmap_roundup:start dhp=%p off=%lx len=%lx",
2715 (void *)dhp, offset, len);
2716 DEBUGF(2, (CE_CONT, "devmap_roundup: dhp %p off %lx len %lx\n",
2717 (void *)dhp, offset, len));
2718
2719 /*
2720 * get the max. pagesize that is aligned within the range
2721 * <dh_pfn, dh_pfn+offset>.
2722 *
2723 * The calculations below use physical address to ddetermine
2724 * the page size to use. The same calculations can use the
2725 * virtual address to determine the page size.
2726 */
2727 base = (ulong_t)ptob(dhp->dh_pfn);
2728 for (level = dhp->dh_mmulevel; level >= 0; level--) {
2729 pg = page_get_pagesize(level);
2730 poff = ((base + offset) & ~(pg - 1));
2731 uvaddr = dhp->dh_uvaddr + (poff - base);
2732 if ((poff >= base) &&
2733 ((poff + pg) <= (base + dhp->dh_len)) &&
2734 VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg))
2735 break;
2736 }
2737
2738 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK1,
2739 "devmap_roundup: base=%lx poff=%lx dhp=%p",
2740 base, poff, dhp);
2741 DEBUGF(2, (CE_CONT, "devmap_roundup: base %lx poff %lx pfn %lx\n",
2742 base, poff, dhp->dh_pfn));
2743
2744 ASSERT(VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg));
2745 ASSERT(level >= 0);
2746
2747 *pagesize = pg;
2748 *opfn = dhp->dh_pfn + btop(poff - base);
2749
2750 rlen = len + offset - (poff - base + pg);
2751
2752 ASSERT(rlen < (long)len);
2753
2754 TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK2,
2755 "devmap_roundup:ret dhp=%p level=%x rlen=%lx psiz=%p opfn=%p",
2756 (void *)dhp, level, rlen, pagesize, opfn);
2757 DEBUGF(1, (CE_CONT, "devmap_roundup: dhp %p "
2758 "level %x rlen %lx psize %lx opfn %lx\n",
2759 (void *)dhp, level, rlen, *pagesize, *opfn));
2760
2761 return ((ulong_t)((rlen > 0) ? rlen : 0));
2762 }
2763
2764 /*
2765 * find the dhp that contains addr.
2766 */
2767 static devmap_handle_t *
devmap_find_handle(devmap_handle_t * dhp_head,caddr_t addr)2768 devmap_find_handle(devmap_handle_t *dhp_head, caddr_t addr)
2769 {
2770 devmap_handle_t *dhp;
2771
2772 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_FIND_HANDLE,
2773 "devmap_find_handle:start");
2774
2775 dhp = dhp_head;
2776 while (dhp) {
2777 if (addr >= dhp->dh_uvaddr &&
2778 addr < (dhp->dh_uvaddr + dhp->dh_len))
2779 return (dhp);
2780 dhp = dhp->dh_next;
2781 }
2782
2783 return ((devmap_handle_t *)NULL);
2784 }
2785
2786 /*
2787 * devmap_unload:
2788 * Marks a segdev segment or pages if offset->offset+len
2789 * is not the entire segment as intercept and unloads the
2790 * pages in the range offset -> offset+len.
2791 */
2792 int
devmap_unload(devmap_cookie_t dhc,offset_t offset,size_t len)2793 devmap_unload(devmap_cookie_t dhc, offset_t offset, size_t len)
2794 {
2795 register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2796 caddr_t addr;
2797 ulong_t size;
2798 ssize_t soff;
2799
2800 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_UNLOAD,
2801 "devmap_unload:start dhp=%p offset=%llx len=%lx",
2802 (void *)dhp, offset, len);
2803 DEBUGF(7, (CE_CONT, "devmap_unload: dhp %p offset %llx len %lx\n",
2804 (void *)dhp, offset, len));
2805
2806 soff = (ssize_t)(offset - dhp->dh_uoff);
2807 soff = round_down_p2(soff, PAGESIZE);
2808 if (soff < 0 || soff >= dhp->dh_len)
2809 return (FC_MAKE_ERR(EINVAL));
2810
2811 /*
2812 * Address and size must be page aligned. Len is set to the
2813 * number of bytes in the number of pages that are required to
2814 * support len. Offset is set to the byte offset of the first byte
2815 * of the page that contains offset.
2816 */
2817 len = round_up_p2(len, PAGESIZE);
2818
2819 /*
2820 * If len is == 0, then calculate the size by getting
2821 * the number of bytes from offset to the end of the segment.
2822 */
2823 if (len == 0)
2824 size = dhp->dh_len - soff;
2825 else {
2826 size = len;
2827 if ((soff + size) > dhp->dh_len)
2828 return (FC_MAKE_ERR(EINVAL));
2829 }
2830
2831 /*
2832 * The address is offset bytes from the base address of
2833 * the dhp.
2834 */
2835 addr = (caddr_t)(soff + dhp->dh_uvaddr);
2836
2837 /*
2838 * If large page size was used in hat_devload(),
2839 * the same page size must be used in hat_unload().
2840 */
2841 if (dhp->dh_flags & DEVMAP_FLAG_LARGE) {
2842 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
2843 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
2844 } else {
2845 hat_unload(dhp->dh_seg->s_as->a_hat, addr, size,
2846 HAT_UNLOAD|HAT_UNLOAD_OTHER);
2847 }
2848
2849 return (0);
2850 }
2851
2852 /*
2853 * calculates the optimal page size that will be used for hat_devload().
2854 */
2855 static void
devmap_get_large_pgsize(devmap_handle_t * dhp,size_t len,caddr_t addr,size_t * llen,caddr_t * laddr)2856 devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr,
2857 size_t *llen, caddr_t *laddr)
2858 {
2859 ulong_t off;
2860 ulong_t pfn;
2861 ulong_t pgsize;
2862 uint_t first = 1;
2863
2864 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GET_LARGE_PGSIZE,
2865 "devmap_get_large_pgsize:start");
2866
2867 /*
2868 * RFE - Code only supports large page mappings for devmem
2869 * This code could be changed in future if we want to support
2870 * large page mappings for kernel exported memory.
2871 */
2872 ASSERT(dhp_is_devmem(dhp));
2873 ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID));
2874
2875 *llen = 0;
2876 off = (ulong_t)(addr - dhp->dh_uvaddr);
2877 while ((long)len > 0) {
2878 /*
2879 * get the optimal pfn to minimize address translations.
2880 * devmap_roundup() returns residue bytes for next round
2881 * calculations.
2882 */
2883 len = devmap_roundup(dhp, off, len, &pfn, &pgsize);
2884
2885 if (first) {
2886 *laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn);
2887 first = 0;
2888 }
2889
2890 *llen += pgsize;
2891 off = ptob(pfn - dhp->dh_pfn) + pgsize;
2892 }
2893 /* Large page mapping len/addr cover more range than original fault */
2894 ASSERT(*llen >= len && *laddr <= addr);
2895 ASSERT((*laddr + *llen) >= (addr + len));
2896 }
2897
2898 /*
2899 * Initialize the devmap_softlock structure.
2900 */
2901 static struct devmap_softlock *
devmap_softlock_init(dev_t dev,ulong_t id)2902 devmap_softlock_init(dev_t dev, ulong_t id)
2903 {
2904 struct devmap_softlock *slock;
2905 struct devmap_softlock *tmp;
2906
2907 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_INIT,
2908 "devmap_softlock_init:start");
2909
2910 tmp = kmem_zalloc(sizeof (struct devmap_softlock), KM_SLEEP);
2911 mutex_enter(&devmap_slock);
2912
2913 for (slock = devmap_slist; slock != NULL; slock = slock->next)
2914 if ((slock->dev == dev) && (slock->id == id))
2915 break;
2916
2917 if (slock == NULL) {
2918 slock = tmp;
2919 slock->dev = dev;
2920 slock->id = id;
2921 mutex_init(&slock->lock, NULL, MUTEX_DEFAULT, NULL);
2922 cv_init(&slock->cv, NULL, CV_DEFAULT, NULL);
2923 slock->next = devmap_slist;
2924 devmap_slist = slock;
2925 } else
2926 kmem_free(tmp, sizeof (struct devmap_softlock));
2927
2928 mutex_enter(&slock->lock);
2929 slock->refcnt++;
2930 mutex_exit(&slock->lock);
2931 mutex_exit(&devmap_slock);
2932
2933 return (slock);
2934 }
2935
2936 /*
2937 * Wake up processes that sleep on softlocked.
2938 * Free dh_softlock if refcnt is 0.
2939 */
2940 static void
devmap_softlock_rele(devmap_handle_t * dhp)2941 devmap_softlock_rele(devmap_handle_t *dhp)
2942 {
2943 struct devmap_softlock *slock = dhp->dh_softlock;
2944 struct devmap_softlock *tmp;
2945 struct devmap_softlock *parent;
2946
2947 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_RELE,
2948 "devmap_softlock_rele:start");
2949
2950 mutex_enter(&devmap_slock);
2951 mutex_enter(&slock->lock);
2952
2953 ASSERT(slock->refcnt > 0);
2954
2955 slock->refcnt--;
2956
2957 /*
2958 * If no one is using the device, free up the slock data.
2959 */
2960 if (slock->refcnt == 0) {
2961 slock->softlocked = 0;
2962 cv_signal(&slock->cv);
2963
2964 if (devmap_slist == slock)
2965 devmap_slist = slock->next;
2966 else {
2967 parent = devmap_slist;
2968 for (tmp = devmap_slist->next; tmp != NULL;
2969 tmp = tmp->next) {
2970 if (tmp == slock) {
2971 parent->next = tmp->next;
2972 break;
2973 }
2974 parent = tmp;
2975 }
2976 }
2977 mutex_exit(&slock->lock);
2978 mutex_destroy(&slock->lock);
2979 cv_destroy(&slock->cv);
2980 kmem_free(slock, sizeof (struct devmap_softlock));
2981 } else
2982 mutex_exit(&slock->lock);
2983
2984 mutex_exit(&devmap_slock);
2985 }
2986
2987 /*
2988 * Wake up processes that sleep on dh_ctx->locked.
2989 * Free dh_ctx if refcnt is 0.
2990 */
2991 static void
devmap_ctx_rele(devmap_handle_t * dhp)2992 devmap_ctx_rele(devmap_handle_t *dhp)
2993 {
2994 struct devmap_ctx *devctx = dhp->dh_ctx;
2995 struct devmap_ctx *tmp;
2996 struct devmap_ctx *parent;
2997 timeout_id_t tid;
2998
2999 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE,
3000 "devmap_ctx_rele:start");
3001
3002 mutex_enter(&devmapctx_lock);
3003 mutex_enter(&devctx->lock);
3004
3005 ASSERT(devctx->refcnt > 0);
3006
3007 devctx->refcnt--;
3008
3009 /*
3010 * If no one is using the device, free up the devctx data.
3011 */
3012 if (devctx->refcnt == 0) {
3013 /*
3014 * Untimeout any threads using this mapping as they are about
3015 * to go away.
3016 */
3017 if (devctx->timeout != 0) {
3018 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE_CK1,
3019 "devmap_ctx_rele:untimeout ctx->timeout");
3020
3021 tid = devctx->timeout;
3022 mutex_exit(&devctx->lock);
3023 (void) untimeout(tid);
3024 mutex_enter(&devctx->lock);
3025 }
3026
3027 devctx->oncpu = 0;
3028 cv_signal(&devctx->cv);
3029
3030 if (devmapctx_list == devctx)
3031 devmapctx_list = devctx->next;
3032 else {
3033 parent = devmapctx_list;
3034 for (tmp = devmapctx_list->next; tmp != NULL;
3035 tmp = tmp->next) {
3036 if (tmp == devctx) {
3037 parent->next = tmp->next;
3038 break;
3039 }
3040 parent = tmp;
3041 }
3042 }
3043 mutex_exit(&devctx->lock);
3044 mutex_destroy(&devctx->lock);
3045 cv_destroy(&devctx->cv);
3046 kmem_free(devctx, sizeof (struct devmap_ctx));
3047 } else
3048 mutex_exit(&devctx->lock);
3049
3050 mutex_exit(&devmapctx_lock);
3051 }
3052
3053 /*
3054 * devmap_load:
3055 * Marks a segdev segment or pages if offset->offset+len
3056 * is not the entire segment as nointercept and faults in
3057 * the pages in the range offset -> offset+len.
3058 */
3059 int
devmap_load(devmap_cookie_t dhc,offset_t offset,size_t len,uint_t type,uint_t rw)3060 devmap_load(devmap_cookie_t dhc, offset_t offset, size_t len, uint_t type,
3061 uint_t rw)
3062 {
3063 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3064 struct as *asp = dhp->dh_seg->s_as;
3065 caddr_t addr;
3066 ulong_t size;
3067 ssize_t soff; /* offset from the beginning of the segment */
3068 int rc;
3069
3070 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_LOAD,
3071 "devmap_load:start dhp=%p offset=%llx len=%lx",
3072 (void *)dhp, offset, len);
3073
3074 DEBUGF(7, (CE_CONT, "devmap_load: dhp %p offset %llx len %lx\n",
3075 (void *)dhp, offset, len));
3076
3077 /*
3078 * Hat layer only supports devload to process' context for which
3079 * the as lock is held. Verify here and return error if drivers
3080 * inadvertently call devmap_load on a wrong devmap handle.
3081 */
3082 if ((asp != &kas) && !AS_LOCK_HELD(asp, &asp->a_lock))
3083 return (FC_MAKE_ERR(EINVAL));
3084
3085 soff = (ssize_t)(offset - dhp->dh_uoff);
3086 soff = round_down_p2(soff, PAGESIZE);
3087 if (soff < 0 || soff >= dhp->dh_len)
3088 return (FC_MAKE_ERR(EINVAL));
3089
3090 /*
3091 * Address and size must be page aligned. Len is set to the
3092 * number of bytes in the number of pages that are required to
3093 * support len. Offset is set to the byte offset of the first byte
3094 * of the page that contains offset.
3095 */
3096 len = round_up_p2(len, PAGESIZE);
3097
3098 /*
3099 * If len == 0, then calculate the size by getting
3100 * the number of bytes from offset to the end of the segment.
3101 */
3102 if (len == 0)
3103 size = dhp->dh_len - soff;
3104 else {
3105 size = len;
3106 if ((soff + size) > dhp->dh_len)
3107 return (FC_MAKE_ERR(EINVAL));
3108 }
3109
3110 /*
3111 * The address is offset bytes from the base address of
3112 * the segment.
3113 */
3114 addr = (caddr_t)(soff + dhp->dh_uvaddr);
3115
3116 HOLD_DHP_LOCK(dhp);
3117 rc = segdev_faultpages(asp->a_hat,
3118 dhp->dh_seg, addr, size, type, rw, dhp);
3119 RELE_DHP_LOCK(dhp);
3120 return (rc);
3121 }
3122
3123 int
devmap_setup(dev_t dev,offset_t off,struct as * as,caddr_t * addrp,size_t len,uint_t prot,uint_t maxprot,uint_t flags,struct cred * cred)3124 devmap_setup(dev_t dev, offset_t off, struct as *as, caddr_t *addrp,
3125 size_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3126 {
3127 register devmap_handle_t *dhp;
3128 int (*devmap)(dev_t, devmap_cookie_t, offset_t, size_t,
3129 size_t *, uint_t);
3130 int (*mmap)(dev_t, off_t, int);
3131 struct devmap_callback_ctl *callbackops;
3132 devmap_handle_t *dhp_head = NULL;
3133 devmap_handle_t *dhp_prev = NULL;
3134 devmap_handle_t *dhp_curr;
3135 caddr_t addr;
3136 int map_flag;
3137 int ret;
3138 ulong_t total_len;
3139 size_t map_len;
3140 size_t resid_len = len;
3141 offset_t map_off = off;
3142 struct devmap_softlock *slock = NULL;
3143
3144 #ifdef lint
3145 cred = cred;
3146 #endif
3147
3148 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SETUP,
3149 "devmap_setup:start off=%llx len=%lx", off, len);
3150 DEBUGF(3, (CE_CONT, "devmap_setup: off %llx len %lx\n",
3151 off, len));
3152
3153 devmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_devmap;
3154 mmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap;
3155
3156 /*
3157 * driver must provide devmap(9E) entry point in cb_ops to use the
3158 * devmap framework.
3159 */
3160 if (devmap == NULL || devmap == nulldev || devmap == nodev)
3161 return (EINVAL);
3162
3163 /*
3164 * To protect from an inadvertent entry because the devmap entry point
3165 * is not NULL, return error if D_DEVMAP bit is not set in cb_flag and
3166 * mmap is NULL.
3167 */
3168 map_flag = devopsp[getmajor(dev)]->devo_cb_ops->cb_flag;
3169 if ((map_flag & D_DEVMAP) == 0 && (mmap == NULL || mmap == nulldev))
3170 return (EINVAL);
3171
3172 /*
3173 * devmap allows mmap(2) to map multiple registers.
3174 * one devmap_handle is created for each register mapped.
3175 */
3176 for (total_len = 0; total_len < len; total_len += map_len) {
3177 dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP);
3178
3179 if (dhp_prev != NULL)
3180 dhp_prev->dh_next = dhp;
3181 else
3182 dhp_head = dhp;
3183 dhp_prev = dhp;
3184
3185 dhp->dh_prot = prot;
3186 dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot;
3187 dhp->dh_dev = dev;
3188 dhp->dh_timeout_length = CTX_TIMEOUT_VALUE;
3189 dhp->dh_uoff = map_off;
3190
3191 /*
3192 * Get mapping specific info from
3193 * the driver, such as rnumber, roff, len, callbackops,
3194 * accattrp and, if the mapping is for kernel memory,
3195 * ddi_umem_cookie.
3196 */
3197 if ((ret = cdev_devmap(dev, dhp, map_off,
3198 resid_len, &map_len, get_udatamodel())) != 0) {
3199 free_devmap_handle(dhp_head);
3200 return (ENXIO);
3201 }
3202
3203 if (map_len & PAGEOFFSET) {
3204 free_devmap_handle(dhp_head);
3205 return (EINVAL);
3206 }
3207
3208 callbackops = &dhp->dh_callbackops;
3209
3210 if ((callbackops->devmap_access == NULL) ||
3211 (callbackops->devmap_access == nulldev) ||
3212 (callbackops->devmap_access == nodev)) {
3213 /*
3214 * Normally devmap does not support MAP_PRIVATE unless
3215 * the drivers provide a valid devmap_access routine.
3216 */
3217 if ((flags & MAP_PRIVATE) != 0) {
3218 free_devmap_handle(dhp_head);
3219 return (EINVAL);
3220 }
3221 } else {
3222 /*
3223 * Initialize dhp_softlock and dh_ctx if the drivers
3224 * provide devmap_access.
3225 */
3226 dhp->dh_softlock = devmap_softlock_init(dev,
3227 (ulong_t)callbackops->devmap_access);
3228 dhp->dh_ctx = devmap_ctxinit(dev,
3229 (ulong_t)callbackops->devmap_access);
3230
3231 /*
3232 * segdev_fault can only work when all
3233 * dh_softlock in a multi-dhp mapping
3234 * are same. see comments in segdev_fault
3235 * This code keeps track of the first
3236 * dh_softlock allocated in slock and
3237 * compares all later allocations and if
3238 * not similar, returns an error.
3239 */
3240 if (slock == NULL)
3241 slock = dhp->dh_softlock;
3242 if (slock != dhp->dh_softlock) {
3243 free_devmap_handle(dhp_head);
3244 return (ENOTSUP);
3245 }
3246 }
3247
3248 map_off += map_len;
3249 resid_len -= map_len;
3250 }
3251
3252 /*
3253 * get the user virtual address and establish the mapping between
3254 * uvaddr and device physical address.
3255 */
3256 if ((ret = devmap_device(dhp_head, as, addrp, off, len, flags))
3257 != 0) {
3258 /*
3259 * free devmap handles if error during the mapping.
3260 */
3261 free_devmap_handle(dhp_head);
3262
3263 return (ret);
3264 }
3265
3266 /*
3267 * call the driver's devmap_map callback to do more after the mapping,
3268 * such as to allocate driver private data for context management.
3269 */
3270 dhp = dhp_head;
3271 map_off = off;
3272 addr = *addrp;
3273 while (dhp != NULL) {
3274 callbackops = &dhp->dh_callbackops;
3275 dhp->dh_uvaddr = addr;
3276 dhp_curr = dhp;
3277 if (callbackops->devmap_map != NULL) {
3278 ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp,
3279 dev, flags, map_off,
3280 dhp->dh_len, &dhp->dh_pvtp);
3281 if (ret != 0) {
3282 struct segdev_data *sdp;
3283
3284 /*
3285 * call driver's devmap_unmap entry point
3286 * to free driver resources.
3287 */
3288 dhp = dhp_head;
3289 map_off = off;
3290 while (dhp != dhp_curr) {
3291 callbackops = &dhp->dh_callbackops;
3292 if (callbackops->devmap_unmap != NULL) {
3293 (*callbackops->devmap_unmap)(
3294 dhp, dhp->dh_pvtp,
3295 map_off, dhp->dh_len,
3296 NULL, NULL, NULL, NULL);
3297 }
3298 map_off += dhp->dh_len;
3299 dhp = dhp->dh_next;
3300 }
3301 sdp = dhp_head->dh_seg->s_data;
3302 sdp->devmap_data = NULL;
3303 free_devmap_handle(dhp_head);
3304 return (ENXIO);
3305 }
3306 }
3307 map_off += dhp->dh_len;
3308 addr += dhp->dh_len;
3309 dhp = dhp->dh_next;
3310 }
3311
3312 return (0);
3313 }
3314
3315 int
ddi_devmap_segmap(dev_t dev,off_t off,ddi_as_handle_t as,caddr_t * addrp,off_t len,uint_t prot,uint_t maxprot,uint_t flags,struct cred * cred)3316 ddi_devmap_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp,
3317 off_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3318 {
3319 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP,
3320 "devmap_segmap:start");
3321 return (devmap_setup(dev, (offset_t)off, (struct as *)as, addrp,
3322 (size_t)len, prot, maxprot, flags, cred));
3323 }
3324
3325 /*
3326 * Called from devmap_devmem_setup/remap to see if can use large pages for
3327 * this device mapping.
3328 * Also calculate the max. page size for this mapping.
3329 * this page size will be used in fault routine for
3330 * optimal page size calculations.
3331 */
3332 static void
devmap_devmem_large_page_setup(devmap_handle_t * dhp)3333 devmap_devmem_large_page_setup(devmap_handle_t *dhp)
3334 {
3335 ASSERT(dhp_is_devmem(dhp));
3336 dhp->dh_mmulevel = 0;
3337
3338 /*
3339 * use large page size only if:
3340 * 1. device memory.
3341 * 2. mmu supports multiple page sizes,
3342 * 3. Driver did not disallow it
3343 * 4. dhp length is at least as big as the large pagesize
3344 * 5. the uvaddr and pfn are large pagesize aligned
3345 */
3346 if (page_num_pagesizes() > 1 &&
3347 !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) {
3348 ulong_t base;
3349 int level;
3350
3351 base = (ulong_t)ptob(dhp->dh_pfn);
3352 for (level = 1; level < page_num_pagesizes(); level++) {
3353 size_t pgsize = page_get_pagesize(level);
3354 if ((dhp->dh_len < pgsize) ||
3355 (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr,
3356 base, pgsize))) {
3357 break;
3358 }
3359 }
3360 dhp->dh_mmulevel = level - 1;
3361 }
3362 if (dhp->dh_mmulevel > 0) {
3363 dhp->dh_flags |= DEVMAP_FLAG_LARGE;
3364 } else {
3365 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3366 }
3367 }
3368
3369 /*
3370 * Called by driver devmap routine to pass device specific info to
3371 * the framework. used for device memory mapping only.
3372 */
3373 int
devmap_devmem_setup(devmap_cookie_t dhc,dev_info_t * dip,struct devmap_callback_ctl * callbackops,uint_t rnumber,offset_t roff,size_t len,uint_t maxprot,uint_t flags,ddi_device_acc_attr_t * accattrp)3374 devmap_devmem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3375 struct devmap_callback_ctl *callbackops, uint_t rnumber, offset_t roff,
3376 size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp)
3377 {
3378 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3379 ddi_acc_handle_t handle;
3380 ddi_map_req_t mr;
3381 ddi_acc_hdl_t *hp;
3382 int err;
3383
3384 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_SETUP,
3385 "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3386 (void *)dhp, roff, rnumber, (uint_t)len);
3387 DEBUGF(2, (CE_CONT, "devmap_devmem_setup: dhp %p offset %llx "
3388 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3389
3390 /*
3391 * First to check if this function has been called for this dhp.
3392 */
3393 if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3394 return (DDI_FAILURE);
3395
3396 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3397 return (DDI_FAILURE);
3398
3399 if (flags & DEVMAP_MAPPING_INVALID) {
3400 /*
3401 * Don't go up the tree to get pfn if the driver specifies
3402 * DEVMAP_MAPPING_INVALID in flags.
3403 *
3404 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3405 * remap permission.
3406 */
3407 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3408 return (DDI_FAILURE);
3409 }
3410 dhp->dh_pfn = PFN_INVALID;
3411 } else {
3412 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3413 if (handle == NULL)
3414 return (DDI_FAILURE);
3415
3416 hp = impl_acc_hdl_get(handle);
3417 hp->ah_vers = VERS_ACCHDL;
3418 hp->ah_dip = dip;
3419 hp->ah_rnumber = rnumber;
3420 hp->ah_offset = roff;
3421 hp->ah_len = len;
3422 if (accattrp != NULL)
3423 hp->ah_acc = *accattrp;
3424
3425 mr.map_op = DDI_MO_MAP_LOCKED;
3426 mr.map_type = DDI_MT_RNUMBER;
3427 mr.map_obj.rnumber = rnumber;
3428 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3429 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3430 mr.map_handlep = hp;
3431 mr.map_vers = DDI_MAP_VERSION;
3432
3433 /*
3434 * up the device tree to get pfn.
3435 * The rootnex_map_regspec() routine in nexus drivers has been
3436 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3437 */
3438 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn);
3439 dhp->dh_hat_attr = hp->ah_hat_flags;
3440 impl_acc_hdl_free(handle);
3441
3442 if (err)
3443 return (DDI_FAILURE);
3444 }
3445 /* Should not be using devmem setup for memory pages */
3446 ASSERT(!pf_is_memory(dhp->dh_pfn));
3447
3448 /* Only some of the flags bits are settable by the driver */
3449 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3450 dhp->dh_len = ptob(btopr(len));
3451
3452 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3453 dhp->dh_roff = ptob(btop(roff));
3454
3455 /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3456 devmap_devmem_large_page_setup(dhp);
3457 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3458 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3459
3460
3461 if (callbackops != NULL) {
3462 bcopy(callbackops, &dhp->dh_callbackops,
3463 sizeof (struct devmap_callback_ctl));
3464 }
3465
3466 /*
3467 * Initialize dh_lock if we want to do remap.
3468 */
3469 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3470 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3471 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3472 }
3473
3474 dhp->dh_flags |= DEVMAP_SETUP_DONE;
3475
3476 return (DDI_SUCCESS);
3477 }
3478
3479 int
devmap_devmem_remap(devmap_cookie_t dhc,dev_info_t * dip,uint_t rnumber,offset_t roff,size_t len,uint_t maxprot,uint_t flags,ddi_device_acc_attr_t * accattrp)3480 devmap_devmem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3481 uint_t rnumber, offset_t roff, size_t len, uint_t maxprot,
3482 uint_t flags, ddi_device_acc_attr_t *accattrp)
3483 {
3484 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3485 ddi_acc_handle_t handle;
3486 ddi_map_req_t mr;
3487 ddi_acc_hdl_t *hp;
3488 pfn_t pfn;
3489 uint_t hat_flags;
3490 int err;
3491
3492 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_REMAP,
3493 "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3494 (void *)dhp, roff, rnumber, (uint_t)len);
3495 DEBUGF(2, (CE_CONT, "devmap_devmem_remap: dhp %p offset %llx "
3496 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3497
3498 /*
3499 * Return failure if setup has not been done or no remap permission
3500 * has been granted during the setup.
3501 */
3502 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3503 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3504 return (DDI_FAILURE);
3505
3506 /* Only DEVMAP_MAPPING_INVALID flag supported for remap */
3507 if ((flags != 0) && (flags != DEVMAP_MAPPING_INVALID))
3508 return (DDI_FAILURE);
3509
3510 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3511 return (DDI_FAILURE);
3512
3513 if (!(flags & DEVMAP_MAPPING_INVALID)) {
3514 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3515 if (handle == NULL)
3516 return (DDI_FAILURE);
3517 }
3518
3519 HOLD_DHP_LOCK(dhp);
3520
3521 /*
3522 * Unload the old mapping, so next fault will setup the new mappings
3523 * Do this while holding the dhp lock so other faults dont reestablish
3524 * the mappings
3525 */
3526 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3527 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3528
3529 if (flags & DEVMAP_MAPPING_INVALID) {
3530 dhp->dh_flags |= DEVMAP_MAPPING_INVALID;
3531 dhp->dh_pfn = PFN_INVALID;
3532 } else {
3533 /* clear any prior DEVMAP_MAPPING_INVALID flag */
3534 dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID;
3535 hp = impl_acc_hdl_get(handle);
3536 hp->ah_vers = VERS_ACCHDL;
3537 hp->ah_dip = dip;
3538 hp->ah_rnumber = rnumber;
3539 hp->ah_offset = roff;
3540 hp->ah_len = len;
3541 if (accattrp != NULL)
3542 hp->ah_acc = *accattrp;
3543
3544 mr.map_op = DDI_MO_MAP_LOCKED;
3545 mr.map_type = DDI_MT_RNUMBER;
3546 mr.map_obj.rnumber = rnumber;
3547 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3548 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3549 mr.map_handlep = hp;
3550 mr.map_vers = DDI_MAP_VERSION;
3551
3552 /*
3553 * up the device tree to get pfn.
3554 * The rootnex_map_regspec() routine in nexus drivers has been
3555 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3556 */
3557 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&pfn);
3558 hat_flags = hp->ah_hat_flags;
3559 impl_acc_hdl_free(handle);
3560 if (err) {
3561 RELE_DHP_LOCK(dhp);
3562 return (DDI_FAILURE);
3563 }
3564 /*
3565 * Store result of ddi_map first in local variables, as we do
3566 * not want to overwrite the existing dhp with wrong data.
3567 */
3568 dhp->dh_pfn = pfn;
3569 dhp->dh_hat_attr = hat_flags;
3570 }
3571
3572 /* clear the large page size flag */
3573 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3574
3575 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3576 dhp->dh_roff = ptob(btop(roff));
3577
3578 /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3579 devmap_devmem_large_page_setup(dhp);
3580 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3581 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3582
3583 RELE_DHP_LOCK(dhp);
3584 return (DDI_SUCCESS);
3585 }
3586
3587 /*
3588 * called by driver devmap routine to pass kernel virtual address mapping
3589 * info to the framework. used only for kernel memory
3590 * allocated from ddi_umem_alloc().
3591 */
3592 int
devmap_umem_setup(devmap_cookie_t dhc,dev_info_t * dip,struct devmap_callback_ctl * callbackops,ddi_umem_cookie_t cookie,offset_t off,size_t len,uint_t maxprot,uint_t flags,ddi_device_acc_attr_t * accattrp)3593 devmap_umem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3594 struct devmap_callback_ctl *callbackops, ddi_umem_cookie_t cookie,
3595 offset_t off, size_t len, uint_t maxprot, uint_t flags,
3596 ddi_device_acc_attr_t *accattrp)
3597 {
3598 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3599 struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3600
3601 #ifdef lint
3602 dip = dip;
3603 #endif
3604
3605 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_SETUP,
3606 "devmap_umem_setup:start dhp=%p offset=%llx cookie=%p len=%lx",
3607 (void *)dhp, off, cookie, len);
3608 DEBUGF(2, (CE_CONT, "devmap_umem_setup: dhp %p offset %llx "
3609 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3610
3611 if (cookie == NULL)
3612 return (DDI_FAILURE);
3613
3614 /* For UMEM_TRASH, this restriction is not needed */
3615 if ((off + len) > cp->size)
3616 return (DDI_FAILURE);
3617
3618 /* check if the cache attributes are supported */
3619 if (i_ddi_check_cache_attr(flags) == B_FALSE)
3620 return (DDI_FAILURE);
3621
3622 /*
3623 * First to check if this function has been called for this dhp.
3624 */
3625 if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3626 return (DDI_FAILURE);
3627
3628 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3629 return (DDI_FAILURE);
3630
3631 if (flags & DEVMAP_MAPPING_INVALID) {
3632 /*
3633 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3634 * remap permission.
3635 */
3636 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3637 return (DDI_FAILURE);
3638 }
3639 } else {
3640 dhp->dh_cookie = cookie;
3641 dhp->dh_roff = ptob(btop(off));
3642 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3643 /* set HAT cache attributes */
3644 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3645 /* set HAT endianess attributes */
3646 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3647 }
3648
3649 /*
3650 * The default is _not_ to pass HAT_LOAD_NOCONSIST to hat_devload();
3651 * we pass HAT_LOAD_NOCONSIST _only_ in cases where hat tries to
3652 * create consistent mappings but our intention was to create
3653 * non-consistent mappings.
3654 *
3655 * DEVMEM: hat figures it out it's DEVMEM and creates non-consistent
3656 * mappings.
3657 *
3658 * kernel exported memory: hat figures it out it's memory and always
3659 * creates consistent mappings.
3660 *
3661 * /dev/mem: non-consistent mappings. See comments in common/io/mem.c
3662 *
3663 * /dev/kmem: consistent mappings are created unless they are
3664 * MAP_FIXED. We _explicitly_ tell hat to create non-consistent
3665 * mappings by passing HAT_LOAD_NOCONSIST in case of MAP_FIXED
3666 * mappings of /dev/kmem. See common/io/mem.c
3667 */
3668
3669 /* Only some of the flags bits are settable by the driver */
3670 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3671
3672 dhp->dh_len = ptob(btopr(len));
3673 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3674 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3675
3676 if (callbackops != NULL) {
3677 bcopy(callbackops, &dhp->dh_callbackops,
3678 sizeof (struct devmap_callback_ctl));
3679 }
3680 /*
3681 * Initialize dh_lock if we want to do remap.
3682 */
3683 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3684 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3685 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3686 }
3687
3688 dhp->dh_flags |= DEVMAP_SETUP_DONE;
3689
3690 return (DDI_SUCCESS);
3691 }
3692
3693 int
devmap_umem_remap(devmap_cookie_t dhc,dev_info_t * dip,ddi_umem_cookie_t cookie,offset_t off,size_t len,uint_t maxprot,uint_t flags,ddi_device_acc_attr_t * accattrp)3694 devmap_umem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3695 ddi_umem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot,
3696 uint_t flags, ddi_device_acc_attr_t *accattrp)
3697 {
3698 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3699 struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3700
3701 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_REMAP,
3702 "devmap_umem_remap:start dhp=%p offset=%llx cookie=%p len=%lx",
3703 (void *)dhp, off, cookie, len);
3704 DEBUGF(2, (CE_CONT, "devmap_umem_remap: dhp %p offset %llx "
3705 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3706
3707 #ifdef lint
3708 dip = dip;
3709 accattrp = accattrp;
3710 #endif
3711 /*
3712 * Reture failure if setup has not been done or no remap permission
3713 * has been granted during the setup.
3714 */
3715 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3716 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3717 return (DDI_FAILURE);
3718
3719 /* No flags supported for remap yet */
3720 if (flags != 0)
3721 return (DDI_FAILURE);
3722
3723 /* check if the cache attributes are supported */
3724 if (i_ddi_check_cache_attr(flags) == B_FALSE)
3725 return (DDI_FAILURE);
3726
3727 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3728 return (DDI_FAILURE);
3729
3730 /* For UMEM_TRASH, this restriction is not needed */
3731 if ((off + len) > cp->size)
3732 return (DDI_FAILURE);
3733
3734 HOLD_DHP_LOCK(dhp);
3735 /*
3736 * Unload the old mapping, so next fault will setup the new mappings
3737 * Do this while holding the dhp lock so other faults dont reestablish
3738 * the mappings
3739 */
3740 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3741 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3742
3743 dhp->dh_cookie = cookie;
3744 dhp->dh_roff = ptob(btop(off));
3745 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3746 /* set HAT cache attributes */
3747 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3748 /* set HAT endianess attributes */
3749 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3750
3751 /* clear the large page size flag */
3752 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3753
3754 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3755 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3756 RELE_DHP_LOCK(dhp);
3757 return (DDI_SUCCESS);
3758 }
3759
3760 /*
3761 * to set timeout value for the driver's context management callback, e.g.
3762 * devmap_access().
3763 */
3764 void
devmap_set_ctx_timeout(devmap_cookie_t dhc,clock_t ticks)3765 devmap_set_ctx_timeout(devmap_cookie_t dhc, clock_t ticks)
3766 {
3767 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3768
3769 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SET_CTX_TIMEOUT,
3770 "devmap_set_ctx_timeout:start dhp=%p ticks=%x",
3771 (void *)dhp, ticks);
3772 dhp->dh_timeout_length = ticks;
3773 }
3774
3775 int
devmap_default_access(devmap_cookie_t dhp,void * pvtp,offset_t off,size_t len,uint_t type,uint_t rw)3776 devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off,
3777 size_t len, uint_t type, uint_t rw)
3778 {
3779 #ifdef lint
3780 pvtp = pvtp;
3781 #endif
3782
3783 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DEFAULT_ACCESS,
3784 "devmap_default_access:start");
3785 return (devmap_load(dhp, off, len, type, rw));
3786 }
3787
3788 /*
3789 * segkmem_alloc() wrapper to allocate memory which is both
3790 * non-relocatable (for DR) and sharelocked, since the rest
3791 * of this segment driver requires it.
3792 */
3793 static void *
devmap_alloc_pages(vmem_t * vmp,size_t size,int vmflag)3794 devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag)
3795 {
3796 ASSERT(vmp != NULL);
3797 ASSERT(kvseg.s_base != NULL);
3798 vmflag |= (VM_NORELOC | SEGKMEM_SHARELOCKED);
3799 return (segkmem_alloc(vmp, size, vmflag));
3800 }
3801
3802 /*
3803 * This is where things are a bit incestuous with seg_kmem: unlike
3804 * seg_kp, seg_kmem does not keep its pages long-term sharelocked, so
3805 * we need to do a bit of a dance around that to prevent duplication of
3806 * code until we decide to bite the bullet and implement a new kernel
3807 * segment for driver-allocated memory that is exported to user space.
3808 */
3809 static void
devmap_free_pages(vmem_t * vmp,void * inaddr,size_t size)3810 devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size)
3811 {
3812 page_t *pp;
3813 caddr_t addr = inaddr;
3814 caddr_t eaddr;
3815 pgcnt_t npages = btopr(size);
3816
3817 ASSERT(vmp != NULL);
3818 ASSERT(kvseg.s_base != NULL);
3819 ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
3820
3821 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
3822
3823 for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
3824 /*
3825 * Use page_find() instead of page_lookup() to find the page
3826 * since we know that it is hashed and has a shared lock.
3827 */
3828 pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr);
3829
3830 if (pp == NULL)
3831 panic("devmap_free_pages: page not found");
3832 if (!page_tryupgrade(pp)) {
3833 page_unlock(pp);
3834 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr,
3835 SE_EXCL);
3836 if (pp == NULL)
3837 panic("devmap_free_pages: page already freed");
3838 }
3839 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
3840 pp->p_lckcnt = 0;
3841 page_destroy(pp, 0);
3842 }
3843 page_unresv(npages);
3844
3845 if (vmp != NULL)
3846 vmem_free(vmp, inaddr, size);
3847 }
3848
3849 /*
3850 * devmap_umem_alloc_np() replaces kmem_zalloc() as the method for
3851 * allocating non-pageable kmem in response to a ddi_umem_alloc()
3852 * default request. For now we allocate our own pages and we keep
3853 * them long-term sharelocked, since: A) the fault routines expect the
3854 * memory to already be locked; B) pageable umem is already long-term
3855 * locked; C) it's a lot of work to make it otherwise, particularly
3856 * since the nexus layer expects the pages to never fault. An RFE is to
3857 * not keep the pages long-term locked, but instead to be able to
3858 * take faults on them and simply look them up in kvp in case we
3859 * fault on them. Even then, we must take care not to let pageout
3860 * steal them from us since the data must remain resident; if we
3861 * do this we must come up with some way to pin the pages to prevent
3862 * faults while a driver is doing DMA to/from them.
3863 */
3864 static void *
devmap_umem_alloc_np(size_t size,size_t flags)3865 devmap_umem_alloc_np(size_t size, size_t flags)
3866 {
3867 void *buf;
3868 int vmflags = (flags & DDI_UMEM_NOSLEEP)? VM_NOSLEEP : VM_SLEEP;
3869
3870 buf = vmem_alloc(umem_np_arena, size, vmflags);
3871 if (buf != NULL)
3872 bzero(buf, size);
3873 return (buf);
3874 }
3875
3876 static void
devmap_umem_free_np(void * addr,size_t size)3877 devmap_umem_free_np(void *addr, size_t size)
3878 {
3879 vmem_free(umem_np_arena, addr, size);
3880 }
3881
3882 /*
3883 * allocate page aligned kernel memory for exporting to user land.
3884 * The devmap framework will use the cookie allocated by ddi_umem_alloc()
3885 * to find a user virtual address that is in same color as the address
3886 * allocated here.
3887 */
3888 void *
ddi_umem_alloc(size_t size,int flags,ddi_umem_cookie_t * cookie)3889 ddi_umem_alloc(size_t size, int flags, ddi_umem_cookie_t *cookie)
3890 {
3891 register size_t len = ptob(btopr(size));
3892 void *buf = NULL;
3893 struct ddi_umem_cookie *cp;
3894 int iflags = 0;
3895
3896 *cookie = NULL;
3897
3898 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_ALLOC,
3899 "devmap_umem_alloc:start");
3900 if (len == 0)
3901 return ((void *)NULL);
3902
3903 /*
3904 * allocate cookie
3905 */
3906 if ((cp = kmem_zalloc(sizeof (struct ddi_umem_cookie),
3907 flags & DDI_UMEM_NOSLEEP ? KM_NOSLEEP : KM_SLEEP)) == NULL) {
3908 ASSERT(flags & DDI_UMEM_NOSLEEP);
3909 return ((void *)NULL);
3910 }
3911
3912 if (flags & DDI_UMEM_PAGEABLE) {
3913 /* Only one of the flags is allowed */
3914 ASSERT(!(flags & DDI_UMEM_TRASH));
3915 /* initialize resource with 0 */
3916 iflags = KPD_ZERO;
3917
3918 /*
3919 * to allocate unlocked pageable memory, use segkp_get() to
3920 * create a segkp segment. Since segkp can only service kas,
3921 * other segment drivers such as segdev have to do
3922 * as_fault(segkp, SOFTLOCK) in its fault routine,
3923 */
3924 if (flags & DDI_UMEM_NOSLEEP)
3925 iflags |= KPD_NOWAIT;
3926
3927 if ((buf = segkp_get(segkp, len, iflags)) == NULL) {
3928 kmem_free(cp, sizeof (struct ddi_umem_cookie));
3929 return ((void *)NULL);
3930 }
3931 cp->type = KMEM_PAGEABLE;
3932 mutex_init(&cp->lock, NULL, MUTEX_DEFAULT, NULL);
3933 cp->locked = 0;
3934 } else if (flags & DDI_UMEM_TRASH) {
3935 /* Only one of the flags is allowed */
3936 ASSERT(!(flags & DDI_UMEM_PAGEABLE));
3937 cp->type = UMEM_TRASH;
3938 buf = NULL;
3939 } else {
3940 if ((buf = devmap_umem_alloc_np(len, flags)) == NULL) {
3941 kmem_free(cp, sizeof (struct ddi_umem_cookie));
3942 return ((void *)NULL);
3943 }
3944
3945 cp->type = KMEM_NON_PAGEABLE;
3946 }
3947
3948 /*
3949 * need to save size here. size will be used when
3950 * we do kmem_free.
3951 */
3952 cp->size = len;
3953 cp->cvaddr = (caddr_t)buf;
3954
3955 *cookie = (void *)cp;
3956 return (buf);
3957 }
3958
3959 void
ddi_umem_free(ddi_umem_cookie_t cookie)3960 ddi_umem_free(ddi_umem_cookie_t cookie)
3961 {
3962 struct ddi_umem_cookie *cp;
3963
3964 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_FREE,
3965 "devmap_umem_free:start");
3966
3967 /*
3968 * if cookie is NULL, no effects on the system
3969 */
3970 if (cookie == NULL)
3971 return;
3972
3973 cp = (struct ddi_umem_cookie *)cookie;
3974
3975 switch (cp->type) {
3976 case KMEM_PAGEABLE :
3977 ASSERT(cp->cvaddr != NULL && cp->size != 0);
3978 /*
3979 * Check if there are still any pending faults on the cookie
3980 * while the driver is deleting it,
3981 * XXX - could change to an ASSERT but wont catch errant drivers
3982 */
3983 mutex_enter(&cp->lock);
3984 if (cp->locked) {
3985 mutex_exit(&cp->lock);
3986 panic("ddi_umem_free for cookie with pending faults %p",
3987 (void *)cp);
3988 return;
3989 }
3990
3991 segkp_release(segkp, cp->cvaddr);
3992
3993 /*
3994 * release mutex associated with this cookie.
3995 */
3996 mutex_destroy(&cp->lock);
3997 break;
3998 case KMEM_NON_PAGEABLE :
3999 ASSERT(cp->cvaddr != NULL && cp->size != 0);
4000 devmap_umem_free_np(cp->cvaddr, cp->size);
4001 break;
4002 case UMEM_TRASH :
4003 break;
4004 case UMEM_LOCKED :
4005 /* Callers should use ddi_umem_unlock for this type */
4006 ddi_umem_unlock(cookie);
4007 /* Frees the cookie too */
4008 return;
4009 default:
4010 /* panic so we can diagnose the underlying cause */
4011 panic("ddi_umem_free: illegal cookie type 0x%x\n",
4012 cp->type);
4013 }
4014
4015 kmem_free(cookie, sizeof (struct ddi_umem_cookie));
4016 }
4017
4018
4019 static int
segdev_getmemid(struct seg * seg,caddr_t addr,memid_t * memidp)4020 segdev_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
4021 {
4022 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4023
4024 /*
4025 * It looks as if it is always mapped shared
4026 */
4027 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GETMEMID,
4028 "segdev_getmemid:start");
4029 memidp->val[0] = (uintptr_t)VTOCVP(sdp->vp);
4030 memidp->val[1] = sdp->offset + (uintptr_t)(addr - seg->s_base);
4031 return (0);
4032 }
4033
4034 /*ARGSUSED*/
4035 static lgrp_mem_policy_info_t *
segdev_getpolicy(struct seg * seg,caddr_t addr)4036 segdev_getpolicy(struct seg *seg, caddr_t addr)
4037 {
4038 return (NULL);
4039 }
4040
4041 /*ARGSUSED*/
4042 static int
segdev_capable(struct seg * seg,segcapability_t capability)4043 segdev_capable(struct seg *seg, segcapability_t capability)
4044 {
4045 return (0);
4046 }
4047
4048 /*
4049 * ddi_umem_alloc() non-pageable quantum cache max size.
4050 * This is just a SWAG.
4051 */
4052 #define DEVMAP_UMEM_QUANTUM (8*PAGESIZE)
4053
4054 /*
4055 * Initialize seg_dev from boot. This routine sets up the trash page
4056 * and creates the umem_np_arena used to back non-pageable memory
4057 * requests.
4058 */
4059 void
segdev_init(void)4060 segdev_init(void)
4061 {
4062 struct seg kseg;
4063
4064 umem_np_arena = vmem_create("umem_np", NULL, 0, PAGESIZE,
4065 devmap_alloc_pages, devmap_free_pages, heap_arena,
4066 DEVMAP_UMEM_QUANTUM, VM_SLEEP);
4067
4068 kseg.s_as = &kas;
4069 trashpp = page_create_va(&trashvp, 0, PAGESIZE,
4070 PG_NORELOC | PG_EXCL | PG_WAIT, &kseg, NULL);
4071 if (trashpp == NULL)
4072 panic("segdev_init: failed to create trash page");
4073 pagezero(trashpp, 0, PAGESIZE);
4074 page_downgrade(trashpp);
4075 }
4076
4077 /*
4078 * Invoke platform-dependent support routines so that /proc can have
4079 * the platform code deal with curious hardware.
4080 */
4081 int
segdev_copyfrom(struct seg * seg,caddr_t uaddr,const void * devaddr,void * kaddr,size_t len)4082 segdev_copyfrom(struct seg *seg,
4083 caddr_t uaddr, const void *devaddr, void *kaddr, size_t len)
4084 {
4085 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4086 struct snode *sp = VTOS(VTOCVP(sdp->vp));
4087
4088 return (e_ddi_copyfromdev(sp->s_dip,
4089 (off_t)(uaddr - seg->s_base), devaddr, kaddr, len));
4090 }
4091
4092 int
segdev_copyto(struct seg * seg,caddr_t uaddr,const void * kaddr,void * devaddr,size_t len)4093 segdev_copyto(struct seg *seg,
4094 caddr_t uaddr, const void *kaddr, void *devaddr, size_t len)
4095 {
4096 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4097 struct snode *sp = VTOS(VTOCVP(sdp->vp));
4098
4099 return (e_ddi_copytodev(sp->s_dip,
4100 (off_t)(uaddr - seg->s_base), kaddr, devaddr, len));
4101 }
4102