Print this page
6149 use NULL capable segop as a shorthand for no-capabilities
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/vm/seg_dev.c
+++ new/usr/src/uts/common/vm/seg_dev.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright 2010 Sun Microsystems, Inc. All rights reserved.
24 24 * Use is subject to license terms.
25 25 */
26 26
27 27 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
28 28 /* All Rights Reserved */
29 29
30 30 /*
31 31 * University Copyright- Copyright (c) 1982, 1986, 1988
32 32 * The Regents of the University of California
33 33 * All Rights Reserved
34 34 *
35 35 * University Acknowledgment- Portions of this document are derived from
36 36 * software developed by the University of California, Berkeley, and its
37 37 * contributors.
38 38 */
39 39
40 40 /*
41 41 * VM - segment of a mapped device.
42 42 *
43 43 * This segment driver is used when mapping character special devices.
44 44 */
45 45
46 46 #include <sys/types.h>
47 47 #include <sys/t_lock.h>
48 48 #include <sys/sysmacros.h>
49 49 #include <sys/vtrace.h>
50 50 #include <sys/systm.h>
51 51 #include <sys/vmsystm.h>
52 52 #include <sys/mman.h>
53 53 #include <sys/errno.h>
54 54 #include <sys/kmem.h>
55 55 #include <sys/cmn_err.h>
56 56 #include <sys/vnode.h>
57 57 #include <sys/proc.h>
58 58 #include <sys/conf.h>
59 59 #include <sys/debug.h>
60 60 #include <sys/ddidevmap.h>
61 61 #include <sys/ddi_implfuncs.h>
62 62 #include <sys/lgrp.h>
63 63
64 64 #include <vm/page.h>
65 65 #include <vm/hat.h>
66 66 #include <vm/as.h>
67 67 #include <vm/seg.h>
68 68 #include <vm/seg_dev.h>
69 69 #include <vm/seg_kp.h>
70 70 #include <vm/seg_kmem.h>
71 71 #include <vm/vpage.h>
72 72
73 73 #include <sys/sunddi.h>
74 74 #include <sys/esunddi.h>
75 75 #include <sys/fs/snode.h>
76 76
77 77
78 78 #if DEBUG
79 79 int segdev_debug;
80 80 #define DEBUGF(level, args) { if (segdev_debug >= (level)) cmn_err args; }
81 81 #else
82 82 #define DEBUGF(level, args)
83 83 #endif
84 84
85 85 /* Default timeout for devmap context management */
86 86 #define CTX_TIMEOUT_VALUE 0
87 87
88 88 #define HOLD_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
89 89 { mutex_enter(&dhp->dh_lock); }
90 90
91 91 #define RELE_DHP_LOCK(dhp) if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) \
92 92 { mutex_exit(&dhp->dh_lock); }
93 93
94 94 #define round_down_p2(a, s) ((a) & ~((s) - 1))
95 95 #define round_up_p2(a, s) (((a) + (s) - 1) & ~((s) - 1))
96 96
97 97 /*
98 98 * VA_PA_ALIGNED checks to see if both VA and PA are on pgsize boundary
99 99 * VA_PA_PGSIZE_ALIGNED check to see if VA is aligned with PA w.r.t. pgsize
100 100 */
101 101 #define VA_PA_ALIGNED(uvaddr, paddr, pgsize) \
102 102 (((uvaddr | paddr) & (pgsize - 1)) == 0)
103 103 #define VA_PA_PGSIZE_ALIGNED(uvaddr, paddr, pgsize) \
104 104 (((uvaddr ^ paddr) & (pgsize - 1)) == 0)
105 105
106 106 #define vpgtob(n) ((n) * sizeof (struct vpage)) /* For brevity */
107 107
108 108 #define VTOCVP(vp) (VTOS(vp)->s_commonvp) /* we "know" it's an snode */
109 109
110 110 static struct devmap_ctx *devmapctx_list = NULL;
111 111 static struct devmap_softlock *devmap_slist = NULL;
112 112
113 113 /*
114 114 * mutex, vnode and page for the page of zeros we use for the trash mappings.
115 115 * One trash page is allocated on the first ddi_umem_setup call that uses it
116 116 * XXX Eventually, we may want to combine this with what segnf does when all
117 117 * hat layers implement HAT_NOFAULT.
118 118 *
119 119 * The trash page is used when the backing store for a userland mapping is
120 120 * removed but the application semantics do not take kindly to a SIGBUS.
121 121 * In that scenario, the applications pages are mapped to some dummy page
122 122 * which returns garbage on read and writes go into a common place.
123 123 * (Perfect for NO_FAULT semantics)
124 124 * The device driver is responsible to communicating to the app with some
125 125 * other mechanism that such remapping has happened and the app should take
126 126 * corrective action.
127 127 * We can also use an anonymous memory page as there is no requirement to
128 128 * keep the page locked, however this complicates the fault code. RFE.
129 129 */
130 130 static struct vnode trashvp;
131 131 static struct page *trashpp;
132 132
133 133 /* Non-pageable kernel memory is allocated from the umem_np_arena. */
134 134 static vmem_t *umem_np_arena;
135 135
136 136 /* Set the cookie to a value we know will never be a valid umem_cookie */
137 137 #define DEVMAP_DEVMEM_COOKIE ((ddi_umem_cookie_t)0x1)
138 138
139 139 /*
140 140 * Macros to check if type of devmap handle
141 141 */
142 142 #define cookie_is_devmem(c) \
143 143 ((c) == (struct ddi_umem_cookie *)DEVMAP_DEVMEM_COOKIE)
144 144
145 145 #define cookie_is_pmem(c) \
146 146 ((c) == (struct ddi_umem_cookie *)DEVMAP_PMEM_COOKIE)
147 147
148 148 #define cookie_is_kpmem(c) (!cookie_is_devmem(c) && !cookie_is_pmem(c) &&\
149 149 ((c)->type == KMEM_PAGEABLE))
150 150
151 151 #define dhp_is_devmem(dhp) \
152 152 (cookie_is_devmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
153 153
154 154 #define dhp_is_pmem(dhp) \
155 155 (cookie_is_pmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
156 156
157 157 #define dhp_is_kpmem(dhp) \
158 158 (cookie_is_kpmem((struct ddi_umem_cookie *)((dhp)->dh_cookie)))
159 159
160 160 /*
161 161 * Private seg op routines.
162 162 */
163 163 static int segdev_dup(struct seg *, struct seg *);
164 164 static int segdev_unmap(struct seg *, caddr_t, size_t);
165 165 static void segdev_free(struct seg *);
166 166 static faultcode_t segdev_fault(struct hat *, struct seg *, caddr_t, size_t,
167 167 enum fault_type, enum seg_rw);
168 168 static faultcode_t segdev_faulta(struct seg *, caddr_t);
169 169 static int segdev_setprot(struct seg *, caddr_t, size_t, uint_t);
170 170 static int segdev_checkprot(struct seg *, caddr_t, size_t, uint_t);
171 171 static void segdev_badop(void);
172 172 static int segdev_sync(struct seg *, caddr_t, size_t, int, uint_t);
173 173 static size_t segdev_incore(struct seg *, caddr_t, size_t, char *);
174 174 static int segdev_lockop(struct seg *, caddr_t, size_t, int, int,
175 175 ulong_t *, size_t);
|
↓ open down ↓ |
175 lines elided |
↑ open up ↑ |
176 176 static int segdev_getprot(struct seg *, caddr_t, size_t, uint_t *);
177 177 static u_offset_t segdev_getoffset(struct seg *, caddr_t);
178 178 static int segdev_gettype(struct seg *, caddr_t);
179 179 static int segdev_getvp(struct seg *, caddr_t, struct vnode **);
180 180 static int segdev_advise(struct seg *, caddr_t, size_t, uint_t);
181 181 static void segdev_dump(struct seg *);
182 182 static int segdev_pagelock(struct seg *, caddr_t, size_t,
183 183 struct page ***, enum lock_type, enum seg_rw);
184 184 static int segdev_setpagesize(struct seg *, caddr_t, size_t, uint_t);
185 185 static int segdev_getmemid(struct seg *, caddr_t, memid_t *);
186 -static int segdev_capable(struct seg *, segcapability_t);
187 186
188 187 /*
189 188 * XXX this struct is used by rootnex_map_fault to identify
190 189 * the segment it has been passed. So if you make it
191 190 * "static" you'll need to fix rootnex_map_fault.
192 191 */
193 192 struct seg_ops segdev_ops = {
194 193 .dup = segdev_dup,
195 194 .unmap = segdev_unmap,
196 195 .free = segdev_free,
197 196 .fault = segdev_fault,
198 197 .faulta = segdev_faulta,
199 198 .setprot = segdev_setprot,
200 199 .checkprot = segdev_checkprot,
201 200 .kluster = (int (*)())segdev_badop,
202 201 .sync = segdev_sync,
203 202 .incore = segdev_incore,
|
↓ open down ↓ |
7 lines elided |
↑ open up ↑ |
204 203 .lockop = segdev_lockop,
205 204 .getprot = segdev_getprot,
206 205 .getoffset = segdev_getoffset,
207 206 .gettype = segdev_gettype,
208 207 .getvp = segdev_getvp,
209 208 .advise = segdev_advise,
210 209 .dump = segdev_dump,
211 210 .pagelock = segdev_pagelock,
212 211 .setpagesize = segdev_setpagesize,
213 212 .getmemid = segdev_getmemid,
214 - .capable = segdev_capable,
215 213 };
216 214
217 215 /*
218 216 * Private segdev support routines
219 217 */
220 218 static struct segdev_data *sdp_alloc(void);
221 219
222 220 static void segdev_softunlock(struct hat *, struct seg *, caddr_t,
223 221 size_t, enum seg_rw);
224 222
225 223 static faultcode_t segdev_faultpage(struct hat *, struct seg *, caddr_t,
226 224 struct vpage *, enum fault_type, enum seg_rw, devmap_handle_t *);
227 225
228 226 static faultcode_t segdev_faultpages(struct hat *, struct seg *, caddr_t,
229 227 size_t, enum fault_type, enum seg_rw, devmap_handle_t *);
230 228
231 229 static struct devmap_ctx *devmap_ctxinit(dev_t, ulong_t);
232 230 static struct devmap_softlock *devmap_softlock_init(dev_t, ulong_t);
233 231 static void devmap_softlock_rele(devmap_handle_t *);
234 232 static void devmap_ctx_rele(devmap_handle_t *);
235 233
236 234 static void devmap_ctxto(void *);
237 235
238 236 static devmap_handle_t *devmap_find_handle(devmap_handle_t *dhp_head,
239 237 caddr_t addr);
240 238
241 239 static ulong_t devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
242 240 ulong_t *opfn, ulong_t *pagesize);
243 241
244 242 static void free_devmap_handle(devmap_handle_t *dhp);
245 243
246 244 static int devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
247 245 struct seg *newseg);
248 246
249 247 static devmap_handle_t *devmap_handle_unmap(devmap_handle_t *dhp);
250 248
251 249 static void devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len);
252 250
253 251 static void devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr);
254 252
255 253 static int devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
256 254 offset_t off, size_t len, uint_t flags);
257 255
258 256 static void devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len,
259 257 caddr_t addr, size_t *llen, caddr_t *laddr);
260 258
261 259 static void devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len);
262 260
263 261 static void *devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag);
264 262 static void devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size);
265 263
266 264 static void *devmap_umem_alloc_np(size_t size, size_t flags);
267 265 static void devmap_umem_free_np(void *addr, size_t size);
268 266
269 267 /*
270 268 * routines to lock and unlock underlying segkp segment for
271 269 * KMEM_PAGEABLE type cookies.
272 270 */
273 271 static faultcode_t acquire_kpmem_lock(struct ddi_umem_cookie *, size_t);
274 272 static void release_kpmem_lock(struct ddi_umem_cookie *, size_t);
275 273
276 274 /*
277 275 * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
278 276 * drivers with devmap_access callbacks
279 277 */
280 278 static int devmap_softlock_enter(struct devmap_softlock *, size_t,
281 279 enum fault_type);
282 280 static void devmap_softlock_exit(struct devmap_softlock *, size_t,
283 281 enum fault_type);
284 282
285 283 static kmutex_t devmapctx_lock;
286 284
287 285 static kmutex_t devmap_slock;
288 286
289 287 /*
290 288 * Initialize the thread callbacks and thread private data.
291 289 */
292 290 static struct devmap_ctx *
293 291 devmap_ctxinit(dev_t dev, ulong_t id)
294 292 {
295 293 struct devmap_ctx *devctx;
296 294 struct devmap_ctx *tmp;
297 295 dev_info_t *dip;
298 296
299 297 tmp = kmem_zalloc(sizeof (struct devmap_ctx), KM_SLEEP);
300 298
301 299 mutex_enter(&devmapctx_lock);
302 300
303 301 dip = e_ddi_hold_devi_by_dev(dev, 0);
304 302 ASSERT(dip != NULL);
305 303 ddi_release_devi(dip);
306 304
307 305 for (devctx = devmapctx_list; devctx != NULL; devctx = devctx->next)
308 306 if ((devctx->dip == dip) && (devctx->id == id))
309 307 break;
310 308
311 309 if (devctx == NULL) {
312 310 devctx = tmp;
313 311 devctx->dip = dip;
314 312 devctx->id = id;
315 313 mutex_init(&devctx->lock, NULL, MUTEX_DEFAULT, NULL);
316 314 cv_init(&devctx->cv, NULL, CV_DEFAULT, NULL);
317 315 devctx->next = devmapctx_list;
318 316 devmapctx_list = devctx;
319 317 } else
320 318 kmem_free(tmp, sizeof (struct devmap_ctx));
321 319
322 320 mutex_enter(&devctx->lock);
323 321 devctx->refcnt++;
324 322 mutex_exit(&devctx->lock);
325 323 mutex_exit(&devmapctx_lock);
326 324
327 325 return (devctx);
328 326 }
329 327
330 328 /*
331 329 * Timeout callback called if a CPU has not given up the device context
332 330 * within dhp->dh_timeout_length ticks
333 331 */
334 332 static void
335 333 devmap_ctxto(void *data)
336 334 {
337 335 struct devmap_ctx *devctx = data;
338 336
339 337 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_CTXTO,
340 338 "devmap_ctxto:timeout expired, devctx=%p", (void *)devctx);
341 339 mutex_enter(&devctx->lock);
342 340 /*
343 341 * Set oncpu = 0 so the next mapping trying to get the device context
344 342 * can.
345 343 */
346 344 devctx->oncpu = 0;
347 345 devctx->timeout = 0;
348 346 cv_signal(&devctx->cv);
349 347 mutex_exit(&devctx->lock);
350 348 }
351 349
352 350 /*
353 351 * Create a device segment.
354 352 */
355 353 int
356 354 segdev_create(struct seg *seg, void *argsp)
357 355 {
358 356 struct segdev_data *sdp;
359 357 struct segdev_crargs *a = (struct segdev_crargs *)argsp;
360 358 devmap_handle_t *dhp = (devmap_handle_t *)a->devmap_data;
361 359 int error;
362 360
363 361 /*
364 362 * Since the address space is "write" locked, we
365 363 * don't need the segment lock to protect "segdev" data.
366 364 */
367 365 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
368 366
369 367 hat_map(seg->s_as->a_hat, seg->s_base, seg->s_size, HAT_MAP);
370 368
371 369 sdp = sdp_alloc();
372 370
373 371 sdp->mapfunc = a->mapfunc;
374 372 sdp->offset = a->offset;
375 373 sdp->prot = a->prot;
376 374 sdp->maxprot = a->maxprot;
377 375 sdp->type = a->type;
378 376 sdp->pageprot = 0;
379 377 sdp->softlockcnt = 0;
380 378 sdp->vpage = NULL;
381 379
382 380 if (sdp->mapfunc == NULL)
383 381 sdp->devmap_data = dhp;
384 382 else
385 383 sdp->devmap_data = dhp = NULL;
386 384
387 385 sdp->hat_flags = a->hat_flags;
388 386 sdp->hat_attr = a->hat_attr;
389 387
390 388 /*
391 389 * Currently, hat_flags supports only HAT_LOAD_NOCONSIST
392 390 */
393 391 ASSERT(!(sdp->hat_flags & ~HAT_LOAD_NOCONSIST));
394 392
395 393 /*
396 394 * Hold shadow vnode -- segdev only deals with
397 395 * character (VCHR) devices. We use the common
398 396 * vp to hang pages on.
399 397 */
400 398 sdp->vp = specfind(a->dev, VCHR);
401 399 ASSERT(sdp->vp != NULL);
402 400
403 401 seg->s_ops = &segdev_ops;
404 402 seg->s_data = sdp;
405 403
406 404 while (dhp != NULL) {
407 405 dhp->dh_seg = seg;
408 406 dhp = dhp->dh_next;
409 407 }
410 408
411 409 /*
412 410 * Inform the vnode of the new mapping.
413 411 */
414 412 /*
415 413 * It is ok to use pass sdp->maxprot to ADDMAP rather than to use
416 414 * dhp specific maxprot because spec_addmap does not use maxprot.
417 415 */
418 416 error = VOP_ADDMAP(VTOCVP(sdp->vp), sdp->offset,
419 417 seg->s_as, seg->s_base, seg->s_size,
420 418 sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
421 419
422 420 if (error != 0) {
423 421 sdp->devmap_data = NULL;
424 422 hat_unload(seg->s_as->a_hat, seg->s_base, seg->s_size,
425 423 HAT_UNLOAD_UNMAP);
426 424 } else {
427 425 /*
428 426 * Mappings of /dev/null don't count towards the VSZ of a
429 427 * process. Mappings of /dev/null have no mapping type.
430 428 */
431 429 if ((segop_gettype(seg, seg->s_base) & (MAP_SHARED |
432 430 MAP_PRIVATE)) == 0) {
433 431 seg->s_as->a_resvsize -= seg->s_size;
434 432 }
435 433 }
436 434
437 435 return (error);
438 436 }
439 437
440 438 static struct segdev_data *
441 439 sdp_alloc(void)
442 440 {
443 441 struct segdev_data *sdp;
444 442
445 443 sdp = kmem_zalloc(sizeof (struct segdev_data), KM_SLEEP);
446 444 rw_init(&sdp->lock, NULL, RW_DEFAULT, NULL);
447 445
448 446 return (sdp);
449 447 }
450 448
451 449 /*
452 450 * Duplicate seg and return new segment in newseg.
453 451 */
454 452 static int
455 453 segdev_dup(struct seg *seg, struct seg *newseg)
456 454 {
457 455 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
458 456 struct segdev_data *newsdp;
459 457 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
460 458 size_t npages;
461 459 int ret;
462 460
463 461 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DUP,
464 462 "segdev_dup:start dhp=%p, seg=%p", (void *)dhp, (void *)seg);
465 463
466 464 DEBUGF(3, (CE_CONT, "segdev_dup: dhp %p seg %p\n",
467 465 (void *)dhp, (void *)seg));
468 466
469 467 /*
470 468 * Since the address space is "write" locked, we
471 469 * don't need the segment lock to protect "segdev" data.
472 470 */
473 471 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
474 472
475 473 newsdp = sdp_alloc();
476 474
477 475 newseg->s_ops = seg->s_ops;
478 476 newseg->s_data = (void *)newsdp;
479 477
480 478 VN_HOLD(sdp->vp);
481 479 newsdp->vp = sdp->vp;
482 480 newsdp->mapfunc = sdp->mapfunc;
483 481 newsdp->offset = sdp->offset;
484 482 newsdp->pageprot = sdp->pageprot;
485 483 newsdp->prot = sdp->prot;
486 484 newsdp->maxprot = sdp->maxprot;
487 485 newsdp->type = sdp->type;
488 486 newsdp->hat_attr = sdp->hat_attr;
489 487 newsdp->hat_flags = sdp->hat_flags;
490 488 newsdp->softlockcnt = 0;
491 489
492 490 /*
493 491 * Initialize per page data if the segment we are
494 492 * dup'ing has per page information.
495 493 */
496 494 npages = seg_pages(newseg);
497 495
498 496 if (sdp->vpage != NULL) {
499 497 size_t nbytes = vpgtob(npages);
500 498
501 499 newsdp->vpage = kmem_zalloc(nbytes, KM_SLEEP);
502 500 bcopy(sdp->vpage, newsdp->vpage, nbytes);
503 501 } else
504 502 newsdp->vpage = NULL;
505 503
506 504 /*
507 505 * duplicate devmap handles
508 506 */
509 507 if (dhp != NULL) {
510 508 ret = devmap_handle_dup(dhp,
511 509 (devmap_handle_t **)&newsdp->devmap_data, newseg);
512 510 if (ret != 0) {
513 511 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DUP_CK1,
514 512 "segdev_dup:ret1 ret=%x, dhp=%p seg=%p",
515 513 ret, (void *)dhp, (void *)seg);
516 514 DEBUGF(1, (CE_CONT,
517 515 "segdev_dup: ret %x dhp %p seg %p\n",
518 516 ret, (void *)dhp, (void *)seg));
519 517 return (ret);
520 518 }
521 519 }
522 520
523 521 /*
524 522 * Inform the common vnode of the new mapping.
525 523 */
526 524 return (VOP_ADDMAP(VTOCVP(newsdp->vp),
527 525 newsdp->offset, newseg->s_as,
528 526 newseg->s_base, newseg->s_size, newsdp->prot,
529 527 newsdp->maxprot, sdp->type, CRED(), NULL));
530 528 }
531 529
532 530 /*
533 531 * duplicate devmap handles
534 532 */
535 533 static int
536 534 devmap_handle_dup(devmap_handle_t *dhp, devmap_handle_t **new_dhp,
537 535 struct seg *newseg)
538 536 {
539 537 devmap_handle_t *newdhp_save = NULL;
540 538 devmap_handle_t *newdhp = NULL;
541 539 struct devmap_callback_ctl *callbackops;
542 540
543 541 while (dhp != NULL) {
544 542 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
545 543
546 544 /* Need to lock the original dhp while copying if REMAP */
547 545 HOLD_DHP_LOCK(dhp);
548 546 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
549 547 RELE_DHP_LOCK(dhp);
550 548 newdhp->dh_seg = newseg;
551 549 newdhp->dh_next = NULL;
552 550 if (newdhp_save != NULL)
553 551 newdhp_save->dh_next = newdhp;
554 552 else
555 553 *new_dhp = newdhp;
556 554 newdhp_save = newdhp;
557 555
558 556 callbackops = &newdhp->dh_callbackops;
559 557
560 558 if (dhp->dh_softlock != NULL)
561 559 newdhp->dh_softlock = devmap_softlock_init(
562 560 newdhp->dh_dev,
563 561 (ulong_t)callbackops->devmap_access);
564 562 if (dhp->dh_ctx != NULL)
565 563 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
566 564 (ulong_t)callbackops->devmap_access);
567 565
568 566 /*
569 567 * Initialize dh_lock if we want to do remap.
570 568 */
571 569 if (newdhp->dh_flags & DEVMAP_ALLOW_REMAP) {
572 570 mutex_init(&newdhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
573 571 newdhp->dh_flags |= DEVMAP_LOCK_INITED;
574 572 }
575 573
576 574 if (callbackops->devmap_dup != NULL) {
577 575 int ret;
578 576
579 577 /*
580 578 * Call the dup callback so that the driver can
581 579 * duplicate its private data.
582 580 */
583 581 ret = (*callbackops->devmap_dup)(dhp, dhp->dh_pvtp,
584 582 (devmap_cookie_t *)newdhp, &newdhp->dh_pvtp);
585 583
586 584 if (ret != 0) {
587 585 /*
588 586 * We want to free up this segment as the driver
589 587 * has indicated that we can't dup it. But we
590 588 * don't want to call the drivers, devmap_unmap,
591 589 * callback function as the driver does not
592 590 * think this segment exists. The caller of
593 591 * devmap_dup will call seg_free on newseg
594 592 * as it was the caller that allocated the
595 593 * segment.
596 594 */
597 595 DEBUGF(1, (CE_CONT, "devmap_handle_dup ERROR: "
598 596 "newdhp %p dhp %p\n", (void *)newdhp,
599 597 (void *)dhp));
600 598 callbackops->devmap_unmap = NULL;
601 599 return (ret);
602 600 }
603 601 }
604 602
605 603 dhp = dhp->dh_next;
606 604 }
607 605
608 606 return (0);
609 607 }
610 608
611 609 /*
612 610 * Split a segment at addr for length len.
613 611 */
614 612 /*ARGSUSED*/
615 613 static int
616 614 segdev_unmap(struct seg *seg, caddr_t addr, size_t len)
617 615 {
618 616 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
619 617 register struct segdev_data *nsdp;
620 618 register struct seg *nseg;
621 619 register size_t opages; /* old segment size in pages */
622 620 register size_t npages; /* new segment size in pages */
623 621 register size_t dpages; /* pages being deleted (unmapped) */
624 622 register size_t nbytes;
625 623 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
626 624 devmap_handle_t *dhpp;
627 625 devmap_handle_t *newdhp;
628 626 struct devmap_callback_ctl *callbackops;
629 627 caddr_t nbase;
630 628 offset_t off;
631 629 ulong_t nsize;
632 630 size_t mlen, sz;
633 631
634 632 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP,
635 633 "segdev_unmap:start dhp=%p, seg=%p addr=%p len=%lx",
636 634 (void *)dhp, (void *)seg, (void *)addr, len);
637 635
638 636 DEBUGF(3, (CE_CONT, "segdev_unmap: dhp %p seg %p addr %p len %lx\n",
639 637 (void *)dhp, (void *)seg, (void *)addr, len));
640 638
641 639 /*
642 640 * Since the address space is "write" locked, we
643 641 * don't need the segment lock to protect "segdev" data.
644 642 */
645 643 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
646 644
647 645 if ((sz = sdp->softlockcnt) > 0) {
648 646 /*
649 647 * Fail the unmap if pages are SOFTLOCKed through this mapping.
650 648 * softlockcnt is protected from change by the as write lock.
651 649 */
652 650 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK1,
653 651 "segdev_unmap:error softlockcnt = %ld", sz);
654 652 DEBUGF(1, (CE_CONT, "segdev_unmap: softlockcnt %ld\n", sz));
655 653 return (EAGAIN);
656 654 }
657 655
658 656 /*
659 657 * Check for bad sizes
660 658 */
661 659 if (addr < seg->s_base || addr + len > seg->s_base + seg->s_size ||
662 660 (len & PAGEOFFSET) || ((uintptr_t)addr & PAGEOFFSET))
663 661 panic("segdev_unmap");
664 662
665 663 if (dhp != NULL) {
666 664 devmap_handle_t *tdhp;
667 665 /*
668 666 * If large page size was used in hat_devload(),
669 667 * the same page size must be used in hat_unload().
670 668 */
671 669 dhpp = tdhp = devmap_find_handle(dhp, addr);
672 670 while (tdhp != NULL) {
673 671 if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
674 672 break;
675 673 }
676 674 tdhp = tdhp->dh_next;
677 675 }
678 676 if (tdhp != NULL) { /* found a dhp using large pages */
679 677 size_t slen = len;
680 678 size_t mlen;
681 679 size_t soff;
682 680
683 681 soff = (ulong_t)(addr - dhpp->dh_uvaddr);
684 682 while (slen != 0) {
685 683 mlen = MIN(slen, (dhpp->dh_len - soff));
686 684 hat_unload(seg->s_as->a_hat, dhpp->dh_uvaddr,
687 685 dhpp->dh_len, HAT_UNLOAD_UNMAP);
688 686 dhpp = dhpp->dh_next;
689 687 ASSERT(slen >= mlen);
690 688 slen -= mlen;
691 689 soff = 0;
692 690 }
693 691 } else
694 692 hat_unload(seg->s_as->a_hat, addr, len,
695 693 HAT_UNLOAD_UNMAP);
696 694 } else {
697 695 /*
698 696 * Unload any hardware translations in the range
699 697 * to be taken out.
700 698 */
701 699 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD_UNMAP);
702 700 }
703 701
704 702 /*
705 703 * get the user offset which will used in the driver callbacks
706 704 */
707 705 off = sdp->offset + (offset_t)(addr - seg->s_base);
708 706
709 707 /*
710 708 * Inform the vnode of the unmapping.
711 709 */
712 710 ASSERT(sdp->vp != NULL);
713 711 (void) VOP_DELMAP(VTOCVP(sdp->vp), off, seg->s_as, addr, len,
714 712 sdp->prot, sdp->maxprot, sdp->type, CRED(), NULL);
715 713
716 714 /*
717 715 * Check for entire segment
718 716 */
719 717 if (addr == seg->s_base && len == seg->s_size) {
720 718 seg_free(seg);
721 719 return (0);
722 720 }
723 721
724 722 opages = seg_pages(seg);
725 723 dpages = btop(len);
726 724 npages = opages - dpages;
727 725
728 726 /*
729 727 * Check for beginning of segment
730 728 */
731 729 if (addr == seg->s_base) {
732 730 if (sdp->vpage != NULL) {
733 731 register struct vpage *ovpage;
734 732
735 733 ovpage = sdp->vpage; /* keep pointer to vpage */
736 734
737 735 nbytes = vpgtob(npages);
738 736 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
739 737 bcopy(&ovpage[dpages], sdp->vpage, nbytes);
740 738
741 739 /* free up old vpage */
742 740 kmem_free(ovpage, vpgtob(opages));
743 741 }
744 742
745 743 /*
746 744 * free devmap handles from the beginning of the mapping.
747 745 */
748 746 if (dhp != NULL)
749 747 devmap_handle_unmap_head(dhp, len);
750 748
751 749 sdp->offset += (offset_t)len;
752 750
753 751 seg->s_base += len;
754 752 seg->s_size -= len;
755 753
756 754 return (0);
757 755 }
758 756
759 757 /*
760 758 * Check for end of segment
761 759 */
762 760 if (addr + len == seg->s_base + seg->s_size) {
763 761 if (sdp->vpage != NULL) {
764 762 register struct vpage *ovpage;
765 763
766 764 ovpage = sdp->vpage; /* keep pointer to vpage */
767 765
768 766 nbytes = vpgtob(npages);
769 767 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
770 768 bcopy(ovpage, sdp->vpage, nbytes);
771 769
772 770 /* free up old vpage */
773 771 kmem_free(ovpage, vpgtob(opages));
774 772 }
775 773 seg->s_size -= len;
776 774
777 775 /*
778 776 * free devmap handles from addr to the end of the mapping.
779 777 */
780 778 if (dhp != NULL)
781 779 devmap_handle_unmap_tail(dhp, addr);
782 780
783 781 return (0);
784 782 }
785 783
786 784 /*
787 785 * The section to go is in the middle of the segment,
788 786 * have to make it into two segments. nseg is made for
789 787 * the high end while seg is cut down at the low end.
790 788 */
791 789 nbase = addr + len; /* new seg base */
792 790 nsize = (seg->s_base + seg->s_size) - nbase; /* new seg size */
793 791 seg->s_size = addr - seg->s_base; /* shrink old seg */
794 792 nseg = seg_alloc(seg->s_as, nbase, nsize);
795 793 if (nseg == NULL)
796 794 panic("segdev_unmap seg_alloc");
797 795
798 796 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK2,
799 797 "segdev_unmap: seg=%p nseg=%p", (void *)seg, (void *)nseg);
800 798 DEBUGF(3, (CE_CONT, "segdev_unmap: segdev_dup seg %p nseg %p\n",
801 799 (void *)seg, (void *)nseg));
802 800 nsdp = sdp_alloc();
803 801
804 802 nseg->s_ops = seg->s_ops;
805 803 nseg->s_data = (void *)nsdp;
806 804
807 805 VN_HOLD(sdp->vp);
808 806 nsdp->mapfunc = sdp->mapfunc;
809 807 nsdp->offset = sdp->offset + (offset_t)(nseg->s_base - seg->s_base);
810 808 nsdp->vp = sdp->vp;
811 809 nsdp->pageprot = sdp->pageprot;
812 810 nsdp->prot = sdp->prot;
813 811 nsdp->maxprot = sdp->maxprot;
814 812 nsdp->type = sdp->type;
815 813 nsdp->hat_attr = sdp->hat_attr;
816 814 nsdp->hat_flags = sdp->hat_flags;
817 815 nsdp->softlockcnt = 0;
818 816
819 817 /*
820 818 * Initialize per page data if the segment we are
821 819 * dup'ing has per page information.
822 820 */
823 821 if (sdp->vpage != NULL) {
824 822 /* need to split vpage into two arrays */
825 823 register size_t nnbytes;
826 824 register size_t nnpages;
827 825 register struct vpage *ovpage;
828 826
829 827 ovpage = sdp->vpage; /* keep pointer to vpage */
830 828
831 829 npages = seg_pages(seg); /* seg has shrunk */
832 830 nbytes = vpgtob(npages);
833 831 nnpages = seg_pages(nseg);
834 832 nnbytes = vpgtob(nnpages);
835 833
836 834 sdp->vpage = kmem_alloc(nbytes, KM_SLEEP);
837 835 bcopy(ovpage, sdp->vpage, nbytes);
838 836
839 837 nsdp->vpage = kmem_alloc(nnbytes, KM_SLEEP);
840 838 bcopy(&ovpage[npages + dpages], nsdp->vpage, nnbytes);
841 839
842 840 /* free up old vpage */
843 841 kmem_free(ovpage, vpgtob(opages));
844 842 } else
845 843 nsdp->vpage = NULL;
846 844
847 845 /*
848 846 * unmap dhps.
849 847 */
850 848 if (dhp == NULL) {
851 849 nsdp->devmap_data = NULL;
852 850 return (0);
853 851 }
854 852 while (dhp != NULL) {
855 853 callbackops = &dhp->dh_callbackops;
856 854 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_UNMAP_CK3,
857 855 "segdev_unmap: dhp=%p addr=%p", dhp, addr);
858 856 DEBUGF(3, (CE_CONT, "unmap: dhp %p addr %p uvaddr %p len %lx\n",
859 857 (void *)dhp, (void *)addr,
860 858 (void *)dhp->dh_uvaddr, dhp->dh_len));
861 859
862 860 if (addr == (dhp->dh_uvaddr + dhp->dh_len)) {
863 861 dhpp = dhp->dh_next;
864 862 dhp->dh_next = NULL;
865 863 dhp = dhpp;
866 864 } else if (addr > (dhp->dh_uvaddr + dhp->dh_len)) {
867 865 dhp = dhp->dh_next;
868 866 } else if (addr > dhp->dh_uvaddr &&
869 867 (addr + len) < (dhp->dh_uvaddr + dhp->dh_len)) {
870 868 /*
871 869 * <addr, addr+len> is enclosed by dhp.
872 870 * create a newdhp that begins at addr+len and
873 871 * ends at dhp->dh_uvaddr+dhp->dh_len.
874 872 */
875 873 newdhp = kmem_alloc(sizeof (devmap_handle_t), KM_SLEEP);
876 874 HOLD_DHP_LOCK(dhp);
877 875 bcopy(dhp, newdhp, sizeof (devmap_handle_t));
878 876 RELE_DHP_LOCK(dhp);
879 877 newdhp->dh_seg = nseg;
880 878 newdhp->dh_next = dhp->dh_next;
881 879 if (dhp->dh_softlock != NULL)
882 880 newdhp->dh_softlock = devmap_softlock_init(
883 881 newdhp->dh_dev,
884 882 (ulong_t)callbackops->devmap_access);
885 883 if (dhp->dh_ctx != NULL)
886 884 newdhp->dh_ctx = devmap_ctxinit(newdhp->dh_dev,
887 885 (ulong_t)callbackops->devmap_access);
888 886 if (newdhp->dh_flags & DEVMAP_LOCK_INITED) {
889 887 mutex_init(&newdhp->dh_lock,
890 888 NULL, MUTEX_DEFAULT, NULL);
891 889 }
892 890 if (callbackops->devmap_unmap != NULL)
893 891 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
894 892 off, len, dhp, &dhp->dh_pvtp,
895 893 newdhp, &newdhp->dh_pvtp);
896 894 mlen = len + (addr - dhp->dh_uvaddr);
897 895 devmap_handle_reduce_len(newdhp, mlen);
898 896 nsdp->devmap_data = newdhp;
899 897 /* XX Changing len should recalculate LARGE flag */
900 898 dhp->dh_len = addr - dhp->dh_uvaddr;
901 899 dhpp = dhp->dh_next;
902 900 dhp->dh_next = NULL;
903 901 dhp = dhpp;
904 902 } else if ((addr > dhp->dh_uvaddr) &&
905 903 ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len))) {
906 904 mlen = dhp->dh_len + dhp->dh_uvaddr - addr;
907 905 /*
908 906 * <addr, addr+len> spans over dhps.
909 907 */
910 908 if (callbackops->devmap_unmap != NULL)
911 909 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
912 910 off, mlen, (devmap_cookie_t *)dhp,
913 911 &dhp->dh_pvtp, NULL, NULL);
914 912 /* XX Changing len should recalculate LARGE flag */
915 913 dhp->dh_len = addr - dhp->dh_uvaddr;
916 914 dhpp = dhp->dh_next;
917 915 dhp->dh_next = NULL;
918 916 dhp = dhpp;
919 917 nsdp->devmap_data = dhp;
920 918 } else if ((addr + len) >= (dhp->dh_uvaddr + dhp->dh_len)) {
921 919 /*
922 920 * dhp is enclosed by <addr, addr+len>.
923 921 */
924 922 dhp->dh_seg = nseg;
925 923 nsdp->devmap_data = dhp;
926 924 dhp = devmap_handle_unmap(dhp);
927 925 nsdp->devmap_data = dhp; /* XX redundant? */
928 926 } else if (((addr + len) > dhp->dh_uvaddr) &&
929 927 ((addr + len) < (dhp->dh_uvaddr + dhp->dh_len))) {
930 928 mlen = addr + len - dhp->dh_uvaddr;
931 929 if (callbackops->devmap_unmap != NULL)
932 930 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
933 931 dhp->dh_uoff, mlen, NULL,
934 932 NULL, dhp, &dhp->dh_pvtp);
935 933 devmap_handle_reduce_len(dhp, mlen);
936 934 nsdp->devmap_data = dhp;
937 935 dhp->dh_seg = nseg;
938 936 dhp = dhp->dh_next;
939 937 } else {
940 938 dhp->dh_seg = nseg;
941 939 dhp = dhp->dh_next;
942 940 }
943 941 }
944 942 return (0);
945 943 }
946 944
947 945 /*
948 946 * Utility function handles reducing the length of a devmap handle during unmap
949 947 * Note that is only used for unmapping the front portion of the handler,
950 948 * i.e., we are bumping up the offset/pfn etc up by len
951 949 * Do not use if reducing length at the tail.
952 950 */
953 951 static void
954 952 devmap_handle_reduce_len(devmap_handle_t *dhp, size_t len)
955 953 {
956 954 struct ddi_umem_cookie *cp;
957 955 struct devmap_pmem_cookie *pcp;
958 956 /*
959 957 * adjust devmap handle fields
960 958 */
961 959 ASSERT(len < dhp->dh_len);
962 960
963 961 /* Make sure only page-aligned changes are done */
964 962 ASSERT((len & PAGEOFFSET) == 0);
965 963
966 964 dhp->dh_len -= len;
967 965 dhp->dh_uoff += (offset_t)len;
968 966 dhp->dh_roff += (offset_t)len;
969 967 dhp->dh_uvaddr += len;
970 968 /* Need to grab dhp lock if REMAP */
971 969 HOLD_DHP_LOCK(dhp);
972 970 cp = dhp->dh_cookie;
973 971 if (!(dhp->dh_flags & DEVMAP_MAPPING_INVALID)) {
974 972 if (cookie_is_devmem(cp)) {
975 973 dhp->dh_pfn += btop(len);
976 974 } else if (cookie_is_pmem(cp)) {
977 975 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
978 976 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
979 977 dhp->dh_roff < ptob(pcp->dp_npages));
980 978 } else {
981 979 ASSERT(dhp->dh_roff < cp->size);
982 980 ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
983 981 dhp->dh_cvaddr < (cp->cvaddr + cp->size));
984 982 ASSERT((dhp->dh_cvaddr + len) <=
985 983 (cp->cvaddr + cp->size));
986 984
987 985 dhp->dh_cvaddr += len;
988 986 }
989 987 }
990 988 /* XXX - Should recalculate the DEVMAP_FLAG_LARGE after changes */
991 989 RELE_DHP_LOCK(dhp);
992 990 }
993 991
994 992 /*
995 993 * Free devmap handle, dhp.
996 994 * Return the next devmap handle on the linked list.
997 995 */
998 996 static devmap_handle_t *
999 997 devmap_handle_unmap(devmap_handle_t *dhp)
1000 998 {
1001 999 struct devmap_callback_ctl *callbackops = &dhp->dh_callbackops;
1002 1000 struct segdev_data *sdp = (struct segdev_data *)dhp->dh_seg->s_data;
1003 1001 devmap_handle_t *dhpp = (devmap_handle_t *)sdp->devmap_data;
1004 1002
1005 1003 ASSERT(dhp != NULL);
1006 1004
1007 1005 /*
1008 1006 * before we free up dhp, call the driver's devmap_unmap entry point
1009 1007 * to free resources allocated for this dhp.
1010 1008 */
1011 1009 if (callbackops->devmap_unmap != NULL) {
1012 1010 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp, dhp->dh_uoff,
1013 1011 dhp->dh_len, NULL, NULL, NULL, NULL);
1014 1012 }
1015 1013
1016 1014 if (dhpp == dhp) { /* releasing first dhp, change sdp data */
1017 1015 sdp->devmap_data = dhp->dh_next;
1018 1016 } else {
1019 1017 while (dhpp->dh_next != dhp) {
1020 1018 dhpp = dhpp->dh_next;
1021 1019 }
1022 1020 dhpp->dh_next = dhp->dh_next;
1023 1021 }
1024 1022 dhpp = dhp->dh_next; /* return value is next dhp in chain */
1025 1023
1026 1024 if (dhp->dh_softlock != NULL)
1027 1025 devmap_softlock_rele(dhp);
1028 1026
1029 1027 if (dhp->dh_ctx != NULL)
1030 1028 devmap_ctx_rele(dhp);
1031 1029
1032 1030 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1033 1031 mutex_destroy(&dhp->dh_lock);
1034 1032 }
1035 1033 kmem_free(dhp, sizeof (devmap_handle_t));
1036 1034
1037 1035 return (dhpp);
1038 1036 }
1039 1037
1040 1038 /*
1041 1039 * Free complete devmap handles from dhp for len bytes
1042 1040 * dhp can be either the first handle or a subsequent handle
1043 1041 */
1044 1042 static void
1045 1043 devmap_handle_unmap_head(devmap_handle_t *dhp, size_t len)
1046 1044 {
1047 1045 struct devmap_callback_ctl *callbackops;
1048 1046
1049 1047 /*
1050 1048 * free the devmap handles covered by len.
1051 1049 */
1052 1050 while (len >= dhp->dh_len) {
1053 1051 len -= dhp->dh_len;
1054 1052 dhp = devmap_handle_unmap(dhp);
1055 1053 }
1056 1054 if (len != 0) { /* partial unmap at head of first remaining dhp */
1057 1055 callbackops = &dhp->dh_callbackops;
1058 1056
1059 1057 /*
1060 1058 * Call the unmap callback so the drivers can make
1061 1059 * adjustment on its private data.
1062 1060 */
1063 1061 if (callbackops->devmap_unmap != NULL)
1064 1062 (*callbackops->devmap_unmap)(dhp, dhp->dh_pvtp,
1065 1063 dhp->dh_uoff, len, NULL, NULL, dhp, &dhp->dh_pvtp);
1066 1064 devmap_handle_reduce_len(dhp, len);
1067 1065 }
1068 1066 }
1069 1067
1070 1068 /*
1071 1069 * Free devmap handles to truncate the mapping after addr
1072 1070 * RFE: Simpler to pass in dhp pointing at correct dhp (avoid find again)
1073 1071 * Also could then use the routine in middle unmap case too
1074 1072 */
1075 1073 static void
1076 1074 devmap_handle_unmap_tail(devmap_handle_t *dhp, caddr_t addr)
1077 1075 {
1078 1076 register struct seg *seg = dhp->dh_seg;
1079 1077 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1080 1078 register devmap_handle_t *dhph = (devmap_handle_t *)sdp->devmap_data;
1081 1079 struct devmap_callback_ctl *callbackops;
1082 1080 register devmap_handle_t *dhpp;
1083 1081 size_t maplen;
1084 1082 ulong_t off;
1085 1083 size_t len;
1086 1084
1087 1085 maplen = (size_t)(addr - dhp->dh_uvaddr);
1088 1086 dhph = devmap_find_handle(dhph, addr);
1089 1087
1090 1088 while (dhph != NULL) {
1091 1089 if (maplen == 0) {
1092 1090 dhph = devmap_handle_unmap(dhph);
1093 1091 } else {
1094 1092 callbackops = &dhph->dh_callbackops;
1095 1093 len = dhph->dh_len - maplen;
1096 1094 off = (ulong_t)sdp->offset + (addr - seg->s_base);
1097 1095 /*
1098 1096 * Call the unmap callback so the driver
1099 1097 * can make adjustments on its private data.
1100 1098 */
1101 1099 if (callbackops->devmap_unmap != NULL)
1102 1100 (*callbackops->devmap_unmap)(dhph,
1103 1101 dhph->dh_pvtp, off, len,
1104 1102 (devmap_cookie_t *)dhph,
1105 1103 &dhph->dh_pvtp, NULL, NULL);
1106 1104 /* XXX Reducing len needs to recalculate LARGE flag */
1107 1105 dhph->dh_len = maplen;
1108 1106 maplen = 0;
1109 1107 dhpp = dhph->dh_next;
1110 1108 dhph->dh_next = NULL;
1111 1109 dhph = dhpp;
1112 1110 }
1113 1111 } /* end while */
1114 1112 }
1115 1113
1116 1114 /*
1117 1115 * Free a segment.
1118 1116 */
1119 1117 static void
1120 1118 segdev_free(struct seg *seg)
1121 1119 {
1122 1120 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1123 1121 devmap_handle_t *dhp = (devmap_handle_t *)sdp->devmap_data;
1124 1122
1125 1123 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FREE,
1126 1124 "segdev_free: dhp=%p seg=%p", (void *)dhp, (void *)seg);
1127 1125 DEBUGF(3, (CE_CONT, "segdev_free: dhp %p seg %p\n",
1128 1126 (void *)dhp, (void *)seg));
1129 1127
1130 1128 /*
1131 1129 * Since the address space is "write" locked, we
1132 1130 * don't need the segment lock to protect "segdev" data.
1133 1131 */
1134 1132 ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
1135 1133
1136 1134 while (dhp != NULL)
1137 1135 dhp = devmap_handle_unmap(dhp);
1138 1136
1139 1137 VN_RELE(sdp->vp);
1140 1138 if (sdp->vpage != NULL)
1141 1139 kmem_free(sdp->vpage, vpgtob(seg_pages(seg)));
1142 1140
1143 1141 rw_destroy(&sdp->lock);
1144 1142 kmem_free(sdp, sizeof (*sdp));
1145 1143 }
1146 1144
1147 1145 static void
1148 1146 free_devmap_handle(devmap_handle_t *dhp)
1149 1147 {
1150 1148 register devmap_handle_t *dhpp;
1151 1149
1152 1150 /*
1153 1151 * free up devmap handle
1154 1152 */
1155 1153 while (dhp != NULL) {
1156 1154 dhpp = dhp->dh_next;
1157 1155 if (dhp->dh_flags & DEVMAP_LOCK_INITED) {
1158 1156 mutex_destroy(&dhp->dh_lock);
1159 1157 }
1160 1158
1161 1159 if (dhp->dh_softlock != NULL)
1162 1160 devmap_softlock_rele(dhp);
1163 1161
1164 1162 if (dhp->dh_ctx != NULL)
1165 1163 devmap_ctx_rele(dhp);
1166 1164
1167 1165 kmem_free(dhp, sizeof (devmap_handle_t));
1168 1166 dhp = dhpp;
1169 1167 }
1170 1168 }
1171 1169
1172 1170 /*
1173 1171 * routines to lock and unlock underlying segkp segment for
1174 1172 * KMEM_PAGEABLE type cookies.
1175 1173 * segkp only allows a single pending F_SOFTLOCK
1176 1174 * we keep track of number of locks in the cookie so we can
1177 1175 * have multiple pending faults and manage the calls to segkp.
1178 1176 * RFE: if segkp supports either pagelock or can support multiple
1179 1177 * calls to F_SOFTLOCK, then these routines can go away.
1180 1178 * If pagelock, segdev_faultpage can fault on a page by page basis
1181 1179 * and simplifies the code quite a bit.
1182 1180 * if multiple calls allowed but not partial ranges, then need for
1183 1181 * cookie->lock and locked count goes away, code can call as_fault directly
1184 1182 */
1185 1183 static faultcode_t
1186 1184 acquire_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1187 1185 {
1188 1186 int err = 0;
1189 1187 ASSERT(cookie_is_kpmem(cookie));
1190 1188 /*
1191 1189 * Fault in pages in segkp with F_SOFTLOCK.
1192 1190 * We want to hold the lock until all pages have been loaded.
1193 1191 * segkp only allows single caller to hold SOFTLOCK, so cookie
1194 1192 * holds a count so we dont call into segkp multiple times
1195 1193 */
1196 1194 mutex_enter(&cookie->lock);
1197 1195
1198 1196 /*
1199 1197 * Check for overflow in locked field
1200 1198 */
1201 1199 if ((UINT32_MAX - cookie->locked) < npages) {
1202 1200 err = FC_MAKE_ERR(ENOMEM);
1203 1201 } else if (cookie->locked == 0) {
1204 1202 /* First time locking */
1205 1203 err = as_fault(kas.a_hat, &kas, cookie->cvaddr,
1206 1204 cookie->size, F_SOFTLOCK, PROT_READ|PROT_WRITE);
1207 1205 }
1208 1206 if (!err) {
1209 1207 cookie->locked += npages;
1210 1208 }
1211 1209 mutex_exit(&cookie->lock);
1212 1210 return (err);
1213 1211 }
1214 1212
1215 1213 static void
1216 1214 release_kpmem_lock(struct ddi_umem_cookie *cookie, size_t npages)
1217 1215 {
1218 1216 mutex_enter(&cookie->lock);
1219 1217 ASSERT(cookie_is_kpmem(cookie));
1220 1218 ASSERT(cookie->locked >= npages);
1221 1219 cookie->locked -= (uint_t)npages;
1222 1220 if (cookie->locked == 0) {
1223 1221 /* Last unlock */
1224 1222 if (as_fault(kas.a_hat, &kas, cookie->cvaddr,
1225 1223 cookie->size, F_SOFTUNLOCK, PROT_READ|PROT_WRITE))
1226 1224 panic("segdev releasing kpmem lock %p", (void *)cookie);
1227 1225 }
1228 1226 mutex_exit(&cookie->lock);
1229 1227 }
1230 1228
1231 1229 /*
1232 1230 * Routines to synchronize F_SOFTLOCK and F_INVAL faults for
1233 1231 * drivers with devmap_access callbacks
1234 1232 * slock->softlocked basically works like a rw lock
1235 1233 * -ve counts => F_SOFTLOCK in progress
1236 1234 * +ve counts => F_INVAL/F_PROT in progress
1237 1235 * We allow only one F_SOFTLOCK at a time
1238 1236 * but can have multiple pending F_INVAL/F_PROT calls
1239 1237 *
1240 1238 * This routine waits using cv_wait_sig so killing processes is more graceful
1241 1239 * Returns EINTR if coming out of this routine due to a signal, 0 otherwise
1242 1240 */
1243 1241 static int devmap_softlock_enter(
1244 1242 struct devmap_softlock *slock,
1245 1243 size_t npages,
1246 1244 enum fault_type type)
1247 1245 {
1248 1246 if (npages == 0)
1249 1247 return (0);
1250 1248 mutex_enter(&(slock->lock));
1251 1249 switch (type) {
1252 1250 case F_SOFTLOCK :
1253 1251 while (slock->softlocked) {
1254 1252 if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1255 1253 /* signalled */
1256 1254 mutex_exit(&(slock->lock));
1257 1255 return (EINTR);
1258 1256 }
1259 1257 }
1260 1258 slock->softlocked -= npages; /* -ve count => locked */
1261 1259 break;
1262 1260 case F_INVAL :
1263 1261 case F_PROT :
1264 1262 while (slock->softlocked < 0)
1265 1263 if (cv_wait_sig(&(slock)->cv, &(slock)->lock) == 0) {
1266 1264 /* signalled */
1267 1265 mutex_exit(&(slock->lock));
1268 1266 return (EINTR);
1269 1267 }
1270 1268 slock->softlocked += npages; /* +ve count => f_invals */
1271 1269 break;
1272 1270 default:
1273 1271 ASSERT(0);
1274 1272 }
1275 1273 mutex_exit(&(slock->lock));
1276 1274 return (0);
1277 1275 }
1278 1276
1279 1277 static void devmap_softlock_exit(
1280 1278 struct devmap_softlock *slock,
1281 1279 size_t npages,
1282 1280 enum fault_type type)
1283 1281 {
1284 1282 if (slock == NULL)
1285 1283 return;
1286 1284 mutex_enter(&(slock->lock));
1287 1285 switch (type) {
1288 1286 case F_SOFTLOCK :
1289 1287 ASSERT(-slock->softlocked >= npages);
1290 1288 slock->softlocked += npages; /* -ve count is softlocked */
1291 1289 if (slock->softlocked == 0)
1292 1290 cv_signal(&slock->cv);
1293 1291 break;
1294 1292 case F_INVAL :
1295 1293 case F_PROT:
1296 1294 ASSERT(slock->softlocked >= npages);
1297 1295 slock->softlocked -= npages;
1298 1296 if (slock->softlocked == 0)
1299 1297 cv_signal(&slock->cv);
1300 1298 break;
1301 1299 default:
1302 1300 ASSERT(0);
1303 1301 }
1304 1302 mutex_exit(&(slock->lock));
1305 1303 }
1306 1304
1307 1305 /*
1308 1306 * Do a F_SOFTUNLOCK call over the range requested.
1309 1307 * The range must have already been F_SOFTLOCK'ed.
1310 1308 * The segment lock should be held, (but not the segment private lock?)
1311 1309 * The softunlock code below does not adjust for large page sizes
1312 1310 * assumes the caller already did any addr/len adjustments for
1313 1311 * pagesize mappings before calling.
1314 1312 */
1315 1313 /*ARGSUSED*/
1316 1314 static void
1317 1315 segdev_softunlock(
1318 1316 struct hat *hat, /* the hat */
1319 1317 struct seg *seg, /* seg_dev of interest */
1320 1318 caddr_t addr, /* base address of range */
1321 1319 size_t len, /* number of bytes */
1322 1320 enum seg_rw rw) /* type of access at fault */
1323 1321 {
1324 1322 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1325 1323 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1326 1324
1327 1325 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SOFTUNLOCK,
1328 1326 "segdev_softunlock:dhp_head=%p sdp=%p addr=%p len=%lx",
1329 1327 dhp_head, sdp, addr, len);
1330 1328 DEBUGF(3, (CE_CONT, "segdev_softunlock: dhp %p lockcnt %lx "
1331 1329 "addr %p len %lx\n",
1332 1330 (void *)dhp_head, sdp->softlockcnt, (void *)addr, len));
1333 1331
1334 1332 hat_unlock(hat, addr, len);
1335 1333
1336 1334 if (dhp_head != NULL) {
1337 1335 devmap_handle_t *dhp;
1338 1336 size_t mlen;
1339 1337 size_t tlen = len;
1340 1338 ulong_t off;
1341 1339
1342 1340 dhp = devmap_find_handle(dhp_head, addr);
1343 1341 ASSERT(dhp != NULL);
1344 1342
1345 1343 off = (ulong_t)(addr - dhp->dh_uvaddr);
1346 1344 while (tlen != 0) {
1347 1345 mlen = MIN(tlen, (dhp->dh_len - off));
1348 1346
1349 1347 /*
1350 1348 * unlock segkp memory, locked during F_SOFTLOCK
1351 1349 */
1352 1350 if (dhp_is_kpmem(dhp)) {
1353 1351 release_kpmem_lock(
1354 1352 (struct ddi_umem_cookie *)dhp->dh_cookie,
1355 1353 btopr(mlen));
1356 1354 }
1357 1355
1358 1356 /*
1359 1357 * Do the softlock accounting for devmap_access
1360 1358 */
1361 1359 if (dhp->dh_callbackops.devmap_access != NULL) {
1362 1360 devmap_softlock_exit(dhp->dh_softlock,
1363 1361 btopr(mlen), F_SOFTLOCK);
1364 1362 }
1365 1363
1366 1364 tlen -= mlen;
1367 1365 dhp = dhp->dh_next;
1368 1366 off = 0;
1369 1367 }
1370 1368 }
1371 1369
1372 1370 mutex_enter(&freemem_lock);
1373 1371 ASSERT(sdp->softlockcnt >= btopr(len));
1374 1372 sdp->softlockcnt -= btopr(len);
1375 1373 mutex_exit(&freemem_lock);
1376 1374 if (sdp->softlockcnt == 0) {
1377 1375 /*
1378 1376 * All SOFTLOCKS are gone. Wakeup any waiting
1379 1377 * unmappers so they can try again to unmap.
1380 1378 * Check for waiters first without the mutex
1381 1379 * held so we don't always grab the mutex on
1382 1380 * softunlocks.
1383 1381 */
1384 1382 if (AS_ISUNMAPWAIT(seg->s_as)) {
1385 1383 mutex_enter(&seg->s_as->a_contents);
1386 1384 if (AS_ISUNMAPWAIT(seg->s_as)) {
1387 1385 AS_CLRUNMAPWAIT(seg->s_as);
1388 1386 cv_broadcast(&seg->s_as->a_cv);
1389 1387 }
1390 1388 mutex_exit(&seg->s_as->a_contents);
1391 1389 }
1392 1390 }
1393 1391
1394 1392 }
1395 1393
1396 1394 /*
1397 1395 * Handle fault for a single page.
1398 1396 * Done in a separate routine so we can handle errors more easily.
1399 1397 * This routine is called only from segdev_faultpages()
1400 1398 * when looping over the range of addresses requested. The segment lock is held.
1401 1399 */
1402 1400 static faultcode_t
1403 1401 segdev_faultpage(
1404 1402 struct hat *hat, /* the hat */
1405 1403 struct seg *seg, /* seg_dev of interest */
1406 1404 caddr_t addr, /* address in as */
1407 1405 struct vpage *vpage, /* pointer to vpage for seg, addr */
1408 1406 enum fault_type type, /* type of fault */
1409 1407 enum seg_rw rw, /* type of access at fault */
1410 1408 devmap_handle_t *dhp) /* devmap handle if any for this page */
1411 1409 {
1412 1410 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1413 1411 uint_t prot;
1414 1412 pfn_t pfnum = PFN_INVALID;
1415 1413 u_offset_t offset;
1416 1414 uint_t hat_flags;
1417 1415 dev_info_t *dip;
1418 1416
1419 1417 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE,
1420 1418 "segdev_faultpage: dhp=%p seg=%p addr=%p", dhp, seg, addr);
1421 1419 DEBUGF(8, (CE_CONT, "segdev_faultpage: dhp %p seg %p addr %p \n",
1422 1420 (void *)dhp, (void *)seg, (void *)addr));
1423 1421
1424 1422 /*
1425 1423 * Initialize protection value for this page.
1426 1424 * If we have per page protection values check it now.
1427 1425 */
1428 1426 if (sdp->pageprot) {
1429 1427 uint_t protchk;
1430 1428
1431 1429 switch (rw) {
1432 1430 case S_READ:
1433 1431 protchk = PROT_READ;
1434 1432 break;
1435 1433 case S_WRITE:
1436 1434 protchk = PROT_WRITE;
1437 1435 break;
1438 1436 case S_EXEC:
1439 1437 protchk = PROT_EXEC;
1440 1438 break;
1441 1439 case S_OTHER:
1442 1440 default:
1443 1441 protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1444 1442 break;
1445 1443 }
1446 1444
1447 1445 prot = VPP_PROT(vpage);
1448 1446 if ((prot & protchk) == 0)
1449 1447 return (FC_PROT); /* illegal access type */
1450 1448 } else {
1451 1449 prot = sdp->prot;
1452 1450 /* caller has already done segment level protection check */
1453 1451 }
1454 1452
1455 1453 if (type == F_SOFTLOCK) {
1456 1454 mutex_enter(&freemem_lock);
1457 1455 sdp->softlockcnt++;
1458 1456 mutex_exit(&freemem_lock);
1459 1457 }
1460 1458
1461 1459 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
1462 1460 offset = sdp->offset + (u_offset_t)(addr - seg->s_base);
1463 1461 /*
1464 1462 * In the devmap framework, sdp->mapfunc is set to NULL. we can get
1465 1463 * pfnum from dhp->dh_pfn (at beginning of segment) and offset from
1466 1464 * seg->s_base.
1467 1465 */
1468 1466 if (dhp == NULL) {
1469 1467 /* If segment has devmap_data, then dhp should be non-NULL */
1470 1468 ASSERT(sdp->devmap_data == NULL);
1471 1469 pfnum = (pfn_t)cdev_mmap(sdp->mapfunc, sdp->vp->v_rdev,
1472 1470 (off_t)offset, prot);
1473 1471 prot |= sdp->hat_attr;
1474 1472 } else {
1475 1473 ulong_t off;
1476 1474 struct ddi_umem_cookie *cp;
1477 1475 struct devmap_pmem_cookie *pcp;
1478 1476
1479 1477 /* ensure the dhp passed in contains addr. */
1480 1478 ASSERT(dhp == devmap_find_handle(
1481 1479 (devmap_handle_t *)sdp->devmap_data, addr));
1482 1480
1483 1481 off = addr - dhp->dh_uvaddr;
1484 1482
1485 1483 /*
1486 1484 * This routine assumes that the caller makes sure that the
1487 1485 * fields in dhp used below are unchanged due to remap during
1488 1486 * this call. Caller does HOLD_DHP_LOCK if neeed
1489 1487 */
1490 1488 cp = dhp->dh_cookie;
1491 1489 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1492 1490 pfnum = PFN_INVALID;
1493 1491 } else if (cookie_is_devmem(cp)) {
1494 1492 pfnum = dhp->dh_pfn + btop(off);
1495 1493 } else if (cookie_is_pmem(cp)) {
1496 1494 pcp = (struct devmap_pmem_cookie *)dhp->dh_pcookie;
1497 1495 ASSERT((dhp->dh_roff & PAGEOFFSET) == 0 &&
1498 1496 dhp->dh_roff < ptob(pcp->dp_npages));
1499 1497 pfnum = page_pptonum(
1500 1498 pcp->dp_pparray[btop(off + dhp->dh_roff)]);
1501 1499 } else {
1502 1500 ASSERT(dhp->dh_roff < cp->size);
1503 1501 ASSERT(dhp->dh_cvaddr >= cp->cvaddr &&
1504 1502 dhp->dh_cvaddr < (cp->cvaddr + cp->size));
1505 1503 ASSERT((dhp->dh_cvaddr + off) <=
1506 1504 (cp->cvaddr + cp->size));
1507 1505 ASSERT((dhp->dh_cvaddr + off + PAGESIZE) <=
1508 1506 (cp->cvaddr + cp->size));
1509 1507
1510 1508 switch (cp->type) {
1511 1509 case UMEM_LOCKED :
1512 1510 if (cp->pparray != NULL) {
1513 1511 ASSERT((dhp->dh_roff &
1514 1512 PAGEOFFSET) == 0);
1515 1513 pfnum = page_pptonum(
1516 1514 cp->pparray[btop(off +
1517 1515 dhp->dh_roff)]);
1518 1516 } else {
1519 1517 pfnum = hat_getpfnum(
1520 1518 ((proc_t *)cp->procp)->p_as->a_hat,
1521 1519 cp->cvaddr + off);
1522 1520 }
1523 1521 break;
1524 1522 case UMEM_TRASH :
1525 1523 pfnum = page_pptonum(trashpp);
1526 1524 /*
1527 1525 * We should set hat_flags to HAT_NOFAULT also
1528 1526 * However, not all hat layers implement this
1529 1527 */
1530 1528 break;
1531 1529 case KMEM_PAGEABLE:
1532 1530 case KMEM_NON_PAGEABLE:
1533 1531 pfnum = hat_getpfnum(kas.a_hat,
1534 1532 dhp->dh_cvaddr + off);
1535 1533 break;
1536 1534 default :
1537 1535 pfnum = PFN_INVALID;
1538 1536 break;
1539 1537 }
1540 1538 }
1541 1539 prot |= dhp->dh_hat_attr;
1542 1540 }
1543 1541 if (pfnum == PFN_INVALID) {
1544 1542 return (FC_MAKE_ERR(EFAULT));
1545 1543 }
1546 1544 /* prot should already be OR'ed in with hat_attributes if needed */
1547 1545
1548 1546 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGE_CK1,
1549 1547 "segdev_faultpage: pfnum=%lx memory=%x prot=%x flags=%x",
1550 1548 pfnum, pf_is_memory(pfnum), prot, hat_flags);
1551 1549 DEBUGF(9, (CE_CONT, "segdev_faultpage: pfnum %lx memory %x "
1552 1550 "prot %x flags %x\n", pfnum, pf_is_memory(pfnum), prot, hat_flags));
1553 1551
1554 1552 if (pf_is_memory(pfnum) || (dhp != NULL)) {
1555 1553 /*
1556 1554 * It's not _really_ required here to pass sdp->hat_flags
1557 1555 * to hat_devload even though we do it.
1558 1556 * This is because hat figures it out DEVMEM mappings
1559 1557 * are non-consistent, anyway.
1560 1558 */
1561 1559 hat_devload(hat, addr, PAGESIZE, pfnum,
1562 1560 prot, hat_flags | sdp->hat_flags);
1563 1561 return (0);
1564 1562 }
1565 1563
1566 1564 /*
1567 1565 * Fall through to the case where devmap is not used and need to call
1568 1566 * up the device tree to set up the mapping
1569 1567 */
1570 1568
1571 1569 dip = VTOS(VTOCVP(sdp->vp))->s_dip;
1572 1570 ASSERT(dip);
1573 1571
1574 1572 /*
1575 1573 * When calling ddi_map_fault, we do not OR in sdp->hat_attr
1576 1574 * This is because this calls drivers which may not expect
1577 1575 * prot to have any other values than PROT_ALL
1578 1576 * The root nexus driver has a hack to peek into the segment
1579 1577 * structure and then OR in sdp->hat_attr.
1580 1578 * XX In case the bus_ops interfaces are ever revisited
1581 1579 * we need to fix this. prot should include other hat attributes
1582 1580 */
1583 1581 if (ddi_map_fault(dip, hat, seg, addr, NULL, pfnum, prot & PROT_ALL,
1584 1582 (uint_t)(type == F_SOFTLOCK)) != DDI_SUCCESS) {
1585 1583 return (FC_MAKE_ERR(EFAULT));
1586 1584 }
1587 1585 return (0);
1588 1586 }
1589 1587
1590 1588 static faultcode_t
1591 1589 segdev_fault(
1592 1590 struct hat *hat, /* the hat */
1593 1591 struct seg *seg, /* the seg_dev of interest */
1594 1592 caddr_t addr, /* the address of the fault */
1595 1593 size_t len, /* the length of the range */
1596 1594 enum fault_type type, /* type of fault */
1597 1595 enum seg_rw rw) /* type of access at fault */
1598 1596 {
1599 1597 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1600 1598 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
1601 1599 devmap_handle_t *dhp;
1602 1600 struct devmap_softlock *slock = NULL;
1603 1601 ulong_t slpage = 0;
1604 1602 ulong_t off;
1605 1603 caddr_t maddr = addr;
1606 1604 int err;
1607 1605 int err_is_faultcode = 0;
1608 1606
1609 1607 TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_FAULT,
1610 1608 "segdev_fault: dhp_head=%p seg=%p addr=%p len=%lx type=%x",
1611 1609 (void *)dhp_head, (void *)seg, (void *)addr, len, type);
1612 1610 DEBUGF(7, (CE_CONT, "segdev_fault: dhp_head %p seg %p "
1613 1611 "addr %p len %lx type %x\n",
1614 1612 (void *)dhp_head, (void *)seg, (void *)addr, len, type));
1615 1613
1616 1614 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
1617 1615
1618 1616 /* Handle non-devmap case */
1619 1617 if (dhp_head == NULL)
1620 1618 return (segdev_faultpages(hat, seg, addr, len, type, rw, NULL));
1621 1619
1622 1620 /* Find devmap handle */
1623 1621 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
1624 1622 return (FC_NOMAP);
1625 1623
1626 1624 /*
1627 1625 * The seg_dev driver does not implement copy-on-write,
1628 1626 * and always loads translations with maximal allowed permissions
1629 1627 * but we got an fault trying to access the device.
1630 1628 * Servicing the fault is not going to result in any better result
1631 1629 * RFE: If we want devmap_access callbacks to be involved in F_PROT
1632 1630 * faults, then the code below is written for that
1633 1631 * Pending resolution of the following:
1634 1632 * - determine if the F_INVAL/F_SOFTLOCK syncing
1635 1633 * is needed for F_PROT also or not. The code below assumes it does
1636 1634 * - If driver sees F_PROT and calls devmap_load with same type,
1637 1635 * then segdev_faultpages will fail with FC_PROT anyway, need to
1638 1636 * change that so calls from devmap_load to segdev_faultpages for
1639 1637 * F_PROT type are retagged to F_INVAL.
1640 1638 * RFE: Today we dont have drivers that use devmap and want to handle
1641 1639 * F_PROT calls. The code in segdev_fault* is written to allow
1642 1640 * this case but is not tested. A driver that needs this capability
1643 1641 * should be able to remove the short-circuit case; resolve the
1644 1642 * above issues and "should" work.
1645 1643 */
1646 1644 if (type == F_PROT) {
1647 1645 return (FC_PROT);
1648 1646 }
1649 1647
1650 1648 /*
1651 1649 * Loop through dhp list calling devmap_access or segdev_faultpages for
1652 1650 * each devmap handle.
1653 1651 * drivers which implement devmap_access can interpose on faults and do
1654 1652 * device-appropriate special actions before calling devmap_load.
1655 1653 */
1656 1654
1657 1655 /*
1658 1656 * Unfortunately, this simple loop has turned out to expose a variety
1659 1657 * of complex problems which results in the following convoluted code.
1660 1658 *
1661 1659 * First, a desire to handle a serialization of F_SOFTLOCK calls
1662 1660 * to the driver within the framework.
1663 1661 * This results in a dh_softlock structure that is on a per device
1664 1662 * (or device instance) basis and serializes devmap_access calls.
1665 1663 * Ideally we would need to do this for underlying
1666 1664 * memory/device regions that are being faulted on
1667 1665 * but that is hard to identify and with REMAP, harder
1668 1666 * Second, a desire to serialize F_INVAL(and F_PROT) calls w.r.t.
1669 1667 * to F_SOFTLOCK calls to the driver.
1670 1668 * These serializations are to simplify the driver programmer model.
1671 1669 * To support these two features, the code first goes through the
1672 1670 * devmap handles and counts the pages (slpage) that are covered
1673 1671 * by devmap_access callbacks.
1674 1672 * This part ends with a devmap_softlock_enter call
1675 1673 * which allows only one F_SOFTLOCK active on a device instance,
1676 1674 * but multiple F_INVAL/F_PROTs can be active except when a
1677 1675 * F_SOFTLOCK is active
1678 1676 *
1679 1677 * Next, we dont short-circuit the fault code upfront to call
1680 1678 * segdev_softunlock for F_SOFTUNLOCK, because we must use
1681 1679 * the same length when we softlock and softunlock.
1682 1680 *
1683 1681 * -Hat layers may not support softunlocking lengths less than the
1684 1682 * original length when there is large page support.
1685 1683 * -kpmem locking is dependent on keeping the lengths same.
1686 1684 * -if drivers handled F_SOFTLOCK, they probably also expect to
1687 1685 * see an F_SOFTUNLOCK of the same length
1688 1686 * Hence, if extending lengths during softlock,
1689 1687 * softunlock has to make the same adjustments and goes through
1690 1688 * the same loop calling segdev_faultpages/segdev_softunlock
1691 1689 * But some of the synchronization and error handling is different
1692 1690 */
1693 1691
1694 1692 if (type != F_SOFTUNLOCK) {
1695 1693 devmap_handle_t *dhpp = dhp;
1696 1694 size_t slen = len;
1697 1695
1698 1696 /*
1699 1697 * Calculate count of pages that are :
1700 1698 * a) within the (potentially extended) fault region
1701 1699 * b) AND covered by devmap handle with devmap_access
1702 1700 */
1703 1701 off = (ulong_t)(addr - dhpp->dh_uvaddr);
1704 1702 while (slen != 0) {
1705 1703 size_t mlen;
1706 1704
1707 1705 /*
1708 1706 * Softlocking on a region that allows remap is
1709 1707 * unsupported due to unresolved locking issues
1710 1708 * XXX: unclear what these are?
1711 1709 * One potential is that if there is a pending
1712 1710 * softlock, then a remap should not be allowed
1713 1711 * until the unlock is done. This is easily
1714 1712 * fixed by returning error in devmap*remap on
1715 1713 * checking the dh->dh_softlock->softlocked value
1716 1714 */
1717 1715 if ((type == F_SOFTLOCK) &&
1718 1716 (dhpp->dh_flags & DEVMAP_ALLOW_REMAP)) {
1719 1717 return (FC_NOSUPPORT);
1720 1718 }
1721 1719
1722 1720 mlen = MIN(slen, (dhpp->dh_len - off));
1723 1721 if (dhpp->dh_callbackops.devmap_access) {
1724 1722 size_t llen;
1725 1723 caddr_t laddr;
1726 1724 /*
1727 1725 * use extended length for large page mappings
1728 1726 */
1729 1727 HOLD_DHP_LOCK(dhpp);
1730 1728 if ((sdp->pageprot == 0) &&
1731 1729 (dhpp->dh_flags & DEVMAP_FLAG_LARGE)) {
1732 1730 devmap_get_large_pgsize(dhpp,
1733 1731 mlen, maddr, &llen, &laddr);
1734 1732 } else {
1735 1733 llen = mlen;
1736 1734 }
1737 1735 RELE_DHP_LOCK(dhpp);
1738 1736
1739 1737 slpage += btopr(llen);
1740 1738 slock = dhpp->dh_softlock;
1741 1739 }
1742 1740 maddr += mlen;
1743 1741 ASSERT(slen >= mlen);
1744 1742 slen -= mlen;
1745 1743 dhpp = dhpp->dh_next;
1746 1744 off = 0;
1747 1745 }
1748 1746 /*
1749 1747 * synchonize with other faulting threads and wait till safe
1750 1748 * devmap_softlock_enter might return due to signal in cv_wait
1751 1749 *
1752 1750 * devmap_softlock_enter has to be called outside of while loop
1753 1751 * to prevent a deadlock if len spans over multiple dhps.
1754 1752 * dh_softlock is based on device instance and if multiple dhps
1755 1753 * use the same device instance, the second dhp's LOCK call
1756 1754 * will hang waiting on the first to complete.
1757 1755 * devmap_setup verifies that slocks in a dhp_chain are same.
1758 1756 * RFE: this deadlock only hold true for F_SOFTLOCK. For
1759 1757 * F_INVAL/F_PROT, since we now allow multiple in parallel,
1760 1758 * we could have done the softlock_enter inside the loop
1761 1759 * and supported multi-dhp mappings with dissimilar devices
1762 1760 */
1763 1761 if (err = devmap_softlock_enter(slock, slpage, type))
1764 1762 return (FC_MAKE_ERR(err));
1765 1763 }
1766 1764
1767 1765 /* reset 'maddr' to the start addr of the range of fault. */
1768 1766 maddr = addr;
1769 1767
1770 1768 /* calculate the offset corresponds to 'addr' in the first dhp. */
1771 1769 off = (ulong_t)(addr - dhp->dh_uvaddr);
1772 1770
1773 1771 /*
1774 1772 * The fault length may span over multiple dhps.
1775 1773 * Loop until the total length is satisfied.
1776 1774 */
1777 1775 while (len != 0) {
1778 1776 size_t llen;
1779 1777 size_t mlen;
1780 1778 caddr_t laddr;
1781 1779
1782 1780 /*
1783 1781 * mlen is the smaller of 'len' and the length
1784 1782 * from addr to the end of mapping defined by dhp.
1785 1783 */
1786 1784 mlen = MIN(len, (dhp->dh_len - off));
1787 1785
1788 1786 HOLD_DHP_LOCK(dhp);
1789 1787 /*
1790 1788 * Pass the extended length and address to devmap_access
1791 1789 * if large pagesize is used for loading address translations.
1792 1790 */
1793 1791 if ((sdp->pageprot == 0) &&
1794 1792 (dhp->dh_flags & DEVMAP_FLAG_LARGE)) {
1795 1793 devmap_get_large_pgsize(dhp, mlen, maddr,
1796 1794 &llen, &laddr);
1797 1795 ASSERT(maddr == addr || laddr == maddr);
1798 1796 } else {
1799 1797 llen = mlen;
1800 1798 laddr = maddr;
1801 1799 }
1802 1800
1803 1801 if (dhp->dh_callbackops.devmap_access != NULL) {
1804 1802 offset_t aoff;
1805 1803
1806 1804 aoff = sdp->offset + (offset_t)(laddr - seg->s_base);
1807 1805
1808 1806 /*
1809 1807 * call driver's devmap_access entry point which will
1810 1808 * call devmap_load/contextmgmt to load the translations
1811 1809 *
1812 1810 * We drop the dhp_lock before calling access so
1813 1811 * drivers can call devmap_*_remap within access
1814 1812 */
1815 1813 RELE_DHP_LOCK(dhp);
1816 1814
1817 1815 err = (*dhp->dh_callbackops.devmap_access)(
1818 1816 dhp, (void *)dhp->dh_pvtp, aoff, llen, type, rw);
1819 1817 } else {
1820 1818 /*
1821 1819 * If no devmap_access entry point, then load mappings
1822 1820 * hold dhp_lock across faultpages if REMAP
1823 1821 */
1824 1822 err = segdev_faultpages(hat, seg, laddr, llen,
1825 1823 type, rw, dhp);
1826 1824 err_is_faultcode = 1;
1827 1825 RELE_DHP_LOCK(dhp);
1828 1826 }
1829 1827
1830 1828 if (err) {
1831 1829 if ((type == F_SOFTLOCK) && (maddr > addr)) {
1832 1830 /*
1833 1831 * If not first dhp, use
1834 1832 * segdev_fault(F_SOFTUNLOCK) for prior dhps
1835 1833 * While this is recursion, it is incorrect to
1836 1834 * call just segdev_softunlock
1837 1835 * if we are using either large pages
1838 1836 * or devmap_access. It will be more right
1839 1837 * to go through the same loop as above
1840 1838 * rather than call segdev_softunlock directly
1841 1839 * It will use the right lenghths as well as
1842 1840 * call into the driver devmap_access routines.
1843 1841 */
1844 1842 size_t done = (size_t)(maddr - addr);
1845 1843 (void) segdev_fault(hat, seg, addr, done,
1846 1844 F_SOFTUNLOCK, S_OTHER);
1847 1845 /*
1848 1846 * reduce slpage by number of pages
1849 1847 * released by segdev_softunlock
1850 1848 */
1851 1849 ASSERT(slpage >= btopr(done));
1852 1850 devmap_softlock_exit(slock,
1853 1851 slpage - btopr(done), type);
1854 1852 } else {
1855 1853 devmap_softlock_exit(slock, slpage, type);
1856 1854 }
1857 1855
1858 1856
1859 1857 /*
1860 1858 * Segdev_faultpages() already returns a faultcode,
1861 1859 * hence, result from segdev_faultpages() should be
1862 1860 * returned directly.
1863 1861 */
1864 1862 if (err_is_faultcode)
1865 1863 return (err);
1866 1864 return (FC_MAKE_ERR(err));
1867 1865 }
1868 1866
1869 1867 maddr += mlen;
1870 1868 ASSERT(len >= mlen);
1871 1869 len -= mlen;
1872 1870 dhp = dhp->dh_next;
1873 1871 off = 0;
1874 1872
1875 1873 ASSERT(!dhp || len == 0 || maddr == dhp->dh_uvaddr);
1876 1874 }
1877 1875 /*
1878 1876 * release the softlock count at end of fault
1879 1877 * For F_SOFTLOCk this is done in the later F_SOFTUNLOCK
1880 1878 */
1881 1879 if ((type == F_INVAL) || (type == F_PROT))
1882 1880 devmap_softlock_exit(slock, slpage, type);
1883 1881 return (0);
1884 1882 }
1885 1883
1886 1884 /*
1887 1885 * segdev_faultpages
1888 1886 *
1889 1887 * Used to fault in seg_dev segment pages. Called by segdev_fault or devmap_load
1890 1888 * This routine assumes that the callers makes sure that the fields
1891 1889 * in dhp used below are not changed due to remap during this call.
1892 1890 * Caller does HOLD_DHP_LOCK if neeed
1893 1891 * This routine returns a faultcode_t as a return value for segdev_fault.
1894 1892 */
1895 1893 static faultcode_t
1896 1894 segdev_faultpages(
1897 1895 struct hat *hat, /* the hat */
1898 1896 struct seg *seg, /* the seg_dev of interest */
1899 1897 caddr_t addr, /* the address of the fault */
1900 1898 size_t len, /* the length of the range */
1901 1899 enum fault_type type, /* type of fault */
1902 1900 enum seg_rw rw, /* type of access at fault */
1903 1901 devmap_handle_t *dhp) /* devmap handle */
1904 1902 {
1905 1903 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
1906 1904 register caddr_t a;
1907 1905 struct vpage *vpage;
1908 1906 struct ddi_umem_cookie *kpmem_cookie = NULL;
1909 1907 int err;
1910 1908
1911 1909 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_FAULTPAGES,
1912 1910 "segdev_faultpages: dhp=%p seg=%p addr=%p len=%lx",
1913 1911 (void *)dhp, (void *)seg, (void *)addr, len);
1914 1912 DEBUGF(5, (CE_CONT, "segdev_faultpages: "
1915 1913 "dhp %p seg %p addr %p len %lx\n",
1916 1914 (void *)dhp, (void *)seg, (void *)addr, len));
1917 1915
1918 1916 /*
1919 1917 * The seg_dev driver does not implement copy-on-write,
1920 1918 * and always loads translations with maximal allowed permissions
1921 1919 * but we got an fault trying to access the device.
1922 1920 * Servicing the fault is not going to result in any better result
1923 1921 * XXX: If we want to allow devmap_access to handle F_PROT calls,
1924 1922 * This code should be removed and let the normal fault handling
1925 1923 * take care of finding the error
1926 1924 */
1927 1925 if (type == F_PROT) {
1928 1926 return (FC_PROT);
1929 1927 }
1930 1928
1931 1929 if (type == F_SOFTUNLOCK) {
1932 1930 segdev_softunlock(hat, seg, addr, len, rw);
1933 1931 return (0);
1934 1932 }
1935 1933
1936 1934 /*
1937 1935 * For kernel pageable memory, fault/lock segkp pages
1938 1936 * We hold this until the completion of this
1939 1937 * fault (INVAL/PROT) or till unlock (SOFTLOCK).
1940 1938 */
1941 1939 if ((dhp != NULL) && dhp_is_kpmem(dhp)) {
1942 1940 kpmem_cookie = (struct ddi_umem_cookie *)dhp->dh_cookie;
1943 1941 if (err = acquire_kpmem_lock(kpmem_cookie, btopr(len)))
1944 1942 return (err);
1945 1943 }
1946 1944
1947 1945 /*
1948 1946 * If we have the same protections for the entire segment,
1949 1947 * insure that the access being attempted is legitimate.
1950 1948 */
1951 1949 rw_enter(&sdp->lock, RW_READER);
1952 1950 if (sdp->pageprot == 0) {
1953 1951 uint_t protchk;
1954 1952
1955 1953 switch (rw) {
1956 1954 case S_READ:
1957 1955 protchk = PROT_READ;
1958 1956 break;
1959 1957 case S_WRITE:
1960 1958 protchk = PROT_WRITE;
1961 1959 break;
1962 1960 case S_EXEC:
1963 1961 protchk = PROT_EXEC;
1964 1962 break;
1965 1963 case S_OTHER:
1966 1964 default:
1967 1965 protchk = PROT_READ | PROT_WRITE | PROT_EXEC;
1968 1966 break;
1969 1967 }
1970 1968
1971 1969 if ((sdp->prot & protchk) == 0) {
1972 1970 rw_exit(&sdp->lock);
1973 1971 /* undo kpmem locking */
1974 1972 if (kpmem_cookie != NULL) {
1975 1973 release_kpmem_lock(kpmem_cookie, btopr(len));
1976 1974 }
1977 1975 return (FC_PROT); /* illegal access type */
1978 1976 }
1979 1977 }
1980 1978
1981 1979 /*
1982 1980 * we do a single hat_devload for the range if
1983 1981 * - devmap framework (dhp is not NULL),
1984 1982 * - pageprot == 0, i.e., no per-page protection set and
1985 1983 * - is device pages, irrespective of whether we are using large pages
1986 1984 */
1987 1985 if ((sdp->pageprot == 0) && (dhp != NULL) && dhp_is_devmem(dhp)) {
1988 1986 pfn_t pfnum;
1989 1987 uint_t hat_flags;
1990 1988
1991 1989 if (dhp->dh_flags & DEVMAP_MAPPING_INVALID) {
1992 1990 rw_exit(&sdp->lock);
1993 1991 return (FC_NOMAP);
1994 1992 }
1995 1993
1996 1994 if (type == F_SOFTLOCK) {
1997 1995 mutex_enter(&freemem_lock);
1998 1996 sdp->softlockcnt += btopr(len);
1999 1997 mutex_exit(&freemem_lock);
2000 1998 }
2001 1999
2002 2000 hat_flags = ((type == F_SOFTLOCK) ? HAT_LOAD_LOCK : HAT_LOAD);
2003 2001 pfnum = dhp->dh_pfn + btop((uintptr_t)(addr - dhp->dh_uvaddr));
2004 2002 ASSERT(!pf_is_memory(pfnum));
2005 2003
2006 2004 hat_devload(hat, addr, len, pfnum, sdp->prot | dhp->dh_hat_attr,
2007 2005 hat_flags | sdp->hat_flags);
2008 2006 rw_exit(&sdp->lock);
2009 2007 return (0);
2010 2008 }
2011 2009
2012 2010 /* Handle cases where we have to loop through fault handling per-page */
2013 2011
2014 2012 if (sdp->vpage == NULL)
2015 2013 vpage = NULL;
2016 2014 else
2017 2015 vpage = &sdp->vpage[seg_page(seg, addr)];
2018 2016
2019 2017 /* loop over the address range handling each fault */
2020 2018 for (a = addr; a < addr + len; a += PAGESIZE) {
2021 2019 if (err = segdev_faultpage(hat, seg, a, vpage, type, rw, dhp)) {
2022 2020 break;
2023 2021 }
2024 2022 if (vpage != NULL)
2025 2023 vpage++;
2026 2024 }
2027 2025 rw_exit(&sdp->lock);
2028 2026 if (err && (type == F_SOFTLOCK)) { /* error handling for F_SOFTLOCK */
2029 2027 size_t done = (size_t)(a - addr); /* pages fault successfully */
2030 2028 if (done > 0) {
2031 2029 /* use softunlock for those pages */
2032 2030 segdev_softunlock(hat, seg, addr, done, S_OTHER);
2033 2031 }
2034 2032 if (kpmem_cookie != NULL) {
2035 2033 /* release kpmem lock for rest of pages */
2036 2034 ASSERT(len >= done);
2037 2035 release_kpmem_lock(kpmem_cookie, btopr(len - done));
2038 2036 }
2039 2037 } else if ((kpmem_cookie != NULL) && (type != F_SOFTLOCK)) {
2040 2038 /* for non-SOFTLOCK cases, release kpmem */
2041 2039 release_kpmem_lock(kpmem_cookie, btopr(len));
2042 2040 }
2043 2041 return (err);
2044 2042 }
2045 2043
2046 2044 /*
2047 2045 * Asynchronous page fault. We simply do nothing since this
2048 2046 * entry point is not supposed to load up the translation.
2049 2047 */
2050 2048 /*ARGSUSED*/
2051 2049 static faultcode_t
2052 2050 segdev_faulta(struct seg *seg, caddr_t addr)
2053 2051 {
2054 2052 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_FAULTA,
2055 2053 "segdev_faulta: seg=%p addr=%p", (void *)seg, (void *)addr);
2056 2054 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2057 2055
2058 2056 return (0);
2059 2057 }
2060 2058
2061 2059 static int
2062 2060 segdev_setprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2063 2061 {
2064 2062 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2065 2063 register devmap_handle_t *dhp;
2066 2064 register struct vpage *vp, *evp;
2067 2065 devmap_handle_t *dhp_head = (devmap_handle_t *)sdp->devmap_data;
2068 2066 ulong_t off;
2069 2067 size_t mlen, sz;
2070 2068
2071 2069 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT,
2072 2070 "segdev_setprot:start seg=%p addr=%p len=%lx prot=%x",
2073 2071 (void *)seg, (void *)addr, len, prot);
2074 2072 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2075 2073
2076 2074 if ((sz = sdp->softlockcnt) > 0 && dhp_head != NULL) {
2077 2075 /*
2078 2076 * Fail the setprot if pages are SOFTLOCKed through this
2079 2077 * mapping.
2080 2078 * Softlockcnt is protected from change by the as read lock.
2081 2079 */
2082 2080 TRACE_1(TR_FAC_DEVMAP, TR_DEVMAP_SETPROT_CK1,
2083 2081 "segdev_setprot:error softlockcnt=%lx", sz);
2084 2082 DEBUGF(1, (CE_CONT, "segdev_setprot: softlockcnt %ld\n", sz));
2085 2083 return (EAGAIN);
2086 2084 }
2087 2085
2088 2086 if (dhp_head != NULL) {
2089 2087 if ((dhp = devmap_find_handle(dhp_head, addr)) == NULL)
2090 2088 return (EINVAL);
2091 2089
2092 2090 /*
2093 2091 * check if violate maxprot.
2094 2092 */
2095 2093 off = (ulong_t)(addr - dhp->dh_uvaddr);
2096 2094 mlen = len;
2097 2095 while (dhp) {
2098 2096 if ((dhp->dh_maxprot & prot) != prot)
2099 2097 return (EACCES); /* violated maxprot */
2100 2098
2101 2099 if (mlen > (dhp->dh_len - off)) {
2102 2100 mlen -= dhp->dh_len - off;
2103 2101 dhp = dhp->dh_next;
2104 2102 off = 0;
2105 2103 } else
2106 2104 break;
2107 2105 }
2108 2106 } else {
2109 2107 if ((sdp->maxprot & prot) != prot)
2110 2108 return (EACCES);
2111 2109 }
2112 2110
2113 2111 rw_enter(&sdp->lock, RW_WRITER);
2114 2112 if (addr == seg->s_base && len == seg->s_size && sdp->pageprot == 0) {
2115 2113 if (sdp->prot == prot) {
2116 2114 rw_exit(&sdp->lock);
2117 2115 return (0); /* all done */
2118 2116 }
2119 2117 sdp->prot = (uchar_t)prot;
2120 2118 } else {
2121 2119 sdp->pageprot = 1;
2122 2120 if (sdp->vpage == NULL) {
2123 2121 /*
2124 2122 * First time through setting per page permissions,
2125 2123 * initialize all the vpage structures to prot
2126 2124 */
2127 2125 sdp->vpage = kmem_zalloc(vpgtob(seg_pages(seg)),
2128 2126 KM_SLEEP);
2129 2127 evp = &sdp->vpage[seg_pages(seg)];
2130 2128 for (vp = sdp->vpage; vp < evp; vp++)
2131 2129 VPP_SETPROT(vp, sdp->prot);
2132 2130 }
2133 2131 /*
2134 2132 * Now go change the needed vpages protections.
2135 2133 */
2136 2134 evp = &sdp->vpage[seg_page(seg, addr + len)];
2137 2135 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++)
2138 2136 VPP_SETPROT(vp, prot);
2139 2137 }
2140 2138 rw_exit(&sdp->lock);
2141 2139
2142 2140 if (dhp_head != NULL) {
2143 2141 devmap_handle_t *tdhp;
2144 2142 /*
2145 2143 * If large page size was used in hat_devload(),
2146 2144 * the same page size must be used in hat_unload().
2147 2145 */
2148 2146 dhp = tdhp = devmap_find_handle(dhp_head, addr);
2149 2147 while (tdhp != NULL) {
2150 2148 if (tdhp->dh_flags & DEVMAP_FLAG_LARGE) {
2151 2149 break;
2152 2150 }
2153 2151 tdhp = tdhp->dh_next;
2154 2152 }
2155 2153 if (tdhp) {
2156 2154 size_t slen = len;
2157 2155 size_t mlen;
2158 2156 size_t soff;
2159 2157
2160 2158 soff = (ulong_t)(addr - dhp->dh_uvaddr);
2161 2159 while (slen != 0) {
2162 2160 mlen = MIN(slen, (dhp->dh_len - soff));
2163 2161 hat_unload(seg->s_as->a_hat, dhp->dh_uvaddr,
2164 2162 dhp->dh_len, HAT_UNLOAD);
2165 2163 dhp = dhp->dh_next;
2166 2164 ASSERT(slen >= mlen);
2167 2165 slen -= mlen;
2168 2166 soff = 0;
2169 2167 }
2170 2168 return (0);
2171 2169 }
2172 2170 }
2173 2171
2174 2172 if ((prot & ~PROT_USER) == PROT_NONE) {
2175 2173 hat_unload(seg->s_as->a_hat, addr, len, HAT_UNLOAD);
2176 2174 } else {
2177 2175 /*
2178 2176 * RFE: the segment should keep track of all attributes
2179 2177 * allowing us to remove the deprecated hat_chgprot
2180 2178 * and use hat_chgattr.
2181 2179 */
2182 2180 hat_chgprot(seg->s_as->a_hat, addr, len, prot);
2183 2181 }
2184 2182
2185 2183 return (0);
2186 2184 }
2187 2185
2188 2186 static int
2189 2187 segdev_checkprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
2190 2188 {
2191 2189 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2192 2190 struct vpage *vp, *evp;
2193 2191
2194 2192 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_CHECKPROT,
2195 2193 "segdev_checkprot:start seg=%p addr=%p len=%lx prot=%x",
2196 2194 (void *)seg, (void *)addr, len, prot);
2197 2195 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2198 2196
2199 2197 /*
2200 2198 * If segment protection can be used, simply check against them
2201 2199 */
2202 2200 rw_enter(&sdp->lock, RW_READER);
2203 2201 if (sdp->pageprot == 0) {
2204 2202 register int err;
2205 2203
2206 2204 err = ((sdp->prot & prot) != prot) ? EACCES : 0;
2207 2205 rw_exit(&sdp->lock);
2208 2206 return (err);
2209 2207 }
2210 2208
2211 2209 /*
2212 2210 * Have to check down to the vpage level
2213 2211 */
2214 2212 evp = &sdp->vpage[seg_page(seg, addr + len)];
2215 2213 for (vp = &sdp->vpage[seg_page(seg, addr)]; vp < evp; vp++) {
2216 2214 if ((VPP_PROT(vp) & prot) != prot) {
2217 2215 rw_exit(&sdp->lock);
2218 2216 return (EACCES);
2219 2217 }
2220 2218 }
2221 2219 rw_exit(&sdp->lock);
2222 2220 return (0);
2223 2221 }
2224 2222
2225 2223 static int
2226 2224 segdev_getprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
2227 2225 {
2228 2226 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2229 2227 size_t pgno;
2230 2228
2231 2229 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_GETPROT,
2232 2230 "segdev_getprot:start seg=%p addr=%p len=%lx protv=%p",
2233 2231 (void *)seg, (void *)addr, len, (void *)protv);
2234 2232 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2235 2233
2236 2234 pgno = seg_page(seg, addr + len) - seg_page(seg, addr) + 1;
2237 2235 if (pgno != 0) {
2238 2236 rw_enter(&sdp->lock, RW_READER);
2239 2237 if (sdp->pageprot == 0) {
2240 2238 do {
2241 2239 protv[--pgno] = sdp->prot;
2242 2240 } while (pgno != 0);
2243 2241 } else {
2244 2242 size_t pgoff = seg_page(seg, addr);
2245 2243
2246 2244 do {
2247 2245 pgno--;
2248 2246 protv[pgno] =
2249 2247 VPP_PROT(&sdp->vpage[pgno + pgoff]);
2250 2248 } while (pgno != 0);
2251 2249 }
2252 2250 rw_exit(&sdp->lock);
2253 2251 }
2254 2252 return (0);
2255 2253 }
2256 2254
2257 2255 static u_offset_t
2258 2256 segdev_getoffset(register struct seg *seg, caddr_t addr)
2259 2257 {
2260 2258 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2261 2259
2262 2260 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETOFFSET,
2263 2261 "segdev_getoffset:start seg=%p addr=%p", (void *)seg, (void *)addr);
2264 2262
2265 2263 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2266 2264
2267 2265 return ((u_offset_t)sdp->offset + (addr - seg->s_base));
2268 2266 }
2269 2267
2270 2268 /*ARGSUSED*/
2271 2269 static int
2272 2270 segdev_gettype(register struct seg *seg, caddr_t addr)
2273 2271 {
2274 2272 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2275 2273
2276 2274 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETTYPE,
2277 2275 "segdev_gettype:start seg=%p addr=%p", (void *)seg, (void *)addr);
2278 2276
2279 2277 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2280 2278
2281 2279 return (sdp->type);
2282 2280 }
2283 2281
2284 2282
2285 2283 /*ARGSUSED*/
2286 2284 static int
2287 2285 segdev_getvp(register struct seg *seg, caddr_t addr, struct vnode **vpp)
2288 2286 {
2289 2287 register struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
2290 2288
2291 2289 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_GETVP,
2292 2290 "segdev_getvp:start seg=%p addr=%p", (void *)seg, (void *)addr);
2293 2291
2294 2292 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2295 2293
2296 2294 /*
2297 2295 * Note that this vp is the common_vp of the device, where the
2298 2296 * pages are hung ..
2299 2297 */
2300 2298 *vpp = VTOCVP(sdp->vp);
2301 2299
2302 2300 return (0);
2303 2301 }
2304 2302
2305 2303 static void
2306 2304 segdev_badop(void)
2307 2305 {
2308 2306 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGDEV_BADOP,
2309 2307 "segdev_badop:start");
2310 2308 panic("segdev_badop");
2311 2309 /*NOTREACHED*/
2312 2310 }
2313 2311
2314 2312 /*
2315 2313 * segdev pages are not in the cache, and thus can't really be controlled.
2316 2314 * Hence, syncs are simply always successful.
2317 2315 */
2318 2316 /*ARGSUSED*/
2319 2317 static int
2320 2318 segdev_sync(struct seg *seg, caddr_t addr, size_t len, int attr, uint_t flags)
2321 2319 {
2322 2320 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SYNC, "segdev_sync:start");
2323 2321
2324 2322 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2325 2323
2326 2324 return (0);
2327 2325 }
2328 2326
2329 2327 /*
2330 2328 * segdev pages are always "in core".
2331 2329 */
2332 2330 /*ARGSUSED*/
2333 2331 static size_t
2334 2332 segdev_incore(struct seg *seg, caddr_t addr, size_t len, char *vec)
2335 2333 {
2336 2334 size_t v = 0;
2337 2335
2338 2336 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_INCORE, "segdev_incore:start");
2339 2337
2340 2338 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2341 2339
2342 2340 for (len = (len + PAGEOFFSET) & PAGEMASK; len; len -= PAGESIZE,
2343 2341 v += PAGESIZE)
2344 2342 *vec++ = 1;
2345 2343 return (v);
2346 2344 }
2347 2345
2348 2346 /*
2349 2347 * segdev pages are not in the cache, and thus can't really be controlled.
2350 2348 * Hence, locks are simply always successful.
2351 2349 */
2352 2350 /*ARGSUSED*/
2353 2351 static int
2354 2352 segdev_lockop(struct seg *seg, caddr_t addr,
2355 2353 size_t len, int attr, int op, ulong_t *lockmap, size_t pos)
2356 2354 {
2357 2355 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_LOCKOP, "segdev_lockop:start");
2358 2356
2359 2357 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2360 2358
2361 2359 return (0);
2362 2360 }
2363 2361
2364 2362 /*
2365 2363 * segdev pages are not in the cache, and thus can't really be controlled.
2366 2364 * Hence, advise is simply always successful.
2367 2365 */
2368 2366 /*ARGSUSED*/
2369 2367 static int
2370 2368 segdev_advise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
2371 2369 {
2372 2370 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_ADVISE, "segdev_advise:start");
2373 2371
2374 2372 ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2375 2373
2376 2374 return (0);
2377 2375 }
2378 2376
2379 2377 /*
2380 2378 * segdev pages are not dumped, so we just return
2381 2379 */
2382 2380 /*ARGSUSED*/
2383 2381 static void
2384 2382 segdev_dump(struct seg *seg)
2385 2383 {}
2386 2384
2387 2385 /*
2388 2386 * ddi_segmap_setup: Used by drivers who wish specify mapping attributes
2389 2387 * for a segment. Called from a drivers segmap(9E)
2390 2388 * routine.
2391 2389 */
2392 2390 /*ARGSUSED*/
2393 2391 int
2394 2392 ddi_segmap_setup(dev_t dev, off_t offset, struct as *as, caddr_t *addrp,
2395 2393 off_t len, uint_t prot, uint_t maxprot, uint_t flags, cred_t *cred,
2396 2394 ddi_device_acc_attr_t *accattrp, uint_t rnumber)
2397 2395 {
2398 2396 struct segdev_crargs dev_a;
2399 2397 int (*mapfunc)(dev_t dev, off_t off, int prot);
2400 2398 uint_t hat_attr;
2401 2399 pfn_t pfn;
2402 2400 int error, i;
2403 2401
2404 2402 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP_SETUP,
2405 2403 "ddi_segmap_setup:start");
2406 2404
2407 2405 if ((mapfunc = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap) == nodev)
2408 2406 return (ENODEV);
2409 2407
2410 2408 /*
2411 2409 * Character devices that support the d_mmap
2412 2410 * interface can only be mmap'ed shared.
2413 2411 */
2414 2412 if ((flags & MAP_TYPE) != MAP_SHARED)
2415 2413 return (EINVAL);
2416 2414
2417 2415 /*
2418 2416 * Check that this region is indeed mappable on this platform.
2419 2417 * Use the mapping function.
2420 2418 */
2421 2419 if (ddi_device_mapping_check(dev, accattrp, rnumber, &hat_attr) == -1)
2422 2420 return (ENXIO);
2423 2421
2424 2422 /*
2425 2423 * Check to ensure that the entire range is
2426 2424 * legal and we are not trying to map in
2427 2425 * more than the device will let us.
2428 2426 */
2429 2427 for (i = 0; i < len; i += PAGESIZE) {
2430 2428 if (i == 0) {
2431 2429 /*
2432 2430 * Save the pfn at offset here. This pfn will be
2433 2431 * used later to get user address.
2434 2432 */
2435 2433 if ((pfn = (pfn_t)cdev_mmap(mapfunc, dev, offset,
2436 2434 maxprot)) == PFN_INVALID)
2437 2435 return (ENXIO);
2438 2436 } else {
2439 2437 if (cdev_mmap(mapfunc, dev, offset + i, maxprot) ==
2440 2438 PFN_INVALID)
2441 2439 return (ENXIO);
2442 2440 }
2443 2441 }
2444 2442
2445 2443 as_rangelock(as);
2446 2444 /* Pick an address w/o worrying about any vac alignment constraints. */
2447 2445 error = choose_addr(as, addrp, len, ptob(pfn), ADDR_NOVACALIGN, flags);
2448 2446 if (error != 0) {
2449 2447 as_rangeunlock(as);
2450 2448 return (error);
2451 2449 }
2452 2450
2453 2451 dev_a.mapfunc = mapfunc;
2454 2452 dev_a.dev = dev;
2455 2453 dev_a.offset = (offset_t)offset;
2456 2454 dev_a.type = flags & MAP_TYPE;
2457 2455 dev_a.prot = (uchar_t)prot;
2458 2456 dev_a.maxprot = (uchar_t)maxprot;
2459 2457 dev_a.hat_attr = hat_attr;
2460 2458 dev_a.hat_flags = 0;
2461 2459 dev_a.devmap_data = NULL;
2462 2460
2463 2461 error = as_map(as, *addrp, len, segdev_create, &dev_a);
2464 2462 as_rangeunlock(as);
2465 2463 return (error);
2466 2464
2467 2465 }
2468 2466
2469 2467 /*ARGSUSED*/
2470 2468 static int
2471 2469 segdev_pagelock(struct seg *seg, caddr_t addr, size_t len,
2472 2470 struct page ***ppp, enum lock_type type, enum seg_rw rw)
2473 2471 {
2474 2472 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_PAGELOCK,
2475 2473 "segdev_pagelock:start");
2476 2474 return (ENOTSUP);
2477 2475 }
2478 2476
2479 2477 /*ARGSUSED*/
2480 2478 static int
2481 2479 segdev_setpagesize(struct seg *seg, caddr_t addr, size_t len,
2482 2480 uint_t szc)
2483 2481 {
2484 2482 return (ENOTSUP);
2485 2483 }
2486 2484
2487 2485 /*
2488 2486 * devmap_device: Used by devmap framework to establish mapping
2489 2487 * called by devmap_seup(9F) during map setup time.
2490 2488 */
2491 2489 /*ARGSUSED*/
2492 2490 static int
2493 2491 devmap_device(devmap_handle_t *dhp, struct as *as, caddr_t *addr,
2494 2492 offset_t off, size_t len, uint_t flags)
2495 2493 {
2496 2494 devmap_handle_t *rdhp, *maxdhp;
2497 2495 struct segdev_crargs dev_a;
2498 2496 int err;
2499 2497 uint_t maxprot = PROT_ALL;
2500 2498 offset_t offset = 0;
2501 2499 pfn_t pfn;
2502 2500 struct devmap_pmem_cookie *pcp;
2503 2501
2504 2502 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVICE,
2505 2503 "devmap_device:start dhp=%p addr=%p off=%llx, len=%lx",
2506 2504 (void *)dhp, (void *)addr, off, len);
2507 2505
2508 2506 DEBUGF(2, (CE_CONT, "devmap_device: dhp %p addr %p off %llx len %lx\n",
2509 2507 (void *)dhp, (void *)addr, off, len));
2510 2508
2511 2509 as_rangelock(as);
2512 2510 if ((flags & MAP_FIXED) == 0) {
2513 2511 offset_t aligned_off;
2514 2512
2515 2513 rdhp = maxdhp = dhp;
2516 2514 while (rdhp != NULL) {
2517 2515 maxdhp = (maxdhp->dh_len > rdhp->dh_len) ?
2518 2516 maxdhp : rdhp;
2519 2517 rdhp = rdhp->dh_next;
2520 2518 maxprot |= dhp->dh_maxprot;
2521 2519 }
2522 2520 offset = maxdhp->dh_uoff - dhp->dh_uoff;
2523 2521
2524 2522 /*
2525 2523 * Use the dhp that has the
2526 2524 * largest len to get user address.
2527 2525 */
2528 2526 /*
2529 2527 * If MAPPING_INVALID, cannot use dh_pfn/dh_cvaddr,
2530 2528 * use 0 which is as good as any other.
2531 2529 */
2532 2530 if (maxdhp->dh_flags & DEVMAP_MAPPING_INVALID) {
2533 2531 aligned_off = (offset_t)0;
2534 2532 } else if (dhp_is_devmem(maxdhp)) {
2535 2533 aligned_off = (offset_t)ptob(maxdhp->dh_pfn) - offset;
2536 2534 } else if (dhp_is_pmem(maxdhp)) {
2537 2535 pcp = (struct devmap_pmem_cookie *)maxdhp->dh_pcookie;
2538 2536 pfn = page_pptonum(
2539 2537 pcp->dp_pparray[btop(maxdhp->dh_roff)]);
2540 2538 aligned_off = (offset_t)ptob(pfn) - offset;
2541 2539 } else {
2542 2540 aligned_off = (offset_t)(uintptr_t)maxdhp->dh_cvaddr -
2543 2541 offset;
2544 2542 }
2545 2543
2546 2544 /*
2547 2545 * Pick an address aligned to dh_cookie.
2548 2546 * for kernel memory/user memory, cookie is cvaddr.
2549 2547 * for device memory, cookie is physical address.
2550 2548 */
2551 2549 map_addr(addr, len, aligned_off, 1, flags);
2552 2550 if (*addr == NULL) {
2553 2551 as_rangeunlock(as);
2554 2552 return (ENOMEM);
2555 2553 }
2556 2554 } else {
2557 2555 /*
2558 2556 * User-specified address; blow away any previous mappings.
2559 2557 */
2560 2558 (void) as_unmap(as, *addr, len);
2561 2559 }
2562 2560
2563 2561 dev_a.mapfunc = NULL;
2564 2562 dev_a.dev = dhp->dh_dev;
2565 2563 dev_a.type = flags & MAP_TYPE;
2566 2564 dev_a.offset = off;
2567 2565 /*
2568 2566 * sdp->maxprot has the least restrict protection of all dhps.
2569 2567 */
2570 2568 dev_a.maxprot = maxprot;
2571 2569 dev_a.prot = dhp->dh_prot;
2572 2570 /*
2573 2571 * devmap uses dhp->dh_hat_attr for hat.
2574 2572 */
2575 2573 dev_a.hat_flags = 0;
2576 2574 dev_a.hat_attr = 0;
2577 2575 dev_a.devmap_data = (void *)dhp;
2578 2576
2579 2577 err = as_map(as, *addr, len, segdev_create, &dev_a);
2580 2578 as_rangeunlock(as);
2581 2579 return (err);
2582 2580 }
2583 2581
2584 2582 int
2585 2583 devmap_do_ctxmgt(devmap_cookie_t dhc, void *pvtp, offset_t off, size_t len,
2586 2584 uint_t type, uint_t rw, int (*ctxmgt)(devmap_cookie_t, void *, offset_t,
2587 2585 size_t, uint_t, uint_t))
2588 2586 {
2589 2587 register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2590 2588 struct devmap_ctx *devctx;
2591 2589 int do_timeout = 0;
2592 2590 int ret;
2593 2591
2594 2592 #ifdef lint
2595 2593 pvtp = pvtp;
2596 2594 #endif
2597 2595
2598 2596 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT,
2599 2597 "devmap_do_ctxmgt:start dhp=%p off=%llx, len=%lx",
2600 2598 (void *)dhp, off, len);
2601 2599 DEBUGF(7, (CE_CONT, "devmap_do_ctxmgt: dhp %p off %llx len %lx\n",
2602 2600 (void *)dhp, off, len));
2603 2601
2604 2602 if (ctxmgt == NULL)
2605 2603 return (FC_HWERR);
2606 2604
2607 2605 devctx = dhp->dh_ctx;
2608 2606
2609 2607 /*
2610 2608 * If we are on an MP system with more than one cpu running
2611 2609 * and if a thread on some CPU already has the context, wait
2612 2610 * for it to finish if there is a hysteresis timeout.
2613 2611 *
2614 2612 * We call cv_wait() instead of cv_wait_sig() because
2615 2613 * it does not matter much if it returned due to a signal
2616 2614 * or due to a cv_signal() or cv_broadcast(). In either event
2617 2615 * we need to complete the mapping otherwise the processes
2618 2616 * will die with a SEGV.
2619 2617 */
2620 2618 if ((dhp->dh_timeout_length > 0) && (ncpus > 1)) {
2621 2619 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK1,
2622 2620 "devmap_do_ctxmgt:doing hysteresis, devctl %p dhp %p",
2623 2621 devctx, dhp);
2624 2622 do_timeout = 1;
2625 2623 mutex_enter(&devctx->lock);
2626 2624 while (devctx->oncpu)
2627 2625 cv_wait(&devctx->cv, &devctx->lock);
2628 2626 devctx->oncpu = 1;
2629 2627 mutex_exit(&devctx->lock);
2630 2628 }
2631 2629
2632 2630 /*
2633 2631 * Call the contextmgt callback so that the driver can handle
2634 2632 * the fault.
2635 2633 */
2636 2634 ret = (*ctxmgt)(dhp, dhp->dh_pvtp, off, len, type, rw);
2637 2635
2638 2636 /*
2639 2637 * If devmap_access() returned -1, then there was a hardware
2640 2638 * error so we need to convert the return value to something
2641 2639 * that trap() will understand. Otherwise, the return value
2642 2640 * is already a fault code generated by devmap_unload()
2643 2641 * or devmap_load().
2644 2642 */
2645 2643 if (ret) {
2646 2644 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK2,
2647 2645 "devmap_do_ctxmgt: ret=%x dhp=%p devctx=%p",
2648 2646 ret, dhp, devctx);
2649 2647 DEBUGF(1, (CE_CONT, "devmap_do_ctxmgt: ret %x dhp %p\n",
2650 2648 ret, (void *)dhp));
2651 2649 if (devctx->oncpu) {
2652 2650 mutex_enter(&devctx->lock);
2653 2651 devctx->oncpu = 0;
2654 2652 cv_signal(&devctx->cv);
2655 2653 mutex_exit(&devctx->lock);
2656 2654 }
2657 2655 return (FC_HWERR);
2658 2656 }
2659 2657
2660 2658 /*
2661 2659 * Setup the timeout if we need to
2662 2660 */
2663 2661 if (do_timeout) {
2664 2662 mutex_enter(&devctx->lock);
2665 2663 if (dhp->dh_timeout_length > 0) {
2666 2664 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK3,
2667 2665 "devmap_do_ctxmgt:timeout set");
2668 2666 devctx->timeout = timeout(devmap_ctxto,
2669 2667 devctx, dhp->dh_timeout_length);
2670 2668 } else {
2671 2669 /*
2672 2670 * We don't want to wait so set oncpu to
2673 2671 * 0 and wake up anyone waiting.
2674 2672 */
2675 2673 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DO_CTXMGT_CK4,
2676 2674 "devmap_do_ctxmgt:timeout not set");
2677 2675 devctx->oncpu = 0;
2678 2676 cv_signal(&devctx->cv);
2679 2677 }
2680 2678 mutex_exit(&devctx->lock);
2681 2679 }
2682 2680
2683 2681 return (DDI_SUCCESS);
2684 2682 }
2685 2683
2686 2684 /*
2687 2685 * end of mapping
2688 2686 * poff fault_offset |
2689 2687 * base | | |
2690 2688 * | | | |
2691 2689 * V V V V
2692 2690 * +-----------+---------------+-------+---------+-------+
2693 2691 * ^ ^ ^ ^
2694 2692 * |<--- offset--->|<-len->| |
2695 2693 * |<--- dh_len(size of mapping) --->|
2696 2694 * |<-- pg -->|
2697 2695 * -->|rlen|<--
2698 2696 */
2699 2697 static ulong_t
2700 2698 devmap_roundup(devmap_handle_t *dhp, ulong_t offset, size_t len,
2701 2699 ulong_t *opfn, ulong_t *pagesize)
2702 2700 {
2703 2701 register int level;
2704 2702 ulong_t pg;
2705 2703 ulong_t poff;
2706 2704 ulong_t base;
2707 2705 caddr_t uvaddr;
2708 2706 long rlen;
2709 2707
2710 2708 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP,
2711 2709 "devmap_roundup:start dhp=%p off=%lx len=%lx",
2712 2710 (void *)dhp, offset, len);
2713 2711 DEBUGF(2, (CE_CONT, "devmap_roundup: dhp %p off %lx len %lx\n",
2714 2712 (void *)dhp, offset, len));
2715 2713
2716 2714 /*
2717 2715 * get the max. pagesize that is aligned within the range
2718 2716 * <dh_pfn, dh_pfn+offset>.
2719 2717 *
2720 2718 * The calculations below use physical address to ddetermine
2721 2719 * the page size to use. The same calculations can use the
2722 2720 * virtual address to determine the page size.
2723 2721 */
2724 2722 base = (ulong_t)ptob(dhp->dh_pfn);
2725 2723 for (level = dhp->dh_mmulevel; level >= 0; level--) {
2726 2724 pg = page_get_pagesize(level);
2727 2725 poff = ((base + offset) & ~(pg - 1));
2728 2726 uvaddr = dhp->dh_uvaddr + (poff - base);
2729 2727 if ((poff >= base) &&
2730 2728 ((poff + pg) <= (base + dhp->dh_len)) &&
2731 2729 VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg))
2732 2730 break;
2733 2731 }
2734 2732
2735 2733 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK1,
2736 2734 "devmap_roundup: base=%lx poff=%lx dhp=%p",
2737 2735 base, poff, dhp);
2738 2736 DEBUGF(2, (CE_CONT, "devmap_roundup: base %lx poff %lx pfn %lx\n",
2739 2737 base, poff, dhp->dh_pfn));
2740 2738
2741 2739 ASSERT(VA_PA_ALIGNED((uintptr_t)uvaddr, poff, pg));
2742 2740 ASSERT(level >= 0);
2743 2741
2744 2742 *pagesize = pg;
2745 2743 *opfn = dhp->dh_pfn + btop(poff - base);
2746 2744
2747 2745 rlen = len + offset - (poff - base + pg);
2748 2746
2749 2747 ASSERT(rlen < (long)len);
2750 2748
2751 2749 TRACE_5(TR_FAC_DEVMAP, TR_DEVMAP_ROUNDUP_CK2,
2752 2750 "devmap_roundup:ret dhp=%p level=%x rlen=%lx psiz=%p opfn=%p",
2753 2751 (void *)dhp, level, rlen, pagesize, opfn);
2754 2752 DEBUGF(1, (CE_CONT, "devmap_roundup: dhp %p "
2755 2753 "level %x rlen %lx psize %lx opfn %lx\n",
2756 2754 (void *)dhp, level, rlen, *pagesize, *opfn));
2757 2755
2758 2756 return ((ulong_t)((rlen > 0) ? rlen : 0));
2759 2757 }
2760 2758
2761 2759 /*
2762 2760 * find the dhp that contains addr.
2763 2761 */
2764 2762 static devmap_handle_t *
2765 2763 devmap_find_handle(devmap_handle_t *dhp_head, caddr_t addr)
2766 2764 {
2767 2765 devmap_handle_t *dhp;
2768 2766
2769 2767 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_FIND_HANDLE,
2770 2768 "devmap_find_handle:start");
2771 2769
2772 2770 dhp = dhp_head;
2773 2771 while (dhp) {
2774 2772 if (addr >= dhp->dh_uvaddr &&
2775 2773 addr < (dhp->dh_uvaddr + dhp->dh_len))
2776 2774 return (dhp);
2777 2775 dhp = dhp->dh_next;
2778 2776 }
2779 2777
2780 2778 return ((devmap_handle_t *)NULL);
2781 2779 }
2782 2780
2783 2781 /*
2784 2782 * devmap_unload:
2785 2783 * Marks a segdev segment or pages if offset->offset+len
2786 2784 * is not the entire segment as intercept and unloads the
2787 2785 * pages in the range offset -> offset+len.
2788 2786 */
2789 2787 int
2790 2788 devmap_unload(devmap_cookie_t dhc, offset_t offset, size_t len)
2791 2789 {
2792 2790 register devmap_handle_t *dhp = (devmap_handle_t *)dhc;
2793 2791 caddr_t addr;
2794 2792 ulong_t size;
2795 2793 ssize_t soff;
2796 2794
2797 2795 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_UNLOAD,
2798 2796 "devmap_unload:start dhp=%p offset=%llx len=%lx",
2799 2797 (void *)dhp, offset, len);
2800 2798 DEBUGF(7, (CE_CONT, "devmap_unload: dhp %p offset %llx len %lx\n",
2801 2799 (void *)dhp, offset, len));
2802 2800
2803 2801 soff = (ssize_t)(offset - dhp->dh_uoff);
2804 2802 soff = round_down_p2(soff, PAGESIZE);
2805 2803 if (soff < 0 || soff >= dhp->dh_len)
2806 2804 return (FC_MAKE_ERR(EINVAL));
2807 2805
2808 2806 /*
2809 2807 * Address and size must be page aligned. Len is set to the
2810 2808 * number of bytes in the number of pages that are required to
2811 2809 * support len. Offset is set to the byte offset of the first byte
2812 2810 * of the page that contains offset.
2813 2811 */
2814 2812 len = round_up_p2(len, PAGESIZE);
2815 2813
2816 2814 /*
2817 2815 * If len is == 0, then calculate the size by getting
2818 2816 * the number of bytes from offset to the end of the segment.
2819 2817 */
2820 2818 if (len == 0)
2821 2819 size = dhp->dh_len - soff;
2822 2820 else {
2823 2821 size = len;
2824 2822 if ((soff + size) > dhp->dh_len)
2825 2823 return (FC_MAKE_ERR(EINVAL));
2826 2824 }
2827 2825
2828 2826 /*
2829 2827 * The address is offset bytes from the base address of
2830 2828 * the dhp.
2831 2829 */
2832 2830 addr = (caddr_t)(soff + dhp->dh_uvaddr);
2833 2831
2834 2832 /*
2835 2833 * If large page size was used in hat_devload(),
2836 2834 * the same page size must be used in hat_unload().
2837 2835 */
2838 2836 if (dhp->dh_flags & DEVMAP_FLAG_LARGE) {
2839 2837 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
2840 2838 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
2841 2839 } else {
2842 2840 hat_unload(dhp->dh_seg->s_as->a_hat, addr, size,
2843 2841 HAT_UNLOAD|HAT_UNLOAD_OTHER);
2844 2842 }
2845 2843
2846 2844 return (0);
2847 2845 }
2848 2846
2849 2847 /*
2850 2848 * calculates the optimal page size that will be used for hat_devload().
2851 2849 */
2852 2850 static void
2853 2851 devmap_get_large_pgsize(devmap_handle_t *dhp, size_t len, caddr_t addr,
2854 2852 size_t *llen, caddr_t *laddr)
2855 2853 {
2856 2854 ulong_t off;
2857 2855 ulong_t pfn;
2858 2856 ulong_t pgsize;
2859 2857 uint_t first = 1;
2860 2858
2861 2859 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GET_LARGE_PGSIZE,
2862 2860 "devmap_get_large_pgsize:start");
2863 2861
2864 2862 /*
2865 2863 * RFE - Code only supports large page mappings for devmem
2866 2864 * This code could be changed in future if we want to support
2867 2865 * large page mappings for kernel exported memory.
2868 2866 */
2869 2867 ASSERT(dhp_is_devmem(dhp));
2870 2868 ASSERT(!(dhp->dh_flags & DEVMAP_MAPPING_INVALID));
2871 2869
2872 2870 *llen = 0;
2873 2871 off = (ulong_t)(addr - dhp->dh_uvaddr);
2874 2872 while ((long)len > 0) {
2875 2873 /*
2876 2874 * get the optimal pfn to minimize address translations.
2877 2875 * devmap_roundup() returns residue bytes for next round
2878 2876 * calculations.
2879 2877 */
2880 2878 len = devmap_roundup(dhp, off, len, &pfn, &pgsize);
2881 2879
2882 2880 if (first) {
2883 2881 *laddr = dhp->dh_uvaddr + ptob(pfn - dhp->dh_pfn);
2884 2882 first = 0;
2885 2883 }
2886 2884
2887 2885 *llen += pgsize;
2888 2886 off = ptob(pfn - dhp->dh_pfn) + pgsize;
2889 2887 }
2890 2888 /* Large page mapping len/addr cover more range than original fault */
2891 2889 ASSERT(*llen >= len && *laddr <= addr);
2892 2890 ASSERT((*laddr + *llen) >= (addr + len));
2893 2891 }
2894 2892
2895 2893 /*
2896 2894 * Initialize the devmap_softlock structure.
2897 2895 */
2898 2896 static struct devmap_softlock *
2899 2897 devmap_softlock_init(dev_t dev, ulong_t id)
2900 2898 {
2901 2899 struct devmap_softlock *slock;
2902 2900 struct devmap_softlock *tmp;
2903 2901
2904 2902 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_INIT,
2905 2903 "devmap_softlock_init:start");
2906 2904
2907 2905 tmp = kmem_zalloc(sizeof (struct devmap_softlock), KM_SLEEP);
2908 2906 mutex_enter(&devmap_slock);
2909 2907
2910 2908 for (slock = devmap_slist; slock != NULL; slock = slock->next)
2911 2909 if ((slock->dev == dev) && (slock->id == id))
2912 2910 break;
2913 2911
2914 2912 if (slock == NULL) {
2915 2913 slock = tmp;
2916 2914 slock->dev = dev;
2917 2915 slock->id = id;
2918 2916 mutex_init(&slock->lock, NULL, MUTEX_DEFAULT, NULL);
2919 2917 cv_init(&slock->cv, NULL, CV_DEFAULT, NULL);
2920 2918 slock->next = devmap_slist;
2921 2919 devmap_slist = slock;
2922 2920 } else
2923 2921 kmem_free(tmp, sizeof (struct devmap_softlock));
2924 2922
2925 2923 mutex_enter(&slock->lock);
2926 2924 slock->refcnt++;
2927 2925 mutex_exit(&slock->lock);
2928 2926 mutex_exit(&devmap_slock);
2929 2927
2930 2928 return (slock);
2931 2929 }
2932 2930
2933 2931 /*
2934 2932 * Wake up processes that sleep on softlocked.
2935 2933 * Free dh_softlock if refcnt is 0.
2936 2934 */
2937 2935 static void
2938 2936 devmap_softlock_rele(devmap_handle_t *dhp)
2939 2937 {
2940 2938 struct devmap_softlock *slock = dhp->dh_softlock;
2941 2939 struct devmap_softlock *tmp;
2942 2940 struct devmap_softlock *parent;
2943 2941
2944 2942 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SOFTLOCK_RELE,
2945 2943 "devmap_softlock_rele:start");
2946 2944
2947 2945 mutex_enter(&devmap_slock);
2948 2946 mutex_enter(&slock->lock);
2949 2947
2950 2948 ASSERT(slock->refcnt > 0);
2951 2949
2952 2950 slock->refcnt--;
2953 2951
2954 2952 /*
2955 2953 * If no one is using the device, free up the slock data.
2956 2954 */
2957 2955 if (slock->refcnt == 0) {
2958 2956 slock->softlocked = 0;
2959 2957 cv_signal(&slock->cv);
2960 2958
2961 2959 if (devmap_slist == slock)
2962 2960 devmap_slist = slock->next;
2963 2961 else {
2964 2962 parent = devmap_slist;
2965 2963 for (tmp = devmap_slist->next; tmp != NULL;
2966 2964 tmp = tmp->next) {
2967 2965 if (tmp == slock) {
2968 2966 parent->next = tmp->next;
2969 2967 break;
2970 2968 }
2971 2969 parent = tmp;
2972 2970 }
2973 2971 }
2974 2972 mutex_exit(&slock->lock);
2975 2973 mutex_destroy(&slock->lock);
2976 2974 cv_destroy(&slock->cv);
2977 2975 kmem_free(slock, sizeof (struct devmap_softlock));
2978 2976 } else
2979 2977 mutex_exit(&slock->lock);
2980 2978
2981 2979 mutex_exit(&devmap_slock);
2982 2980 }
2983 2981
2984 2982 /*
2985 2983 * Wake up processes that sleep on dh_ctx->locked.
2986 2984 * Free dh_ctx if refcnt is 0.
2987 2985 */
2988 2986 static void
2989 2987 devmap_ctx_rele(devmap_handle_t *dhp)
2990 2988 {
2991 2989 struct devmap_ctx *devctx = dhp->dh_ctx;
2992 2990 struct devmap_ctx *tmp;
2993 2991 struct devmap_ctx *parent;
2994 2992 timeout_id_t tid;
2995 2993
2996 2994 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE,
2997 2995 "devmap_ctx_rele:start");
2998 2996
2999 2997 mutex_enter(&devmapctx_lock);
3000 2998 mutex_enter(&devctx->lock);
3001 2999
3002 3000 ASSERT(devctx->refcnt > 0);
3003 3001
3004 3002 devctx->refcnt--;
3005 3003
3006 3004 /*
3007 3005 * If no one is using the device, free up the devctx data.
3008 3006 */
3009 3007 if (devctx->refcnt == 0) {
3010 3008 /*
3011 3009 * Untimeout any threads using this mapping as they are about
3012 3010 * to go away.
3013 3011 */
3014 3012 if (devctx->timeout != 0) {
3015 3013 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_CTX_RELE_CK1,
3016 3014 "devmap_ctx_rele:untimeout ctx->timeout");
3017 3015
3018 3016 tid = devctx->timeout;
3019 3017 mutex_exit(&devctx->lock);
3020 3018 (void) untimeout(tid);
3021 3019 mutex_enter(&devctx->lock);
3022 3020 }
3023 3021
3024 3022 devctx->oncpu = 0;
3025 3023 cv_signal(&devctx->cv);
3026 3024
3027 3025 if (devmapctx_list == devctx)
3028 3026 devmapctx_list = devctx->next;
3029 3027 else {
3030 3028 parent = devmapctx_list;
3031 3029 for (tmp = devmapctx_list->next; tmp != NULL;
3032 3030 tmp = tmp->next) {
3033 3031 if (tmp == devctx) {
3034 3032 parent->next = tmp->next;
3035 3033 break;
3036 3034 }
3037 3035 parent = tmp;
3038 3036 }
3039 3037 }
3040 3038 mutex_exit(&devctx->lock);
3041 3039 mutex_destroy(&devctx->lock);
3042 3040 cv_destroy(&devctx->cv);
3043 3041 kmem_free(devctx, sizeof (struct devmap_ctx));
3044 3042 } else
3045 3043 mutex_exit(&devctx->lock);
3046 3044
3047 3045 mutex_exit(&devmapctx_lock);
3048 3046 }
3049 3047
3050 3048 /*
3051 3049 * devmap_load:
3052 3050 * Marks a segdev segment or pages if offset->offset+len
3053 3051 * is not the entire segment as nointercept and faults in
3054 3052 * the pages in the range offset -> offset+len.
3055 3053 */
3056 3054 int
3057 3055 devmap_load(devmap_cookie_t dhc, offset_t offset, size_t len, uint_t type,
3058 3056 uint_t rw)
3059 3057 {
3060 3058 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3061 3059 struct as *asp = dhp->dh_seg->s_as;
3062 3060 caddr_t addr;
3063 3061 ulong_t size;
3064 3062 ssize_t soff; /* offset from the beginning of the segment */
3065 3063 int rc;
3066 3064
3067 3065 TRACE_3(TR_FAC_DEVMAP, TR_DEVMAP_LOAD,
3068 3066 "devmap_load:start dhp=%p offset=%llx len=%lx",
3069 3067 (void *)dhp, offset, len);
3070 3068
3071 3069 DEBUGF(7, (CE_CONT, "devmap_load: dhp %p offset %llx len %lx\n",
3072 3070 (void *)dhp, offset, len));
3073 3071
3074 3072 /*
3075 3073 * Hat layer only supports devload to process' context for which
3076 3074 * the as lock is held. Verify here and return error if drivers
3077 3075 * inadvertently call devmap_load on a wrong devmap handle.
3078 3076 */
3079 3077 if ((asp != &kas) && !AS_LOCK_HELD(asp, &asp->a_lock))
3080 3078 return (FC_MAKE_ERR(EINVAL));
3081 3079
3082 3080 soff = (ssize_t)(offset - dhp->dh_uoff);
3083 3081 soff = round_down_p2(soff, PAGESIZE);
3084 3082 if (soff < 0 || soff >= dhp->dh_len)
3085 3083 return (FC_MAKE_ERR(EINVAL));
3086 3084
3087 3085 /*
3088 3086 * Address and size must be page aligned. Len is set to the
3089 3087 * number of bytes in the number of pages that are required to
3090 3088 * support len. Offset is set to the byte offset of the first byte
3091 3089 * of the page that contains offset.
3092 3090 */
3093 3091 len = round_up_p2(len, PAGESIZE);
3094 3092
3095 3093 /*
3096 3094 * If len == 0, then calculate the size by getting
3097 3095 * the number of bytes from offset to the end of the segment.
3098 3096 */
3099 3097 if (len == 0)
3100 3098 size = dhp->dh_len - soff;
3101 3099 else {
3102 3100 size = len;
3103 3101 if ((soff + size) > dhp->dh_len)
3104 3102 return (FC_MAKE_ERR(EINVAL));
3105 3103 }
3106 3104
3107 3105 /*
3108 3106 * The address is offset bytes from the base address of
3109 3107 * the segment.
3110 3108 */
3111 3109 addr = (caddr_t)(soff + dhp->dh_uvaddr);
3112 3110
3113 3111 HOLD_DHP_LOCK(dhp);
3114 3112 rc = segdev_faultpages(asp->a_hat,
3115 3113 dhp->dh_seg, addr, size, type, rw, dhp);
3116 3114 RELE_DHP_LOCK(dhp);
3117 3115 return (rc);
3118 3116 }
3119 3117
3120 3118 int
3121 3119 devmap_setup(dev_t dev, offset_t off, struct as *as, caddr_t *addrp,
3122 3120 size_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3123 3121 {
3124 3122 register devmap_handle_t *dhp;
3125 3123 int (*devmap)(dev_t, devmap_cookie_t, offset_t, size_t,
3126 3124 size_t *, uint_t);
3127 3125 int (*mmap)(dev_t, off_t, int);
3128 3126 struct devmap_callback_ctl *callbackops;
3129 3127 devmap_handle_t *dhp_head = NULL;
3130 3128 devmap_handle_t *dhp_prev = NULL;
3131 3129 devmap_handle_t *dhp_curr;
3132 3130 caddr_t addr;
3133 3131 int map_flag;
3134 3132 int ret;
3135 3133 ulong_t total_len;
3136 3134 size_t map_len;
3137 3135 size_t resid_len = len;
3138 3136 offset_t map_off = off;
3139 3137 struct devmap_softlock *slock = NULL;
3140 3138
3141 3139 #ifdef lint
3142 3140 cred = cred;
3143 3141 #endif
3144 3142
3145 3143 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SETUP,
3146 3144 "devmap_setup:start off=%llx len=%lx", off, len);
3147 3145 DEBUGF(3, (CE_CONT, "devmap_setup: off %llx len %lx\n",
3148 3146 off, len));
3149 3147
3150 3148 devmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_devmap;
3151 3149 mmap = devopsp[getmajor(dev)]->devo_cb_ops->cb_mmap;
3152 3150
3153 3151 /*
3154 3152 * driver must provide devmap(9E) entry point in cb_ops to use the
3155 3153 * devmap framework.
3156 3154 */
3157 3155 if (devmap == NULL || devmap == nulldev || devmap == nodev)
3158 3156 return (EINVAL);
3159 3157
3160 3158 /*
3161 3159 * To protect from an inadvertent entry because the devmap entry point
3162 3160 * is not NULL, return error if D_DEVMAP bit is not set in cb_flag and
3163 3161 * mmap is NULL.
3164 3162 */
3165 3163 map_flag = devopsp[getmajor(dev)]->devo_cb_ops->cb_flag;
3166 3164 if ((map_flag & D_DEVMAP) == 0 && (mmap == NULL || mmap == nulldev))
3167 3165 return (EINVAL);
3168 3166
3169 3167 /*
3170 3168 * devmap allows mmap(2) to map multiple registers.
3171 3169 * one devmap_handle is created for each register mapped.
3172 3170 */
3173 3171 for (total_len = 0; total_len < len; total_len += map_len) {
3174 3172 dhp = kmem_zalloc(sizeof (devmap_handle_t), KM_SLEEP);
3175 3173
3176 3174 if (dhp_prev != NULL)
3177 3175 dhp_prev->dh_next = dhp;
3178 3176 else
3179 3177 dhp_head = dhp;
3180 3178 dhp_prev = dhp;
3181 3179
3182 3180 dhp->dh_prot = prot;
3183 3181 dhp->dh_orig_maxprot = dhp->dh_maxprot = maxprot;
3184 3182 dhp->dh_dev = dev;
3185 3183 dhp->dh_timeout_length = CTX_TIMEOUT_VALUE;
3186 3184 dhp->dh_uoff = map_off;
3187 3185
3188 3186 /*
3189 3187 * Get mapping specific info from
3190 3188 * the driver, such as rnumber, roff, len, callbackops,
3191 3189 * accattrp and, if the mapping is for kernel memory,
3192 3190 * ddi_umem_cookie.
3193 3191 */
3194 3192 if ((ret = cdev_devmap(dev, dhp, map_off,
3195 3193 resid_len, &map_len, get_udatamodel())) != 0) {
3196 3194 free_devmap_handle(dhp_head);
3197 3195 return (ENXIO);
3198 3196 }
3199 3197
3200 3198 if (map_len & PAGEOFFSET) {
3201 3199 free_devmap_handle(dhp_head);
3202 3200 return (EINVAL);
3203 3201 }
3204 3202
3205 3203 callbackops = &dhp->dh_callbackops;
3206 3204
3207 3205 if ((callbackops->devmap_access == NULL) ||
3208 3206 (callbackops->devmap_access == nulldev) ||
3209 3207 (callbackops->devmap_access == nodev)) {
3210 3208 /*
3211 3209 * Normally devmap does not support MAP_PRIVATE unless
3212 3210 * the drivers provide a valid devmap_access routine.
3213 3211 */
3214 3212 if ((flags & MAP_PRIVATE) != 0) {
3215 3213 free_devmap_handle(dhp_head);
3216 3214 return (EINVAL);
3217 3215 }
3218 3216 } else {
3219 3217 /*
3220 3218 * Initialize dhp_softlock and dh_ctx if the drivers
3221 3219 * provide devmap_access.
3222 3220 */
3223 3221 dhp->dh_softlock = devmap_softlock_init(dev,
3224 3222 (ulong_t)callbackops->devmap_access);
3225 3223 dhp->dh_ctx = devmap_ctxinit(dev,
3226 3224 (ulong_t)callbackops->devmap_access);
3227 3225
3228 3226 /*
3229 3227 * segdev_fault can only work when all
3230 3228 * dh_softlock in a multi-dhp mapping
3231 3229 * are same. see comments in segdev_fault
3232 3230 * This code keeps track of the first
3233 3231 * dh_softlock allocated in slock and
3234 3232 * compares all later allocations and if
3235 3233 * not similar, returns an error.
3236 3234 */
3237 3235 if (slock == NULL)
3238 3236 slock = dhp->dh_softlock;
3239 3237 if (slock != dhp->dh_softlock) {
3240 3238 free_devmap_handle(dhp_head);
3241 3239 return (ENOTSUP);
3242 3240 }
3243 3241 }
3244 3242
3245 3243 map_off += map_len;
3246 3244 resid_len -= map_len;
3247 3245 }
3248 3246
3249 3247 /*
3250 3248 * get the user virtual address and establish the mapping between
3251 3249 * uvaddr and device physical address.
3252 3250 */
3253 3251 if ((ret = devmap_device(dhp_head, as, addrp, off, len, flags))
3254 3252 != 0) {
3255 3253 /*
3256 3254 * free devmap handles if error during the mapping.
3257 3255 */
3258 3256 free_devmap_handle(dhp_head);
3259 3257
3260 3258 return (ret);
3261 3259 }
3262 3260
3263 3261 /*
3264 3262 * call the driver's devmap_map callback to do more after the mapping,
3265 3263 * such as to allocate driver private data for context management.
3266 3264 */
3267 3265 dhp = dhp_head;
3268 3266 map_off = off;
3269 3267 addr = *addrp;
3270 3268 while (dhp != NULL) {
3271 3269 callbackops = &dhp->dh_callbackops;
3272 3270 dhp->dh_uvaddr = addr;
3273 3271 dhp_curr = dhp;
3274 3272 if (callbackops->devmap_map != NULL) {
3275 3273 ret = (*callbackops->devmap_map)((devmap_cookie_t)dhp,
3276 3274 dev, flags, map_off,
3277 3275 dhp->dh_len, &dhp->dh_pvtp);
3278 3276 if (ret != 0) {
3279 3277 struct segdev_data *sdp;
3280 3278
3281 3279 /*
3282 3280 * call driver's devmap_unmap entry point
3283 3281 * to free driver resources.
3284 3282 */
3285 3283 dhp = dhp_head;
3286 3284 map_off = off;
3287 3285 while (dhp != dhp_curr) {
3288 3286 callbackops = &dhp->dh_callbackops;
3289 3287 if (callbackops->devmap_unmap != NULL) {
3290 3288 (*callbackops->devmap_unmap)(
3291 3289 dhp, dhp->dh_pvtp,
3292 3290 map_off, dhp->dh_len,
3293 3291 NULL, NULL, NULL, NULL);
3294 3292 }
3295 3293 map_off += dhp->dh_len;
3296 3294 dhp = dhp->dh_next;
3297 3295 }
3298 3296 sdp = dhp_head->dh_seg->s_data;
3299 3297 sdp->devmap_data = NULL;
3300 3298 free_devmap_handle(dhp_head);
3301 3299 return (ENXIO);
3302 3300 }
3303 3301 }
3304 3302 map_off += dhp->dh_len;
3305 3303 addr += dhp->dh_len;
3306 3304 dhp = dhp->dh_next;
3307 3305 }
3308 3306
3309 3307 return (0);
3310 3308 }
3311 3309
3312 3310 int
3313 3311 ddi_devmap_segmap(dev_t dev, off_t off, ddi_as_handle_t as, caddr_t *addrp,
3314 3312 off_t len, uint_t prot, uint_t maxprot, uint_t flags, struct cred *cred)
3315 3313 {
3316 3314 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_SEGMAP,
3317 3315 "devmap_segmap:start");
3318 3316 return (devmap_setup(dev, (offset_t)off, (struct as *)as, addrp,
3319 3317 (size_t)len, prot, maxprot, flags, cred));
3320 3318 }
3321 3319
3322 3320 /*
3323 3321 * Called from devmap_devmem_setup/remap to see if can use large pages for
3324 3322 * this device mapping.
3325 3323 * Also calculate the max. page size for this mapping.
3326 3324 * this page size will be used in fault routine for
3327 3325 * optimal page size calculations.
3328 3326 */
3329 3327 static void
3330 3328 devmap_devmem_large_page_setup(devmap_handle_t *dhp)
3331 3329 {
3332 3330 ASSERT(dhp_is_devmem(dhp));
3333 3331 dhp->dh_mmulevel = 0;
3334 3332
3335 3333 /*
3336 3334 * use large page size only if:
3337 3335 * 1. device memory.
3338 3336 * 2. mmu supports multiple page sizes,
3339 3337 * 3. Driver did not disallow it
3340 3338 * 4. dhp length is at least as big as the large pagesize
3341 3339 * 5. the uvaddr and pfn are large pagesize aligned
3342 3340 */
3343 3341 if (page_num_pagesizes() > 1 &&
3344 3342 !(dhp->dh_flags & (DEVMAP_USE_PAGESIZE | DEVMAP_MAPPING_INVALID))) {
3345 3343 ulong_t base;
3346 3344 int level;
3347 3345
3348 3346 base = (ulong_t)ptob(dhp->dh_pfn);
3349 3347 for (level = 1; level < page_num_pagesizes(); level++) {
3350 3348 size_t pgsize = page_get_pagesize(level);
3351 3349 if ((dhp->dh_len < pgsize) ||
3352 3350 (!VA_PA_PGSIZE_ALIGNED((uintptr_t)dhp->dh_uvaddr,
3353 3351 base, pgsize))) {
3354 3352 break;
3355 3353 }
3356 3354 }
3357 3355 dhp->dh_mmulevel = level - 1;
3358 3356 }
3359 3357 if (dhp->dh_mmulevel > 0) {
3360 3358 dhp->dh_flags |= DEVMAP_FLAG_LARGE;
3361 3359 } else {
3362 3360 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3363 3361 }
3364 3362 }
3365 3363
3366 3364 /*
3367 3365 * Called by driver devmap routine to pass device specific info to
3368 3366 * the framework. used for device memory mapping only.
3369 3367 */
3370 3368 int
3371 3369 devmap_devmem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3372 3370 struct devmap_callback_ctl *callbackops, uint_t rnumber, offset_t roff,
3373 3371 size_t len, uint_t maxprot, uint_t flags, ddi_device_acc_attr_t *accattrp)
3374 3372 {
3375 3373 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3376 3374 ddi_acc_handle_t handle;
3377 3375 ddi_map_req_t mr;
3378 3376 ddi_acc_hdl_t *hp;
3379 3377 int err;
3380 3378
3381 3379 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_SETUP,
3382 3380 "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3383 3381 (void *)dhp, roff, rnumber, (uint_t)len);
3384 3382 DEBUGF(2, (CE_CONT, "devmap_devmem_setup: dhp %p offset %llx "
3385 3383 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3386 3384
3387 3385 /*
3388 3386 * First to check if this function has been called for this dhp.
3389 3387 */
3390 3388 if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3391 3389 return (DDI_FAILURE);
3392 3390
3393 3391 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3394 3392 return (DDI_FAILURE);
3395 3393
3396 3394 if (flags & DEVMAP_MAPPING_INVALID) {
3397 3395 /*
3398 3396 * Don't go up the tree to get pfn if the driver specifies
3399 3397 * DEVMAP_MAPPING_INVALID in flags.
3400 3398 *
3401 3399 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3402 3400 * remap permission.
3403 3401 */
3404 3402 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3405 3403 return (DDI_FAILURE);
3406 3404 }
3407 3405 dhp->dh_pfn = PFN_INVALID;
3408 3406 } else {
3409 3407 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3410 3408 if (handle == NULL)
3411 3409 return (DDI_FAILURE);
3412 3410
3413 3411 hp = impl_acc_hdl_get(handle);
3414 3412 hp->ah_vers = VERS_ACCHDL;
3415 3413 hp->ah_dip = dip;
3416 3414 hp->ah_rnumber = rnumber;
3417 3415 hp->ah_offset = roff;
3418 3416 hp->ah_len = len;
3419 3417 if (accattrp != NULL)
3420 3418 hp->ah_acc = *accattrp;
3421 3419
3422 3420 mr.map_op = DDI_MO_MAP_LOCKED;
3423 3421 mr.map_type = DDI_MT_RNUMBER;
3424 3422 mr.map_obj.rnumber = rnumber;
3425 3423 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3426 3424 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3427 3425 mr.map_handlep = hp;
3428 3426 mr.map_vers = DDI_MAP_VERSION;
3429 3427
3430 3428 /*
3431 3429 * up the device tree to get pfn.
3432 3430 * The rootnex_map_regspec() routine in nexus drivers has been
3433 3431 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3434 3432 */
3435 3433 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&dhp->dh_pfn);
3436 3434 dhp->dh_hat_attr = hp->ah_hat_flags;
3437 3435 impl_acc_hdl_free(handle);
3438 3436
3439 3437 if (err)
3440 3438 return (DDI_FAILURE);
3441 3439 }
3442 3440 /* Should not be using devmem setup for memory pages */
3443 3441 ASSERT(!pf_is_memory(dhp->dh_pfn));
3444 3442
3445 3443 /* Only some of the flags bits are settable by the driver */
3446 3444 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3447 3445 dhp->dh_len = ptob(btopr(len));
3448 3446
3449 3447 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3450 3448 dhp->dh_roff = ptob(btop(roff));
3451 3449
3452 3450 /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3453 3451 devmap_devmem_large_page_setup(dhp);
3454 3452 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3455 3453 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3456 3454
3457 3455
3458 3456 if (callbackops != NULL) {
3459 3457 bcopy(callbackops, &dhp->dh_callbackops,
3460 3458 sizeof (struct devmap_callback_ctl));
3461 3459 }
3462 3460
3463 3461 /*
3464 3462 * Initialize dh_lock if we want to do remap.
3465 3463 */
3466 3464 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3467 3465 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3468 3466 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3469 3467 }
3470 3468
3471 3469 dhp->dh_flags |= DEVMAP_SETUP_DONE;
3472 3470
3473 3471 return (DDI_SUCCESS);
3474 3472 }
3475 3473
3476 3474 int
3477 3475 devmap_devmem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3478 3476 uint_t rnumber, offset_t roff, size_t len, uint_t maxprot,
3479 3477 uint_t flags, ddi_device_acc_attr_t *accattrp)
3480 3478 {
3481 3479 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3482 3480 ddi_acc_handle_t handle;
3483 3481 ddi_map_req_t mr;
3484 3482 ddi_acc_hdl_t *hp;
3485 3483 pfn_t pfn;
3486 3484 uint_t hat_flags;
3487 3485 int err;
3488 3486
3489 3487 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_DEVMEM_REMAP,
3490 3488 "devmap_devmem_setup:start dhp=%p offset=%llx rnum=%d len=%lx",
3491 3489 (void *)dhp, roff, rnumber, (uint_t)len);
3492 3490 DEBUGF(2, (CE_CONT, "devmap_devmem_remap: dhp %p offset %llx "
3493 3491 "rnum %d len %lx\n", (void *)dhp, roff, rnumber, len));
3494 3492
3495 3493 /*
3496 3494 * Return failure if setup has not been done or no remap permission
3497 3495 * has been granted during the setup.
3498 3496 */
3499 3497 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3500 3498 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3501 3499 return (DDI_FAILURE);
3502 3500
3503 3501 /* Only DEVMAP_MAPPING_INVALID flag supported for remap */
3504 3502 if ((flags != 0) && (flags != DEVMAP_MAPPING_INVALID))
3505 3503 return (DDI_FAILURE);
3506 3504
3507 3505 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3508 3506 return (DDI_FAILURE);
3509 3507
3510 3508 if (!(flags & DEVMAP_MAPPING_INVALID)) {
3511 3509 handle = impl_acc_hdl_alloc(KM_SLEEP, NULL);
3512 3510 if (handle == NULL)
3513 3511 return (DDI_FAILURE);
3514 3512 }
3515 3513
3516 3514 HOLD_DHP_LOCK(dhp);
3517 3515
3518 3516 /*
3519 3517 * Unload the old mapping, so next fault will setup the new mappings
3520 3518 * Do this while holding the dhp lock so other faults dont reestablish
3521 3519 * the mappings
3522 3520 */
3523 3521 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3524 3522 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3525 3523
3526 3524 if (flags & DEVMAP_MAPPING_INVALID) {
3527 3525 dhp->dh_flags |= DEVMAP_MAPPING_INVALID;
3528 3526 dhp->dh_pfn = PFN_INVALID;
3529 3527 } else {
3530 3528 /* clear any prior DEVMAP_MAPPING_INVALID flag */
3531 3529 dhp->dh_flags &= ~DEVMAP_MAPPING_INVALID;
3532 3530 hp = impl_acc_hdl_get(handle);
3533 3531 hp->ah_vers = VERS_ACCHDL;
3534 3532 hp->ah_dip = dip;
3535 3533 hp->ah_rnumber = rnumber;
3536 3534 hp->ah_offset = roff;
3537 3535 hp->ah_len = len;
3538 3536 if (accattrp != NULL)
3539 3537 hp->ah_acc = *accattrp;
3540 3538
3541 3539 mr.map_op = DDI_MO_MAP_LOCKED;
3542 3540 mr.map_type = DDI_MT_RNUMBER;
3543 3541 mr.map_obj.rnumber = rnumber;
3544 3542 mr.map_prot = maxprot & dhp->dh_orig_maxprot;
3545 3543 mr.map_flags = DDI_MF_DEVICE_MAPPING;
3546 3544 mr.map_handlep = hp;
3547 3545 mr.map_vers = DDI_MAP_VERSION;
3548 3546
3549 3547 /*
3550 3548 * up the device tree to get pfn.
3551 3549 * The rootnex_map_regspec() routine in nexus drivers has been
3552 3550 * modified to return pfn if map_flags is DDI_MF_DEVICE_MAPPING.
3553 3551 */
3554 3552 err = ddi_map(dip, &mr, roff, len, (caddr_t *)&pfn);
3555 3553 hat_flags = hp->ah_hat_flags;
3556 3554 impl_acc_hdl_free(handle);
3557 3555 if (err) {
3558 3556 RELE_DHP_LOCK(dhp);
3559 3557 return (DDI_FAILURE);
3560 3558 }
3561 3559 /*
3562 3560 * Store result of ddi_map first in local variables, as we do
3563 3561 * not want to overwrite the existing dhp with wrong data.
3564 3562 */
3565 3563 dhp->dh_pfn = pfn;
3566 3564 dhp->dh_hat_attr = hat_flags;
3567 3565 }
3568 3566
3569 3567 /* clear the large page size flag */
3570 3568 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3571 3569
3572 3570 dhp->dh_cookie = DEVMAP_DEVMEM_COOKIE;
3573 3571 dhp->dh_roff = ptob(btop(roff));
3574 3572
3575 3573 /* setup the dh_mmulevel and DEVMAP_FLAG_LARGE */
3576 3574 devmap_devmem_large_page_setup(dhp);
3577 3575 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3578 3576 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3579 3577
3580 3578 RELE_DHP_LOCK(dhp);
3581 3579 return (DDI_SUCCESS);
3582 3580 }
3583 3581
3584 3582 /*
3585 3583 * called by driver devmap routine to pass kernel virtual address mapping
3586 3584 * info to the framework. used only for kernel memory
3587 3585 * allocated from ddi_umem_alloc().
3588 3586 */
3589 3587 int
3590 3588 devmap_umem_setup(devmap_cookie_t dhc, dev_info_t *dip,
3591 3589 struct devmap_callback_ctl *callbackops, ddi_umem_cookie_t cookie,
3592 3590 offset_t off, size_t len, uint_t maxprot, uint_t flags,
3593 3591 ddi_device_acc_attr_t *accattrp)
3594 3592 {
3595 3593 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3596 3594 struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3597 3595
3598 3596 #ifdef lint
3599 3597 dip = dip;
3600 3598 #endif
3601 3599
3602 3600 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_SETUP,
3603 3601 "devmap_umem_setup:start dhp=%p offset=%llx cookie=%p len=%lx",
3604 3602 (void *)dhp, off, cookie, len);
3605 3603 DEBUGF(2, (CE_CONT, "devmap_umem_setup: dhp %p offset %llx "
3606 3604 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3607 3605
3608 3606 if (cookie == NULL)
3609 3607 return (DDI_FAILURE);
3610 3608
3611 3609 /* For UMEM_TRASH, this restriction is not needed */
3612 3610 if ((off + len) > cp->size)
3613 3611 return (DDI_FAILURE);
3614 3612
3615 3613 /* check if the cache attributes are supported */
3616 3614 if (i_ddi_check_cache_attr(flags) == B_FALSE)
3617 3615 return (DDI_FAILURE);
3618 3616
3619 3617 /*
3620 3618 * First to check if this function has been called for this dhp.
3621 3619 */
3622 3620 if (dhp->dh_flags & DEVMAP_SETUP_DONE)
3623 3621 return (DDI_FAILURE);
3624 3622
3625 3623 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3626 3624 return (DDI_FAILURE);
3627 3625
3628 3626 if (flags & DEVMAP_MAPPING_INVALID) {
3629 3627 /*
3630 3628 * If DEVMAP_MAPPING_INVALID is specified, we have to grant
3631 3629 * remap permission.
3632 3630 */
3633 3631 if (!(flags & DEVMAP_ALLOW_REMAP)) {
3634 3632 return (DDI_FAILURE);
3635 3633 }
3636 3634 } else {
3637 3635 dhp->dh_cookie = cookie;
3638 3636 dhp->dh_roff = ptob(btop(off));
3639 3637 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3640 3638 /* set HAT cache attributes */
3641 3639 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3642 3640 /* set HAT endianess attributes */
3643 3641 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3644 3642 }
3645 3643
3646 3644 /*
3647 3645 * The default is _not_ to pass HAT_LOAD_NOCONSIST to hat_devload();
3648 3646 * we pass HAT_LOAD_NOCONSIST _only_ in cases where hat tries to
3649 3647 * create consistent mappings but our intention was to create
3650 3648 * non-consistent mappings.
3651 3649 *
3652 3650 * DEVMEM: hat figures it out it's DEVMEM and creates non-consistent
3653 3651 * mappings.
3654 3652 *
3655 3653 * kernel exported memory: hat figures it out it's memory and always
3656 3654 * creates consistent mappings.
3657 3655 *
3658 3656 * /dev/mem: non-consistent mappings. See comments in common/io/mem.c
3659 3657 *
3660 3658 * /dev/kmem: consistent mappings are created unless they are
3661 3659 * MAP_FIXED. We _explicitly_ tell hat to create non-consistent
3662 3660 * mappings by passing HAT_LOAD_NOCONSIST in case of MAP_FIXED
3663 3661 * mappings of /dev/kmem. See common/io/mem.c
3664 3662 */
3665 3663
3666 3664 /* Only some of the flags bits are settable by the driver */
3667 3665 dhp->dh_flags |= (flags & DEVMAP_SETUP_FLAGS);
3668 3666
3669 3667 dhp->dh_len = ptob(btopr(len));
3670 3668 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3671 3669 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3672 3670
3673 3671 if (callbackops != NULL) {
3674 3672 bcopy(callbackops, &dhp->dh_callbackops,
3675 3673 sizeof (struct devmap_callback_ctl));
3676 3674 }
3677 3675 /*
3678 3676 * Initialize dh_lock if we want to do remap.
3679 3677 */
3680 3678 if (dhp->dh_flags & DEVMAP_ALLOW_REMAP) {
3681 3679 mutex_init(&dhp->dh_lock, NULL, MUTEX_DEFAULT, NULL);
3682 3680 dhp->dh_flags |= DEVMAP_LOCK_INITED;
3683 3681 }
3684 3682
3685 3683 dhp->dh_flags |= DEVMAP_SETUP_DONE;
3686 3684
3687 3685 return (DDI_SUCCESS);
3688 3686 }
3689 3687
3690 3688 int
3691 3689 devmap_umem_remap(devmap_cookie_t dhc, dev_info_t *dip,
3692 3690 ddi_umem_cookie_t cookie, offset_t off, size_t len, uint_t maxprot,
3693 3691 uint_t flags, ddi_device_acc_attr_t *accattrp)
3694 3692 {
3695 3693 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3696 3694 struct ddi_umem_cookie *cp = (struct ddi_umem_cookie *)cookie;
3697 3695
3698 3696 TRACE_4(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_REMAP,
3699 3697 "devmap_umem_remap:start dhp=%p offset=%llx cookie=%p len=%lx",
3700 3698 (void *)dhp, off, cookie, len);
3701 3699 DEBUGF(2, (CE_CONT, "devmap_umem_remap: dhp %p offset %llx "
3702 3700 "cookie %p len %lx\n", (void *)dhp, off, (void *)cookie, len));
3703 3701
3704 3702 #ifdef lint
3705 3703 dip = dip;
3706 3704 accattrp = accattrp;
3707 3705 #endif
3708 3706 /*
3709 3707 * Reture failure if setup has not been done or no remap permission
3710 3708 * has been granted during the setup.
3711 3709 */
3712 3710 if ((dhp->dh_flags & DEVMAP_SETUP_DONE) == 0 ||
3713 3711 (dhp->dh_flags & DEVMAP_ALLOW_REMAP) == 0)
3714 3712 return (DDI_FAILURE);
3715 3713
3716 3714 /* No flags supported for remap yet */
3717 3715 if (flags != 0)
3718 3716 return (DDI_FAILURE);
3719 3717
3720 3718 /* check if the cache attributes are supported */
3721 3719 if (i_ddi_check_cache_attr(flags) == B_FALSE)
3722 3720 return (DDI_FAILURE);
3723 3721
3724 3722 if ((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) != dhp->dh_prot)
3725 3723 return (DDI_FAILURE);
3726 3724
3727 3725 /* For UMEM_TRASH, this restriction is not needed */
3728 3726 if ((off + len) > cp->size)
3729 3727 return (DDI_FAILURE);
3730 3728
3731 3729 HOLD_DHP_LOCK(dhp);
3732 3730 /*
3733 3731 * Unload the old mapping, so next fault will setup the new mappings
3734 3732 * Do this while holding the dhp lock so other faults dont reestablish
3735 3733 * the mappings
3736 3734 */
3737 3735 hat_unload(dhp->dh_seg->s_as->a_hat, dhp->dh_uvaddr,
3738 3736 dhp->dh_len, HAT_UNLOAD|HAT_UNLOAD_OTHER);
3739 3737
3740 3738 dhp->dh_cookie = cookie;
3741 3739 dhp->dh_roff = ptob(btop(off));
3742 3740 dhp->dh_cvaddr = cp->cvaddr + dhp->dh_roff;
3743 3741 /* set HAT cache attributes */
3744 3742 i_ddi_cacheattr_to_hatacc(flags, &dhp->dh_hat_attr);
3745 3743 /* set HAT endianess attributes */
3746 3744 i_ddi_devacc_to_hatacc(accattrp, &dhp->dh_hat_attr);
3747 3745
3748 3746 /* clear the large page size flag */
3749 3747 dhp->dh_flags &= ~DEVMAP_FLAG_LARGE;
3750 3748
3751 3749 dhp->dh_maxprot = maxprot & dhp->dh_orig_maxprot;
3752 3750 ASSERT((dhp->dh_prot & dhp->dh_orig_maxprot & maxprot) == dhp->dh_prot);
3753 3751 RELE_DHP_LOCK(dhp);
3754 3752 return (DDI_SUCCESS);
3755 3753 }
3756 3754
3757 3755 /*
3758 3756 * to set timeout value for the driver's context management callback, e.g.
3759 3757 * devmap_access().
3760 3758 */
3761 3759 void
3762 3760 devmap_set_ctx_timeout(devmap_cookie_t dhc, clock_t ticks)
3763 3761 {
3764 3762 devmap_handle_t *dhp = (devmap_handle_t *)dhc;
3765 3763
3766 3764 TRACE_2(TR_FAC_DEVMAP, TR_DEVMAP_SET_CTX_TIMEOUT,
3767 3765 "devmap_set_ctx_timeout:start dhp=%p ticks=%x",
3768 3766 (void *)dhp, ticks);
3769 3767 dhp->dh_timeout_length = ticks;
3770 3768 }
3771 3769
3772 3770 int
3773 3771 devmap_default_access(devmap_cookie_t dhp, void *pvtp, offset_t off,
3774 3772 size_t len, uint_t type, uint_t rw)
3775 3773 {
3776 3774 #ifdef lint
3777 3775 pvtp = pvtp;
3778 3776 #endif
3779 3777
3780 3778 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_DEFAULT_ACCESS,
3781 3779 "devmap_default_access:start");
3782 3780 return (devmap_load(dhp, off, len, type, rw));
3783 3781 }
3784 3782
3785 3783 /*
3786 3784 * segkmem_alloc() wrapper to allocate memory which is both
3787 3785 * non-relocatable (for DR) and sharelocked, since the rest
3788 3786 * of this segment driver requires it.
3789 3787 */
3790 3788 static void *
3791 3789 devmap_alloc_pages(vmem_t *vmp, size_t size, int vmflag)
3792 3790 {
3793 3791 ASSERT(vmp != NULL);
3794 3792 ASSERT(kvseg.s_base != NULL);
3795 3793 vmflag |= (VM_NORELOC | SEGKMEM_SHARELOCKED);
3796 3794 return (segkmem_alloc(vmp, size, vmflag));
3797 3795 }
3798 3796
3799 3797 /*
3800 3798 * This is where things are a bit incestuous with seg_kmem: unlike
3801 3799 * seg_kp, seg_kmem does not keep its pages long-term sharelocked, so
3802 3800 * we need to do a bit of a dance around that to prevent duplication of
3803 3801 * code until we decide to bite the bullet and implement a new kernel
3804 3802 * segment for driver-allocated memory that is exported to user space.
3805 3803 */
3806 3804 static void
3807 3805 devmap_free_pages(vmem_t *vmp, void *inaddr, size_t size)
3808 3806 {
3809 3807 page_t *pp;
3810 3808 caddr_t addr = inaddr;
3811 3809 caddr_t eaddr;
3812 3810 pgcnt_t npages = btopr(size);
3813 3811
3814 3812 ASSERT(vmp != NULL);
3815 3813 ASSERT(kvseg.s_base != NULL);
3816 3814 ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
3817 3815
3818 3816 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
3819 3817
3820 3818 for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
3821 3819 /*
3822 3820 * Use page_find() instead of page_lookup() to find the page
3823 3821 * since we know that it is hashed and has a shared lock.
3824 3822 */
3825 3823 pp = page_find(&kvp, (u_offset_t)(uintptr_t)addr);
3826 3824
3827 3825 if (pp == NULL)
3828 3826 panic("devmap_free_pages: page not found");
3829 3827 if (!page_tryupgrade(pp)) {
3830 3828 page_unlock(pp);
3831 3829 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr,
3832 3830 SE_EXCL);
3833 3831 if (pp == NULL)
3834 3832 panic("devmap_free_pages: page already freed");
3835 3833 }
3836 3834 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
3837 3835 pp->p_lckcnt = 0;
3838 3836 page_destroy(pp, 0);
3839 3837 }
3840 3838 page_unresv(npages);
3841 3839
3842 3840 if (vmp != NULL)
3843 3841 vmem_free(vmp, inaddr, size);
3844 3842 }
3845 3843
3846 3844 /*
3847 3845 * devmap_umem_alloc_np() replaces kmem_zalloc() as the method for
3848 3846 * allocating non-pageable kmem in response to a ddi_umem_alloc()
3849 3847 * default request. For now we allocate our own pages and we keep
3850 3848 * them long-term sharelocked, since: A) the fault routines expect the
3851 3849 * memory to already be locked; B) pageable umem is already long-term
3852 3850 * locked; C) it's a lot of work to make it otherwise, particularly
3853 3851 * since the nexus layer expects the pages to never fault. An RFE is to
3854 3852 * not keep the pages long-term locked, but instead to be able to
3855 3853 * take faults on them and simply look them up in kvp in case we
3856 3854 * fault on them. Even then, we must take care not to let pageout
3857 3855 * steal them from us since the data must remain resident; if we
3858 3856 * do this we must come up with some way to pin the pages to prevent
3859 3857 * faults while a driver is doing DMA to/from them.
3860 3858 */
3861 3859 static void *
3862 3860 devmap_umem_alloc_np(size_t size, size_t flags)
3863 3861 {
3864 3862 void *buf;
3865 3863 int vmflags = (flags & DDI_UMEM_NOSLEEP)? VM_NOSLEEP : VM_SLEEP;
3866 3864
3867 3865 buf = vmem_alloc(umem_np_arena, size, vmflags);
3868 3866 if (buf != NULL)
3869 3867 bzero(buf, size);
3870 3868 return (buf);
3871 3869 }
3872 3870
3873 3871 static void
3874 3872 devmap_umem_free_np(void *addr, size_t size)
3875 3873 {
3876 3874 vmem_free(umem_np_arena, addr, size);
3877 3875 }
3878 3876
3879 3877 /*
3880 3878 * allocate page aligned kernel memory for exporting to user land.
3881 3879 * The devmap framework will use the cookie allocated by ddi_umem_alloc()
3882 3880 * to find a user virtual address that is in same color as the address
3883 3881 * allocated here.
3884 3882 */
3885 3883 void *
3886 3884 ddi_umem_alloc(size_t size, int flags, ddi_umem_cookie_t *cookie)
3887 3885 {
3888 3886 register size_t len = ptob(btopr(size));
3889 3887 void *buf = NULL;
3890 3888 struct ddi_umem_cookie *cp;
3891 3889 int iflags = 0;
3892 3890
3893 3891 *cookie = NULL;
3894 3892
3895 3893 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_ALLOC,
3896 3894 "devmap_umem_alloc:start");
3897 3895 if (len == 0)
3898 3896 return ((void *)NULL);
3899 3897
3900 3898 /*
3901 3899 * allocate cookie
3902 3900 */
3903 3901 if ((cp = kmem_zalloc(sizeof (struct ddi_umem_cookie),
3904 3902 flags & DDI_UMEM_NOSLEEP ? KM_NOSLEEP : KM_SLEEP)) == NULL) {
3905 3903 ASSERT(flags & DDI_UMEM_NOSLEEP);
3906 3904 return ((void *)NULL);
3907 3905 }
3908 3906
3909 3907 if (flags & DDI_UMEM_PAGEABLE) {
3910 3908 /* Only one of the flags is allowed */
3911 3909 ASSERT(!(flags & DDI_UMEM_TRASH));
3912 3910 /* initialize resource with 0 */
3913 3911 iflags = KPD_ZERO;
3914 3912
3915 3913 /*
3916 3914 * to allocate unlocked pageable memory, use segkp_get() to
3917 3915 * create a segkp segment. Since segkp can only service kas,
3918 3916 * other segment drivers such as segdev have to do
3919 3917 * as_fault(segkp, SOFTLOCK) in its fault routine,
3920 3918 */
3921 3919 if (flags & DDI_UMEM_NOSLEEP)
3922 3920 iflags |= KPD_NOWAIT;
3923 3921
3924 3922 if ((buf = segkp_get(segkp, len, iflags)) == NULL) {
3925 3923 kmem_free(cp, sizeof (struct ddi_umem_cookie));
3926 3924 return ((void *)NULL);
3927 3925 }
3928 3926 cp->type = KMEM_PAGEABLE;
3929 3927 mutex_init(&cp->lock, NULL, MUTEX_DEFAULT, NULL);
3930 3928 cp->locked = 0;
3931 3929 } else if (flags & DDI_UMEM_TRASH) {
3932 3930 /* Only one of the flags is allowed */
3933 3931 ASSERT(!(flags & DDI_UMEM_PAGEABLE));
3934 3932 cp->type = UMEM_TRASH;
3935 3933 buf = NULL;
3936 3934 } else {
3937 3935 if ((buf = devmap_umem_alloc_np(len, flags)) == NULL) {
3938 3936 kmem_free(cp, sizeof (struct ddi_umem_cookie));
3939 3937 return ((void *)NULL);
3940 3938 }
3941 3939
3942 3940 cp->type = KMEM_NON_PAGEABLE;
3943 3941 }
3944 3942
3945 3943 /*
3946 3944 * need to save size here. size will be used when
3947 3945 * we do kmem_free.
3948 3946 */
3949 3947 cp->size = len;
3950 3948 cp->cvaddr = (caddr_t)buf;
3951 3949
3952 3950 *cookie = (void *)cp;
3953 3951 return (buf);
3954 3952 }
3955 3953
3956 3954 void
3957 3955 ddi_umem_free(ddi_umem_cookie_t cookie)
3958 3956 {
3959 3957 struct ddi_umem_cookie *cp;
3960 3958
3961 3959 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_UMEM_FREE,
3962 3960 "devmap_umem_free:start");
3963 3961
3964 3962 /*
3965 3963 * if cookie is NULL, no effects on the system
3966 3964 */
3967 3965 if (cookie == NULL)
3968 3966 return;
3969 3967
3970 3968 cp = (struct ddi_umem_cookie *)cookie;
3971 3969
3972 3970 switch (cp->type) {
3973 3971 case KMEM_PAGEABLE :
3974 3972 ASSERT(cp->cvaddr != NULL && cp->size != 0);
3975 3973 /*
3976 3974 * Check if there are still any pending faults on the cookie
3977 3975 * while the driver is deleting it,
3978 3976 * XXX - could change to an ASSERT but wont catch errant drivers
3979 3977 */
3980 3978 mutex_enter(&cp->lock);
3981 3979 if (cp->locked) {
3982 3980 mutex_exit(&cp->lock);
3983 3981 panic("ddi_umem_free for cookie with pending faults %p",
3984 3982 (void *)cp);
3985 3983 return;
3986 3984 }
3987 3985
3988 3986 segkp_release(segkp, cp->cvaddr);
3989 3987
3990 3988 /*
3991 3989 * release mutex associated with this cookie.
3992 3990 */
3993 3991 mutex_destroy(&cp->lock);
3994 3992 break;
3995 3993 case KMEM_NON_PAGEABLE :
3996 3994 ASSERT(cp->cvaddr != NULL && cp->size != 0);
3997 3995 devmap_umem_free_np(cp->cvaddr, cp->size);
3998 3996 break;
3999 3997 case UMEM_TRASH :
4000 3998 break;
4001 3999 case UMEM_LOCKED :
4002 4000 /* Callers should use ddi_umem_unlock for this type */
4003 4001 ddi_umem_unlock(cookie);
4004 4002 /* Frees the cookie too */
4005 4003 return;
4006 4004 default:
4007 4005 /* panic so we can diagnose the underlying cause */
4008 4006 panic("ddi_umem_free: illegal cookie type 0x%x\n",
4009 4007 cp->type);
4010 4008 }
4011 4009
4012 4010 kmem_free(cookie, sizeof (struct ddi_umem_cookie));
4013 4011 }
4014 4012
4015 4013
4016 4014 static int
4017 4015 segdev_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
|
↓ open down ↓ |
3793 lines elided |
↑ open up ↑ |
4018 4016 {
4019 4017 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4020 4018
4021 4019 /*
4022 4020 * It looks as if it is always mapped shared
4023 4021 */
4024 4022 TRACE_0(TR_FAC_DEVMAP, TR_DEVMAP_GETMEMID,
4025 4023 "segdev_getmemid:start");
4026 4024 memidp->val[0] = (uintptr_t)VTOCVP(sdp->vp);
4027 4025 memidp->val[1] = sdp->offset + (uintptr_t)(addr - seg->s_base);
4028 - return (0);
4029 -}
4030 -
4031 -/*ARGSUSED*/
4032 -static int
4033 -segdev_capable(struct seg *seg, segcapability_t capability)
4034 -{
4035 4026 return (0);
4036 4027 }
4037 4028
4038 4029 /*
4039 4030 * ddi_umem_alloc() non-pageable quantum cache max size.
4040 4031 * This is just a SWAG.
4041 4032 */
4042 4033 #define DEVMAP_UMEM_QUANTUM (8*PAGESIZE)
4043 4034
4044 4035 /*
4045 4036 * Initialize seg_dev from boot. This routine sets up the trash page
4046 4037 * and creates the umem_np_arena used to back non-pageable memory
4047 4038 * requests.
4048 4039 */
4049 4040 void
4050 4041 segdev_init(void)
4051 4042 {
4052 4043 struct seg kseg;
4053 4044
4054 4045 umem_np_arena = vmem_create("umem_np", NULL, 0, PAGESIZE,
4055 4046 devmap_alloc_pages, devmap_free_pages, heap_arena,
4056 4047 DEVMAP_UMEM_QUANTUM, VM_SLEEP);
4057 4048
4058 4049 kseg.s_as = &kas;
4059 4050 trashpp = page_create_va(&trashvp, 0, PAGESIZE,
4060 4051 PG_NORELOC | PG_EXCL | PG_WAIT, &kseg, NULL);
4061 4052 if (trashpp == NULL)
4062 4053 panic("segdev_init: failed to create trash page");
4063 4054 pagezero(trashpp, 0, PAGESIZE);
4064 4055 page_downgrade(trashpp);
4065 4056 }
4066 4057
4067 4058 /*
4068 4059 * Invoke platform-dependent support routines so that /proc can have
4069 4060 * the platform code deal with curious hardware.
4070 4061 */
4071 4062 int
4072 4063 segdev_copyfrom(struct seg *seg,
4073 4064 caddr_t uaddr, const void *devaddr, void *kaddr, size_t len)
4074 4065 {
4075 4066 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4076 4067 struct snode *sp = VTOS(VTOCVP(sdp->vp));
4077 4068
4078 4069 return (e_ddi_copyfromdev(sp->s_dip,
4079 4070 (off_t)(uaddr - seg->s_base), devaddr, kaddr, len));
4080 4071 }
4081 4072
4082 4073 int
4083 4074 segdev_copyto(struct seg *seg,
4084 4075 caddr_t uaddr, const void *kaddr, void *devaddr, size_t len)
4085 4076 {
4086 4077 struct segdev_data *sdp = (struct segdev_data *)seg->s_data;
4087 4078 struct snode *sp = VTOS(VTOCVP(sdp->vp));
4088 4079
4089 4080 return (e_ddi_copytodev(sp->s_dip,
4090 4081 (off_t)(uaddr - seg->s_base), kaddr, devaddr, len));
4091 4082 }
|
↓ open down ↓ |
47 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX