1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
26 /* All Rights Reserved */
27
28 /*
29 * Portions of this source code were derived from Berkeley 4.3 BSD
30 * under license from the Regents of the University of California.
31 */
32
33 /*
34 * segkp is a segment driver that administers the allocation and deallocation
35 * of pageable variable size chunks of kernel virtual address space. Each
36 * allocated resource is page-aligned.
37 *
38 * The user may specify whether the resource should be initialized to 0,
39 * include a redzone, or locked in memory.
40 */
41
42 #include <sys/types.h>
43 #include <sys/t_lock.h>
44 #include <sys/thread.h>
45 #include <sys/param.h>
46 #include <sys/errno.h>
47 #include <sys/sysmacros.h>
48 #include <sys/systm.h>
49 #include <sys/buf.h>
50 #include <sys/mman.h>
51 #include <sys/vnode.h>
52 #include <sys/cmn_err.h>
53 #include <sys/swap.h>
54 #include <sys/tuneable.h>
55 #include <sys/kmem.h>
56 #include <sys/vmem.h>
57 #include <sys/cred.h>
58 #include <sys/dumphdr.h>
59 #include <sys/debug.h>
60 #include <sys/vtrace.h>
61 #include <sys/stack.h>
62 #include <sys/atomic.h>
63 #include <sys/archsystm.h>
64 #include <sys/lgrp.h>
65
66 #include <vm/as.h>
67 #include <vm/seg.h>
68 #include <vm/seg_kp.h>
69 #include <vm/seg_kmem.h>
70 #include <vm/anon.h>
71 #include <vm/page.h>
72 #include <vm/hat.h>
73 #include <sys/bitmap.h>
74
75 /*
76 * Private seg op routines
77 */
78 static void segkp_badop(void);
79 static void segkp_dump(struct seg *seg);
80 static int segkp_checkprot(struct seg *seg, caddr_t addr, size_t len,
81 uint_t prot);
82 static int segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta);
83 static int segkp_pagelock(struct seg *seg, caddr_t addr, size_t len,
84 struct page ***page, enum lock_type type,
85 enum seg_rw rw);
86 static void segkp_insert(struct seg *seg, struct segkp_data *kpd);
87 static void segkp_delete(struct seg *seg, struct segkp_data *kpd);
88 static caddr_t segkp_get_internal(struct seg *seg, size_t len, uint_t flags,
89 struct segkp_data **tkpd, struct anon_map *amp);
90 static void segkp_release_internal(struct seg *seg,
91 struct segkp_data *kpd, size_t len);
92 static int segkp_unlock(struct hat *hat, struct seg *seg, caddr_t vaddr,
93 size_t len, struct segkp_data *kpd, uint_t flags);
94 static int segkp_load(struct hat *hat, struct seg *seg, caddr_t vaddr,
95 size_t len, struct segkp_data *kpd, uint_t flags);
96 static struct segkp_data *segkp_find(struct seg *seg, caddr_t vaddr);
97 static int segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp);
98
99 /*
100 * Lock used to protect the hash table(s) and caches.
101 */
102 static kmutex_t segkp_lock;
103
104 /*
105 * The segkp caches
106 */
107 static struct segkp_cache segkp_cache[SEGKP_MAX_CACHE];
108
109 #define SEGKP_BADOP(t) (t(*)())segkp_badop
110
111 /*
112 * When there are fewer than red_minavail bytes left on the stack,
113 * segkp_map_red() will map in the redzone (if called). 5000 seems
114 * to work reasonably well...
115 */
116 long red_minavail = 5000;
117
118 /*
119 * will be set to 1 for 32 bit x86 systems only, in startup.c
120 */
121 int segkp_fromheap = 0;
122 ulong_t *segkp_bitmap;
123
124 /*
125 * If segkp_map_red() is called with the redzone already mapped and
126 * with less than RED_DEEP_THRESHOLD bytes available on the stack,
127 * then the stack situation has become quite serious; if much more stack
128 * is consumed, we have the potential of scrogging the next thread/LWP
129 * structure. To help debug the "can't happen" panics which may
130 * result from this condition, we record hrestime and the calling thread
131 * in red_deep_hires and red_deep_thread respectively.
132 */
133 #define RED_DEEP_THRESHOLD 2000
134
135 hrtime_t red_deep_hires;
136 kthread_t *red_deep_thread;
137
138 uint32_t red_nmapped;
139 uint32_t red_closest = UINT_MAX;
140 uint32_t red_ndoubles;
141
142 pgcnt_t anon_segkp_pages_locked; /* See vm/anon.h */
143 pgcnt_t anon_segkp_pages_resv; /* anon reserved by seg_kp */
144
145 static struct seg_ops segkp_ops = {
146 .dup = SEGKP_BADOP(int),
147 .unmap = SEGKP_BADOP(int),
148 .free = SEGKP_BADOP(void),
149 .fault = segkp_fault,
150 .faulta = SEGKP_BADOP(faultcode_t),
151 .setprot = SEGKP_BADOP(int),
152 .checkprot = segkp_checkprot,
153 .kluster = segkp_kluster,
154 .swapout = SEGKP_BADOP(size_t),
155 .sync = SEGKP_BADOP(int),
156 .incore = SEGKP_BADOP(size_t),
157 .lockop = SEGKP_BADOP(int),
158 .getprot = SEGKP_BADOP(int),
159 .getoffset = SEGKP_BADOP(u_offset_t),
160 .gettype = SEGKP_BADOP(int),
161 .getvp = SEGKP_BADOP(int),
162 .advise = SEGKP_BADOP(int),
163 .dump = segkp_dump,
164 .pagelock = segkp_pagelock,
165 .setpagesize = SEGKP_BADOP(int),
166 .getmemid = segkp_getmemid,
167 };
168
169
170 static void
171 segkp_badop(void)
172 {
173 panic("segkp_badop");
174 /*NOTREACHED*/
175 }
176
177 static void segkpinit_mem_config(struct seg *);
178
179 static uint32_t segkp_indel;
180
181 /*
182 * Allocate the segment specific private data struct and fill it in
183 * with the per kp segment mutex, anon ptr. array and hash table.
184 */
185 int
186 segkp_create(struct seg *seg)
187 {
188 struct segkp_segdata *kpsd;
189 size_t np;
190
191 ASSERT(seg != NULL && seg->s_as == &kas);
192 ASSERT(RW_WRITE_HELD(&seg->s_as->a_lock));
193
194 if (seg->s_size & PAGEOFFSET) {
195 panic("Bad segkp size");
196 /*NOTREACHED*/
197 }
198
199 kpsd = kmem_zalloc(sizeof (struct segkp_segdata), KM_SLEEP);
200
201 /*
202 * Allocate the virtual memory for segkp and initialize it
203 */
204 if (segkp_fromheap) {
205 np = btop(kvseg.s_size);
206 segkp_bitmap = kmem_zalloc(BT_SIZEOFMAP(np), KM_SLEEP);
207 kpsd->kpsd_arena = vmem_create("segkp", NULL, 0, PAGESIZE,
208 vmem_alloc, vmem_free, heap_arena, 5 * PAGESIZE, VM_SLEEP);
209 } else {
210 segkp_bitmap = NULL;
211 np = btop(seg->s_size);
212 kpsd->kpsd_arena = vmem_create("segkp", seg->s_base,
213 seg->s_size, PAGESIZE, NULL, NULL, NULL, 5 * PAGESIZE,
214 VM_SLEEP);
215 }
216
217 kpsd->kpsd_anon = anon_create(np, ANON_SLEEP | ANON_ALLOC_FORCE);
218
219 kpsd->kpsd_hash = kmem_zalloc(SEGKP_HASHSZ * sizeof (struct segkp *),
220 KM_SLEEP);
221 seg->s_data = (void *)kpsd;
222 seg->s_ops = &segkp_ops;
223 segkpinit_mem_config(seg);
224 return (0);
225 }
226
227
228 /*
229 * Find a free 'freelist' and initialize it with the appropriate attributes
230 */
231 void *
232 segkp_cache_init(struct seg *seg, int maxsize, size_t len, uint_t flags)
233 {
234 int i;
235
236 if ((flags & KPD_NO_ANON) && !(flags & KPD_LOCKED))
237 return ((void *)-1);
238
239 mutex_enter(&segkp_lock);
240 for (i = 0; i < SEGKP_MAX_CACHE; i++) {
241 if (segkp_cache[i].kpf_inuse)
242 continue;
243 segkp_cache[i].kpf_inuse = 1;
244 segkp_cache[i].kpf_max = maxsize;
245 segkp_cache[i].kpf_flags = flags;
246 segkp_cache[i].kpf_seg = seg;
247 segkp_cache[i].kpf_len = len;
248 mutex_exit(&segkp_lock);
249 return ((void *)(uintptr_t)i);
250 }
251 mutex_exit(&segkp_lock);
252 return ((void *)-1);
253 }
254
255 /*
256 * Free all the cache resources.
257 */
258 void
259 segkp_cache_free(void)
260 {
261 struct segkp_data *kpd;
262 struct seg *seg;
263 int i;
264
265 mutex_enter(&segkp_lock);
266 for (i = 0; i < SEGKP_MAX_CACHE; i++) {
267 if (!segkp_cache[i].kpf_inuse)
268 continue;
269 /*
270 * Disconnect the freelist and process each element
271 */
272 kpd = segkp_cache[i].kpf_list;
273 seg = segkp_cache[i].kpf_seg;
274 segkp_cache[i].kpf_list = NULL;
275 segkp_cache[i].kpf_count = 0;
276 mutex_exit(&segkp_lock);
277
278 while (kpd != NULL) {
279 struct segkp_data *next;
280
281 next = kpd->kp_next;
282 segkp_release_internal(seg, kpd, kpd->kp_len);
283 kpd = next;
284 }
285 mutex_enter(&segkp_lock);
286 }
287 mutex_exit(&segkp_lock);
288 }
289
290 /*
291 * There are 2 entries into segkp_get_internal. The first includes a cookie
292 * used to access a pool of cached segkp resources. The second does not
293 * use the cache.
294 */
295 caddr_t
296 segkp_get(struct seg *seg, size_t len, uint_t flags)
297 {
298 struct segkp_data *kpd = NULL;
299
300 if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) {
301 kpd->kp_cookie = -1;
302 return (stom(kpd->kp_base, flags));
303 }
304 return (NULL);
305 }
306
307 /*
308 * Return a 'cached' segkp address
309 */
310 caddr_t
311 segkp_cache_get(void *cookie)
312 {
313 struct segkp_cache *freelist = NULL;
314 struct segkp_data *kpd = NULL;
315 int index = (int)(uintptr_t)cookie;
316 struct seg *seg;
317 size_t len;
318 uint_t flags;
319
320 if (index < 0 || index >= SEGKP_MAX_CACHE)
321 return (NULL);
322 freelist = &segkp_cache[index];
323
324 mutex_enter(&segkp_lock);
325 seg = freelist->kpf_seg;
326 flags = freelist->kpf_flags;
327 if (freelist->kpf_list != NULL) {
328 kpd = freelist->kpf_list;
329 freelist->kpf_list = kpd->kp_next;
330 freelist->kpf_count--;
331 mutex_exit(&segkp_lock);
332 kpd->kp_next = NULL;
333 segkp_insert(seg, kpd);
334 return (stom(kpd->kp_base, flags));
335 }
336 len = freelist->kpf_len;
337 mutex_exit(&segkp_lock);
338 if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) {
339 kpd->kp_cookie = index;
340 return (stom(kpd->kp_base, flags));
341 }
342 return (NULL);
343 }
344
345 caddr_t
346 segkp_get_withanonmap(
347 struct seg *seg,
348 size_t len,
349 uint_t flags,
350 struct anon_map *amp)
351 {
352 struct segkp_data *kpd = NULL;
353
354 ASSERT(amp != NULL);
355 flags |= KPD_HASAMP;
356 if (segkp_get_internal(seg, len, flags, &kpd, amp) != NULL) {
357 kpd->kp_cookie = -1;
358 return (stom(kpd->kp_base, flags));
359 }
360 return (NULL);
361 }
362
363 /*
364 * This does the real work of segkp allocation.
365 * Return to client base addr. len must be page-aligned. A null value is
366 * returned if there are no more vm resources (e.g. pages, swap). The len
367 * and base recorded in the private data structure include the redzone
368 * and the redzone length (if applicable). If the user requests a redzone
369 * either the first or last page is left unmapped depending whether stacks
370 * grow to low or high memory.
371 *
372 * The client may also specify a no-wait flag. If that is set then the
373 * request will choose a non-blocking path when requesting resources.
374 * The default is make the client wait.
375 */
376 static caddr_t
377 segkp_get_internal(
378 struct seg *seg,
379 size_t len,
380 uint_t flags,
381 struct segkp_data **tkpd,
382 struct anon_map *amp)
383 {
384 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
385 struct segkp_data *kpd;
386 caddr_t vbase = NULL; /* always first virtual, may not be mapped */
387 pgcnt_t np = 0; /* number of pages in the resource */
388 pgcnt_t segkpindex;
389 long i;
390 caddr_t va;
391 pgcnt_t pages = 0;
392 ulong_t anon_idx = 0;
393 int kmflag = (flags & KPD_NOWAIT) ? KM_NOSLEEP : KM_SLEEP;
394 caddr_t s_base = (segkp_fromheap) ? kvseg.s_base : seg->s_base;
395
396 if (len & PAGEOFFSET) {
397 panic("segkp_get: len is not page-aligned");
398 /*NOTREACHED*/
399 }
400
401 ASSERT(((flags & KPD_HASAMP) == 0) == (amp == NULL));
402
403 /* Only allow KPD_NO_ANON if we are going to lock it down */
404 if ((flags & (KPD_LOCKED|KPD_NO_ANON)) == KPD_NO_ANON)
405 return (NULL);
406
407 if ((kpd = kmem_zalloc(sizeof (struct segkp_data), kmflag)) == NULL)
408 return (NULL);
409 /*
410 * Fix up the len to reflect the REDZONE if applicable
411 */
412 if (flags & KPD_HASREDZONE)
413 len += PAGESIZE;
414 np = btop(len);
415
416 vbase = vmem_alloc(SEGKP_VMEM(seg), len, kmflag | VM_BESTFIT);
417 if (vbase == NULL) {
418 kmem_free(kpd, sizeof (struct segkp_data));
419 return (NULL);
420 }
421
422 /* If locking, reserve physical memory */
423 if (flags & KPD_LOCKED) {
424 pages = btop(SEGKP_MAPLEN(len, flags));
425 if (page_resv(pages, kmflag) == 0) {
426 vmem_free(SEGKP_VMEM(seg), vbase, len);
427 kmem_free(kpd, sizeof (struct segkp_data));
428 return (NULL);
429 }
430 if ((flags & KPD_NO_ANON) == 0)
431 atomic_add_long(&anon_segkp_pages_locked, pages);
432 }
433
434 /*
435 * Reserve sufficient swap space for this vm resource. We'll
436 * actually allocate it in the loop below, but reserving it
437 * here allows us to back out more gracefully than if we
438 * had an allocation failure in the body of the loop.
439 *
440 * Note that we don't need swap space for the red zone page.
441 */
442 if (amp != NULL) {
443 /*
444 * The swap reservation has been done, if required, and the
445 * anon_hdr is separate.
446 */
447 anon_idx = 0;
448 kpd->kp_anon_idx = anon_idx;
449 kpd->kp_anon = amp->ahp;
450
451 TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u",
452 kpd, vbase, len, flags, 1);
453
454 } else if ((flags & KPD_NO_ANON) == 0) {
455 if (anon_resv_zone(SEGKP_MAPLEN(len, flags), NULL) == 0) {
456 if (flags & KPD_LOCKED) {
457 atomic_add_long(&anon_segkp_pages_locked,
458 -pages);
459 page_unresv(pages);
460 }
461 vmem_free(SEGKP_VMEM(seg), vbase, len);
462 kmem_free(kpd, sizeof (struct segkp_data));
463 return (NULL);
464 }
465 atomic_add_long(&anon_segkp_pages_resv,
466 btop(SEGKP_MAPLEN(len, flags)));
467 anon_idx = ((uintptr_t)(vbase - s_base)) >> PAGESHIFT;
468 kpd->kp_anon_idx = anon_idx;
469 kpd->kp_anon = kpsd->kpsd_anon;
470
471 TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u",
472 kpd, vbase, len, flags, 1);
473 } else {
474 kpd->kp_anon = NULL;
475 kpd->kp_anon_idx = 0;
476 }
477
478 /*
479 * Allocate page and anon resources for the virtual address range
480 * except the redzone
481 */
482 if (segkp_fromheap)
483 segkpindex = btop((uintptr_t)(vbase - kvseg.s_base));
484 for (i = 0, va = vbase; i < np; i++, va += PAGESIZE) {
485 page_t *pl[2];
486 struct vnode *vp;
487 anoff_t off;
488 int err;
489 page_t *pp = NULL;
490
491 /*
492 * Mark this page to be a segkp page in the bitmap.
493 */
494 if (segkp_fromheap) {
495 BT_ATOMIC_SET(segkp_bitmap, segkpindex);
496 segkpindex++;
497 }
498
499 /*
500 * If this page is the red zone page, we don't need swap
501 * space for it. Note that we skip over the code that
502 * establishes MMU mappings, so that the page remains
503 * invalid.
504 */
505 if ((flags & KPD_HASREDZONE) && KPD_REDZONE(kpd) == i)
506 continue;
507
508 if (kpd->kp_anon != NULL) {
509 struct anon *ap;
510
511 ASSERT(anon_get_ptr(kpd->kp_anon, anon_idx + i)
512 == NULL);
513 /*
514 * Determine the "vp" and "off" of the anon slot.
515 */
516 ap = anon_alloc(NULL, 0);
517 if (amp != NULL)
518 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER);
519 (void) anon_set_ptr(kpd->kp_anon, anon_idx + i,
520 ap, ANON_SLEEP);
521 if (amp != NULL)
522 ANON_LOCK_EXIT(&->a_rwlock);
523 swap_xlate(ap, &vp, &off);
524
525 /*
526 * Create a page with the specified identity. The
527 * page is returned with the "shared" lock held.
528 */
529 err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE,
530 NULL, pl, PAGESIZE, seg, va, S_CREATE,
531 kcred, NULL);
532 if (err) {
533 /*
534 * XXX - This should not fail.
535 */
536 panic("segkp_get: no pages");
537 /*NOTREACHED*/
538 }
539 pp = pl[0];
540 } else {
541 ASSERT(page_exists(&kvp,
542 (u_offset_t)(uintptr_t)va) == NULL);
543
544 if ((pp = page_create_va(&kvp,
545 (u_offset_t)(uintptr_t)va, PAGESIZE,
546 (flags & KPD_NOWAIT ? 0 : PG_WAIT) | PG_EXCL |
547 PG_NORELOC, seg, va)) == NULL) {
548 /*
549 * Legitimize resource; then destroy it.
550 * Easier than trying to unwind here.
551 */
552 kpd->kp_flags = flags;
553 kpd->kp_base = vbase;
554 kpd->kp_len = len;
555 segkp_release_internal(seg, kpd, va - vbase);
556 return (NULL);
557 }
558 page_io_unlock(pp);
559 }
560
561 if (flags & KPD_ZERO)
562 pagezero(pp, 0, PAGESIZE);
563
564 /*
565 * Load and lock an MMU translation for the page.
566 */
567 hat_memload(seg->s_as->a_hat, va, pp, (PROT_READ|PROT_WRITE),
568 ((flags & KPD_LOCKED) ? HAT_LOAD_LOCK : HAT_LOAD));
569
570 /*
571 * Now, release lock on the page.
572 */
573 if (flags & KPD_LOCKED) {
574 /*
575 * Indicate to page_retire framework that this
576 * page can only be retired when it is freed.
577 */
578 PP_SETRAF(pp);
579 page_downgrade(pp);
580 } else
581 page_unlock(pp);
582 }
583
584 kpd->kp_flags = flags;
585 kpd->kp_base = vbase;
586 kpd->kp_len = len;
587 segkp_insert(seg, kpd);
588 *tkpd = kpd;
589 return (stom(kpd->kp_base, flags));
590 }
591
592 /*
593 * Release the resource to cache if the pool(designate by the cookie)
594 * has less than the maximum allowable. If inserted in cache,
595 * segkp_delete insures element is taken off of active list.
596 */
597 void
598 segkp_release(struct seg *seg, caddr_t vaddr)
599 {
600 struct segkp_cache *freelist;
601 struct segkp_data *kpd = NULL;
602
603 if ((kpd = segkp_find(seg, vaddr)) == NULL) {
604 panic("segkp_release: null kpd");
605 /*NOTREACHED*/
606 }
607
608 if (kpd->kp_cookie != -1) {
609 freelist = &segkp_cache[kpd->kp_cookie];
610 mutex_enter(&segkp_lock);
611 if (!segkp_indel && freelist->kpf_count < freelist->kpf_max) {
612 segkp_delete(seg, kpd);
613 kpd->kp_next = freelist->kpf_list;
614 freelist->kpf_list = kpd;
615 freelist->kpf_count++;
616 mutex_exit(&segkp_lock);
617 return;
618 } else {
619 mutex_exit(&segkp_lock);
620 kpd->kp_cookie = -1;
621 }
622 }
623 segkp_release_internal(seg, kpd, kpd->kp_len);
624 }
625
626 /*
627 * Free the entire resource. segkp_unlock gets called with the start of the
628 * mapped portion of the resource. The length is the size of the mapped
629 * portion
630 */
631 static void
632 segkp_release_internal(struct seg *seg, struct segkp_data *kpd, size_t len)
633 {
634 caddr_t va;
635 long i;
636 long redzone;
637 size_t np;
638 page_t *pp;
639 struct vnode *vp;
640 anoff_t off;
641 struct anon *ap;
642 pgcnt_t segkpindex;
643
644 ASSERT(kpd != NULL);
645 ASSERT((kpd->kp_flags & KPD_HASAMP) == 0 || kpd->kp_cookie == -1);
646 np = btop(len);
647
648 /* Remove from active hash list */
649 if (kpd->kp_cookie == -1) {
650 mutex_enter(&segkp_lock);
651 segkp_delete(seg, kpd);
652 mutex_exit(&segkp_lock);
653 }
654
655 /*
656 * Precompute redzone page index.
657 */
658 redzone = -1;
659 if (kpd->kp_flags & KPD_HASREDZONE)
660 redzone = KPD_REDZONE(kpd);
661
662
663 va = kpd->kp_base;
664
665 hat_unload(seg->s_as->a_hat, va, (np << PAGESHIFT),
666 ((kpd->kp_flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD));
667 /*
668 * Free up those anon resources that are quiescent.
669 */
670 if (segkp_fromheap)
671 segkpindex = btop((uintptr_t)(va - kvseg.s_base));
672 for (i = 0; i < np; i++, va += PAGESIZE) {
673
674 /*
675 * Clear the bit for this page from the bitmap.
676 */
677 if (segkp_fromheap) {
678 BT_ATOMIC_CLEAR(segkp_bitmap, segkpindex);
679 segkpindex++;
680 }
681
682 if (i == redzone)
683 continue;
684 if (kpd->kp_anon) {
685 /*
686 * Free up anon resources and destroy the
687 * associated pages.
688 *
689 * Release the lock if there is one. Have to get the
690 * page to do this, unfortunately.
691 */
692 if (kpd->kp_flags & KPD_LOCKED) {
693 ap = anon_get_ptr(kpd->kp_anon,
694 kpd->kp_anon_idx + i);
695 swap_xlate(ap, &vp, &off);
696 /* Find the shared-locked page. */
697 pp = page_find(vp, (u_offset_t)off);
698 if (pp == NULL) {
699 panic("segkp_release: "
700 "kp_anon: no page to unlock ");
701 /*NOTREACHED*/
702 }
703 if (PP_ISRAF(pp))
704 PP_CLRRAF(pp);
705
706 page_unlock(pp);
707 }
708 if ((kpd->kp_flags & KPD_HASAMP) == 0) {
709 anon_free(kpd->kp_anon, kpd->kp_anon_idx + i,
710 PAGESIZE);
711 anon_unresv_zone(PAGESIZE, NULL);
712 atomic_dec_ulong(&anon_segkp_pages_resv);
713 }
714 TRACE_5(TR_FAC_VM,
715 TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u",
716 kpd, va, PAGESIZE, 0, 0);
717 } else {
718 if (kpd->kp_flags & KPD_LOCKED) {
719 pp = page_find(&kvp, (u_offset_t)(uintptr_t)va);
720 if (pp == NULL) {
721 panic("segkp_release: "
722 "no page to unlock");
723 /*NOTREACHED*/
724 }
725 if (PP_ISRAF(pp))
726 PP_CLRRAF(pp);
727 /*
728 * We should just upgrade the lock here
729 * but there is no upgrade that waits.
730 */
731 page_unlock(pp);
732 }
733 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)va,
734 SE_EXCL);
735 if (pp != NULL)
736 page_destroy(pp, 0);
737 }
738 }
739
740 /* If locked, release physical memory reservation */
741 if (kpd->kp_flags & KPD_LOCKED) {
742 pgcnt_t pages = btop(SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags));
743 if ((kpd->kp_flags & KPD_NO_ANON) == 0)
744 atomic_add_long(&anon_segkp_pages_locked, -pages);
745 page_unresv(pages);
746 }
747
748 vmem_free(SEGKP_VMEM(seg), kpd->kp_base, kpd->kp_len);
749 kmem_free(kpd, sizeof (struct segkp_data));
750 }
751
752 /*
753 * segkp_map_red() will check the current frame pointer against the
754 * stack base. If the amount of stack remaining is questionable
755 * (less than red_minavail), then segkp_map_red() will map in the redzone
756 * and return 1. Otherwise, it will return 0. segkp_map_red() can
757 * _only_ be called when:
758 *
759 * - it is safe to sleep on page_create_va().
760 * - the caller is non-swappable.
761 *
762 * It is up to the caller to remember whether segkp_map_red() successfully
763 * mapped the redzone, and, if so, to call segkp_unmap_red() at a later
764 * time. Note that the caller must _remain_ non-swappable until after
765 * calling segkp_unmap_red().
766 *
767 * Currently, this routine is only called from pagefault() (which necessarily
768 * satisfies the above conditions).
769 */
770 #if defined(STACK_GROWTH_DOWN)
771 int
772 segkp_map_red(void)
773 {
774 uintptr_t fp = STACK_BIAS + (uintptr_t)getfp();
775 #ifndef _LP64
776 caddr_t stkbase;
777 #endif
778
779 ASSERT(curthread->t_schedflag & TS_DONT_SWAP);
780
781 /*
782 * Optimize for the common case where we simply return.
783 */
784 if ((curthread->t_red_pp == NULL) &&
785 (fp - (uintptr_t)curthread->t_stkbase >= red_minavail))
786 return (0);
787
788 #if defined(_LP64)
789 /*
790 * XXX We probably need something better than this.
791 */
792 panic("kernel stack overflow");
793 /*NOTREACHED*/
794 #else /* _LP64 */
795 if (curthread->t_red_pp == NULL) {
796 page_t *red_pp;
797 struct seg kseg;
798
799 caddr_t red_va = (caddr_t)
800 (((uintptr_t)curthread->t_stkbase & (uintptr_t)PAGEMASK) -
801 PAGESIZE);
802
803 ASSERT(page_exists(&kvp, (u_offset_t)(uintptr_t)red_va) ==
804 NULL);
805
806 /*
807 * Allocate the physical for the red page.
808 */
809 /*
810 * No PG_NORELOC here to avoid waits. Unlikely to get
811 * a relocate happening in the short time the page exists
812 * and it will be OK anyway.
813 */
814
815 kseg.s_as = &kas;
816 red_pp = page_create_va(&kvp, (u_offset_t)(uintptr_t)red_va,
817 PAGESIZE, PG_WAIT | PG_EXCL, &kseg, red_va);
818 ASSERT(red_pp != NULL);
819
820 /*
821 * So we now have a page to jam into the redzone...
822 */
823 page_io_unlock(red_pp);
824
825 hat_memload(kas.a_hat, red_va, red_pp,
826 (PROT_READ|PROT_WRITE), HAT_LOAD_LOCK);
827 page_downgrade(red_pp);
828
829 /*
830 * The page is left SE_SHARED locked so we can hold on to
831 * the page_t pointer.
832 */
833 curthread->t_red_pp = red_pp;
834
835 atomic_inc_32(&red_nmapped);
836 while (fp - (uintptr_t)curthread->t_stkbase < red_closest) {
837 (void) atomic_cas_32(&red_closest, red_closest,
838 (uint32_t)(fp - (uintptr_t)curthread->t_stkbase));
839 }
840 return (1);
841 }
842
843 stkbase = (caddr_t)(((uintptr_t)curthread->t_stkbase &
844 (uintptr_t)PAGEMASK) - PAGESIZE);
845
846 atomic_inc_32(&red_ndoubles);
847
848 if (fp - (uintptr_t)stkbase < RED_DEEP_THRESHOLD) {
849 /*
850 * Oh boy. We're already deep within the mapped-in
851 * redzone page, and the caller is trying to prepare
852 * for a deep stack run. We're running without a
853 * redzone right now: if the caller plows off the
854 * end of the stack, it'll plow another thread or
855 * LWP structure. That situation could result in
856 * a very hard-to-debug panic, so, in the spirit of
857 * recording the name of one's killer in one's own
858 * blood, we're going to record hrestime and the calling
859 * thread.
860 */
861 red_deep_hires = hrestime.tv_nsec;
862 red_deep_thread = curthread;
863 }
864
865 /*
866 * If this is a DEBUG kernel, and we've run too deep for comfort, toss.
867 */
868 ASSERT(fp - (uintptr_t)stkbase >= RED_DEEP_THRESHOLD);
869 return (0);
870 #endif /* _LP64 */
871 }
872
873 void
874 segkp_unmap_red(void)
875 {
876 page_t *pp;
877 caddr_t red_va = (caddr_t)(((uintptr_t)curthread->t_stkbase &
878 (uintptr_t)PAGEMASK) - PAGESIZE);
879
880 ASSERT(curthread->t_red_pp != NULL);
881 ASSERT(curthread->t_schedflag & TS_DONT_SWAP);
882
883 /*
884 * Because we locked the mapping down, we can't simply rely
885 * on page_destroy() to clean everything up; we need to call
886 * hat_unload() to explicitly unlock the mapping resources.
887 */
888 hat_unload(kas.a_hat, red_va, PAGESIZE, HAT_UNLOAD_UNLOCK);
889
890 pp = curthread->t_red_pp;
891
892 ASSERT(pp == page_find(&kvp, (u_offset_t)(uintptr_t)red_va));
893
894 /*
895 * Need to upgrade the SE_SHARED lock to SE_EXCL.
896 */
897 if (!page_tryupgrade(pp)) {
898 /*
899 * As there is now wait for upgrade, release the
900 * SE_SHARED lock and wait for SE_EXCL.
901 */
902 page_unlock(pp);
903 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)red_va, SE_EXCL);
904 /* pp may be NULL here, hence the test below */
905 }
906
907 /*
908 * Destroy the page, with dontfree set to zero (i.e. free it).
909 */
910 if (pp != NULL)
911 page_destroy(pp, 0);
912 curthread->t_red_pp = NULL;
913 }
914 #else
915 #error Red stacks only supported with downwards stack growth.
916 #endif
917
918 /*
919 * Handle a fault on an address corresponding to one of the
920 * resources in the segkp segment.
921 */
922 faultcode_t
923 segkp_fault(
924 struct hat *hat,
925 struct seg *seg,
926 caddr_t vaddr,
927 size_t len,
928 enum fault_type type,
929 enum seg_rw rw)
930 {
931 struct segkp_data *kpd = NULL;
932 int err;
933
934 ASSERT(seg->s_as == &kas && RW_READ_HELD(&seg->s_as->a_lock));
935
936 /*
937 * Sanity checks.
938 */
939 if (type == F_PROT) {
940 panic("segkp_fault: unexpected F_PROT fault");
941 /*NOTREACHED*/
942 }
943
944 if ((kpd = segkp_find(seg, vaddr)) == NULL)
945 return (FC_NOMAP);
946
947 mutex_enter(&kpd->kp_lock);
948
949 if (type == F_SOFTLOCK) {
950 ASSERT(!(kpd->kp_flags & KPD_LOCKED));
951 /*
952 * The F_SOFTLOCK case has more stringent
953 * range requirements: the given range must exactly coincide
954 * with the resource's mapped portion. Note reference to
955 * redzone is handled since vaddr would not equal base
956 */
957 if (vaddr != stom(kpd->kp_base, kpd->kp_flags) ||
958 len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) {
959 mutex_exit(&kpd->kp_lock);
960 return (FC_MAKE_ERR(EFAULT));
961 }
962
963 if ((err = segkp_load(hat, seg, vaddr, len, kpd, KPD_LOCKED))) {
964 mutex_exit(&kpd->kp_lock);
965 return (FC_MAKE_ERR(err));
966 }
967 kpd->kp_flags |= KPD_LOCKED;
968 mutex_exit(&kpd->kp_lock);
969 return (0);
970 }
971
972 if (type == F_INVAL) {
973 ASSERT(!(kpd->kp_flags & KPD_NO_ANON));
974
975 /*
976 * Check if we touched the redzone. Somewhat optimistic
977 * here if we are touching the redzone of our own stack
978 * since we wouldn't have a stack to get this far...
979 */
980 if ((kpd->kp_flags & KPD_HASREDZONE) &&
981 btop((uintptr_t)(vaddr - kpd->kp_base)) == KPD_REDZONE(kpd))
982 panic("segkp_fault: accessing redzone");
983
984 /*
985 * This fault may occur while the page is being F_SOFTLOCK'ed.
986 * Return since a 2nd segkp_load is unnecessary and also would
987 * result in the page being locked twice and eventually
988 * hang the thread_reaper thread.
989 */
990 if (kpd->kp_flags & KPD_LOCKED) {
991 mutex_exit(&kpd->kp_lock);
992 return (0);
993 }
994
995 err = segkp_load(hat, seg, vaddr, len, kpd, kpd->kp_flags);
996 mutex_exit(&kpd->kp_lock);
997 return (err ? FC_MAKE_ERR(err) : 0);
998 }
999
1000 if (type == F_SOFTUNLOCK) {
1001 uint_t flags;
1002
1003 /*
1004 * Make sure the addr is LOCKED and it has anon backing
1005 * before unlocking
1006 */
1007 if ((kpd->kp_flags & (KPD_LOCKED|KPD_NO_ANON)) != KPD_LOCKED) {
1008 panic("segkp_fault: bad unlock");
1009 /*NOTREACHED*/
1010 }
1011
1012 if (vaddr != stom(kpd->kp_base, kpd->kp_flags) ||
1013 len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) {
1014 panic("segkp_fault: bad range");
1015 /*NOTREACHED*/
1016 }
1017
1018 if (rw == S_WRITE)
1019 flags = kpd->kp_flags | KPD_WRITEDIRTY;
1020 else
1021 flags = kpd->kp_flags;
1022 err = segkp_unlock(hat, seg, vaddr, len, kpd, flags);
1023 kpd->kp_flags &= ~KPD_LOCKED;
1024 mutex_exit(&kpd->kp_lock);
1025 return (err ? FC_MAKE_ERR(err) : 0);
1026 }
1027 mutex_exit(&kpd->kp_lock);
1028 panic("segkp_fault: bogus fault type: %d\n", type);
1029 /*NOTREACHED*/
1030 }
1031
1032 /*
1033 * Check that the given protections suffice over the range specified by
1034 * vaddr and len. For this segment type, the only issue is whether or
1035 * not the range lies completely within the mapped part of an allocated
1036 * resource.
1037 */
1038 /* ARGSUSED */
1039 static int
1040 segkp_checkprot(struct seg *seg, caddr_t vaddr, size_t len, uint_t prot)
1041 {
1042 struct segkp_data *kpd = NULL;
1043 caddr_t mbase;
1044 size_t mlen;
1045
1046 if ((kpd = segkp_find(seg, vaddr)) == NULL)
1047 return (EACCES);
1048
1049 mutex_enter(&kpd->kp_lock);
1050 mbase = stom(kpd->kp_base, kpd->kp_flags);
1051 mlen = SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags);
1052 if (len > mlen || vaddr < mbase ||
1053 ((vaddr + len) > (mbase + mlen))) {
1054 mutex_exit(&kpd->kp_lock);
1055 return (EACCES);
1056 }
1057 mutex_exit(&kpd->kp_lock);
1058 return (0);
1059 }
1060
1061
1062 /*
1063 * Check to see if it makes sense to do kluster/read ahead to
1064 * addr + delta relative to the mapping at addr. We assume here
1065 * that delta is a signed PAGESIZE'd multiple (which can be negative).
1066 *
1067 * For seg_u we always "approve" of this action from our standpoint.
1068 */
1069 /*ARGSUSED*/
1070 static int
1071 segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
1072 {
1073 return (0);
1074 }
1075
1076 /*
1077 * Load and possibly lock intra-slot resources in the range given by
1078 * vaddr and len.
1079 */
1080 static int
1081 segkp_load(
1082 struct hat *hat,
1083 struct seg *seg,
1084 caddr_t vaddr,
1085 size_t len,
1086 struct segkp_data *kpd,
1087 uint_t flags)
1088 {
1089 caddr_t va;
1090 caddr_t vlim;
1091 ulong_t i;
1092 uint_t lock;
1093
1094 ASSERT(MUTEX_HELD(&kpd->kp_lock));
1095
1096 len = P2ROUNDUP(len, PAGESIZE);
1097
1098 /* If locking, reserve physical memory */
1099 if (flags & KPD_LOCKED) {
1100 pgcnt_t pages = btop(len);
1101 if ((kpd->kp_flags & KPD_NO_ANON) == 0)
1102 atomic_add_long(&anon_segkp_pages_locked, pages);
1103 (void) page_resv(pages, KM_SLEEP);
1104 }
1105
1106 /*
1107 * Loop through the pages in the given range.
1108 */
1109 va = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK);
1110 vaddr = va;
1111 vlim = va + len;
1112 lock = flags & KPD_LOCKED;
1113 i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT;
1114 for (; va < vlim; va += PAGESIZE, i++) {
1115 page_t *pl[2]; /* second element NULL terminator */
1116 struct vnode *vp;
1117 anoff_t off;
1118 int err;
1119 struct anon *ap;
1120
1121 /*
1122 * Summon the page. If it's not resident, arrange
1123 * for synchronous i/o to pull it in.
1124 */
1125 ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i);
1126 swap_xlate(ap, &vp, &off);
1127
1128 /*
1129 * The returned page list will have exactly one entry,
1130 * which is returned to us already kept.
1131 */
1132 err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE, NULL,
1133 pl, PAGESIZE, seg, va, S_READ, kcred, NULL);
1134
1135 if (err) {
1136 /*
1137 * Back out of what we've done so far.
1138 */
1139 (void) segkp_unlock(hat, seg, vaddr,
1140 (va - vaddr), kpd, flags);
1141 return (err);
1142 }
1143
1144 /*
1145 * Load an MMU translation for the page.
1146 */
1147 hat_memload(hat, va, pl[0], (PROT_READ|PROT_WRITE),
1148 lock ? HAT_LOAD_LOCK : HAT_LOAD);
1149
1150 if (!lock) {
1151 /*
1152 * Now, release "shared" lock on the page.
1153 */
1154 page_unlock(pl[0]);
1155 }
1156 }
1157 return (0);
1158 }
1159
1160 /*
1161 * At the very least unload the mmu-translations and unlock the range if locked
1162 * Can be called with the following flag value KPD_WRITEDIRTY which specifies
1163 * any dirty pages should be written to disk.
1164 */
1165 static int
1166 segkp_unlock(
1167 struct hat *hat,
1168 struct seg *seg,
1169 caddr_t vaddr,
1170 size_t len,
1171 struct segkp_data *kpd,
1172 uint_t flags)
1173 {
1174 caddr_t va;
1175 caddr_t vlim;
1176 ulong_t i;
1177 struct page *pp;
1178 struct vnode *vp;
1179 anoff_t off;
1180 struct anon *ap;
1181
1182 #ifdef lint
1183 seg = seg;
1184 #endif /* lint */
1185
1186 ASSERT(MUTEX_HELD(&kpd->kp_lock));
1187
1188 /*
1189 * Loop through the pages in the given range. It is assumed
1190 * segkp_unlock is called with page aligned base
1191 */
1192 va = vaddr;
1193 vlim = va + len;
1194 i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT;
1195 hat_unload(hat, va, len,
1196 ((flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD));
1197 for (; va < vlim; va += PAGESIZE, i++) {
1198 /*
1199 * Find the page associated with this part of the
1200 * slot, tracking it down through its associated swap
1201 * space.
1202 */
1203 ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i);
1204 swap_xlate(ap, &vp, &off);
1205
1206 if (flags & KPD_LOCKED) {
1207 if ((pp = page_find(vp, off)) == NULL) {
1208 if (flags & KPD_LOCKED) {
1209 panic("segkp_softunlock: missing page");
1210 /*NOTREACHED*/
1211 }
1212 }
1213 } else {
1214 /*
1215 * Nothing to do if the slot is not locked and the
1216 * page doesn't exist.
1217 */
1218 if ((pp = page_lookup(vp, off, SE_SHARED)) == NULL)
1219 continue;
1220 }
1221
1222 /*
1223 * If the page doesn't have any translations, is
1224 * dirty and not being shared, then push it out
1225 * asynchronously and avoid waiting for the
1226 * pageout daemon to do it for us.
1227 *
1228 * XXX - Do we really need to get the "exclusive"
1229 * lock via an upgrade?
1230 */
1231 if ((flags & KPD_WRITEDIRTY) && !hat_page_is_mapped(pp) &&
1232 hat_ismod(pp) && page_tryupgrade(pp)) {
1233 /*
1234 * Hold the vnode before releasing the page lock to
1235 * prevent it from being freed and re-used by some
1236 * other thread.
1237 */
1238 VN_HOLD(vp);
1239 page_unlock(pp);
1240
1241 /*
1242 * Want most powerful credentials we can get so
1243 * use kcred.
1244 */
1245 (void) VOP_PUTPAGE(vp, (offset_t)off, PAGESIZE,
1246 B_ASYNC | B_FREE, kcred, NULL);
1247 VN_RELE(vp);
1248 } else {
1249 page_unlock(pp);
1250 }
1251 }
1252
1253 /* If unlocking, release physical memory */
1254 if (flags & KPD_LOCKED) {
1255 pgcnt_t pages = btopr(len);
1256 if ((kpd->kp_flags & KPD_NO_ANON) == 0)
1257 atomic_add_long(&anon_segkp_pages_locked, -pages);
1258 page_unresv(pages);
1259 }
1260 return (0);
1261 }
1262
1263 /*
1264 * Insert the kpd in the hash table.
1265 */
1266 static void
1267 segkp_insert(struct seg *seg, struct segkp_data *kpd)
1268 {
1269 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1270 int index;
1271
1272 /*
1273 * Insert the kpd based on the address that will be returned
1274 * via segkp_release.
1275 */
1276 index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags));
1277 mutex_enter(&segkp_lock);
1278 kpd->kp_next = kpsd->kpsd_hash[index];
1279 kpsd->kpsd_hash[index] = kpd;
1280 mutex_exit(&segkp_lock);
1281 }
1282
1283 /*
1284 * Remove kpd from the hash table.
1285 */
1286 static void
1287 segkp_delete(struct seg *seg, struct segkp_data *kpd)
1288 {
1289 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1290 struct segkp_data **kpp;
1291 int index;
1292
1293 ASSERT(MUTEX_HELD(&segkp_lock));
1294
1295 index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags));
1296 for (kpp = &kpsd->kpsd_hash[index];
1297 *kpp != NULL; kpp = &((*kpp)->kp_next)) {
1298 if (*kpp == kpd) {
1299 *kpp = kpd->kp_next;
1300 return;
1301 }
1302 }
1303 panic("segkp_delete: unable to find element to delete");
1304 /*NOTREACHED*/
1305 }
1306
1307 /*
1308 * Find the kpd associated with a vaddr.
1309 *
1310 * Most of the callers of segkp_find will pass the vaddr that
1311 * hashes to the desired index, but there are cases where
1312 * this is not true in which case we have to (potentially) scan
1313 * the whole table looking for it. This should be very rare
1314 * (e.g. a segkp_fault(F_INVAL) on an address somewhere in the
1315 * middle of the segkp_data region).
1316 */
1317 static struct segkp_data *
1318 segkp_find(struct seg *seg, caddr_t vaddr)
1319 {
1320 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1321 struct segkp_data *kpd;
1322 int i;
1323 int stop;
1324
1325 i = stop = SEGKP_HASH(vaddr);
1326 mutex_enter(&segkp_lock);
1327 do {
1328 for (kpd = kpsd->kpsd_hash[i]; kpd != NULL;
1329 kpd = kpd->kp_next) {
1330 if (vaddr >= kpd->kp_base &&
1331 vaddr < kpd->kp_base + kpd->kp_len) {
1332 mutex_exit(&segkp_lock);
1333 return (kpd);
1334 }
1335 }
1336 if (--i < 0)
1337 i = SEGKP_HASHSZ - 1; /* Wrap */
1338 } while (i != stop);
1339 mutex_exit(&segkp_lock);
1340 return (NULL); /* Not found */
1341 }
1342
1343 /*
1344 * returns size of swappable area.
1345 */
1346 size_t
1347 swapsize(caddr_t v)
1348 {
1349 struct segkp_data *kpd;
1350
1351 if ((kpd = segkp_find(segkp, v)) != NULL)
1352 return (SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags));
1353 else
1354 return (NULL);
1355 }
1356
1357 /*
1358 * Dump out all the active segkp pages
1359 */
1360 static void
1361 segkp_dump(struct seg *seg)
1362 {
1363 int i;
1364 struct segkp_data *kpd;
1365 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data;
1366
1367 for (i = 0; i < SEGKP_HASHSZ; i++) {
1368 for (kpd = kpsd->kpsd_hash[i];
1369 kpd != NULL; kpd = kpd->kp_next) {
1370 pfn_t pfn;
1371 caddr_t addr;
1372 caddr_t eaddr;
1373
1374 addr = kpd->kp_base;
1375 eaddr = addr + kpd->kp_len;
1376 while (addr < eaddr) {
1377 ASSERT(seg->s_as == &kas);
1378 pfn = hat_getpfnum(seg->s_as->a_hat, addr);
1379 if (pfn != PFN_INVALID)
1380 dump_addpage(seg->s_as, addr, pfn);
1381 addr += PAGESIZE;
1382 dump_timeleft = dump_timeout;
1383 }
1384 }
1385 }
1386 }
1387
1388 /*ARGSUSED*/
1389 static int
1390 segkp_pagelock(struct seg *seg, caddr_t addr, size_t len,
1391 struct page ***ppp, enum lock_type type, enum seg_rw rw)
1392 {
1393 return (ENOTSUP);
1394 }
1395
1396 /*ARGSUSED*/
1397 static int
1398 segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
1399 {
1400 return (ENODEV);
1401 }
1402
1403 #include <sys/mem_config.h>
1404
1405 /*ARGSUSED*/
1406 static void
1407 segkp_mem_config_post_add(void *arg, pgcnt_t delta_pages)
1408 {}
1409
1410 /*
1411 * During memory delete, turn off caches so that pages are not held.
1412 * A better solution may be to unlock the pages while they are
1413 * in the cache so that they may be collected naturally.
1414 */
1415
1416 /*ARGSUSED*/
1417 static int
1418 segkp_mem_config_pre_del(void *arg, pgcnt_t delta_pages)
1419 {
1420 atomic_inc_32(&segkp_indel);
1421 segkp_cache_free();
1422 return (0);
1423 }
1424
1425 /*ARGSUSED*/
1426 static void
1427 segkp_mem_config_post_del(void *arg, pgcnt_t delta_pages, int cancelled)
1428 {
1429 atomic_dec_32(&segkp_indel);
1430 }
1431
1432 static kphysm_setup_vector_t segkp_mem_config_vec = {
1433 KPHYSM_SETUP_VECTOR_VERSION,
1434 segkp_mem_config_post_add,
1435 segkp_mem_config_pre_del,
1436 segkp_mem_config_post_del,
1437 };
1438
1439 static void
1440 segkpinit_mem_config(struct seg *seg)
1441 {
1442 int ret;
1443
1444 ret = kphysm_setup_func_register(&segkp_mem_config_vec, (void *)seg);
1445 ASSERT(ret == 0);
1446 }