1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 /* 22 * Copyright (c) 1991, 2010, Oracle and/or its affiliates. All rights reserved. 23 */ 24 25 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */ 26 /* All Rights Reserved */ 27 28 /* 29 * Portions of this source code were derived from Berkeley 4.3 BSD 30 * under license from the Regents of the University of California. 31 */ 32 33 /* 34 * segkp is a segment driver that administers the allocation and deallocation 35 * of pageable variable size chunks of kernel virtual address space. Each 36 * allocated resource is page-aligned. 37 * 38 * The user may specify whether the resource should be initialized to 0, 39 * include a redzone, or locked in memory. 40 */ 41 42 #include <sys/types.h> 43 #include <sys/t_lock.h> 44 #include <sys/thread.h> 45 #include <sys/param.h> 46 #include <sys/errno.h> 47 #include <sys/sysmacros.h> 48 #include <sys/systm.h> 49 #include <sys/buf.h> 50 #include <sys/mman.h> 51 #include <sys/vnode.h> 52 #include <sys/cmn_err.h> 53 #include <sys/swap.h> 54 #include <sys/tuneable.h> 55 #include <sys/kmem.h> 56 #include <sys/vmem.h> 57 #include <sys/cred.h> 58 #include <sys/dumphdr.h> 59 #include <sys/debug.h> 60 #include <sys/vtrace.h> 61 #include <sys/stack.h> 62 #include <sys/atomic.h> 63 #include <sys/archsystm.h> 64 #include <sys/lgrp.h> 65 66 #include <vm/as.h> 67 #include <vm/seg.h> 68 #include <vm/seg_kp.h> 69 #include <vm/seg_kmem.h> 70 #include <vm/anon.h> 71 #include <vm/page.h> 72 #include <vm/hat.h> 73 #include <sys/bitmap.h> 74 75 /* 76 * Private seg op routines 77 */ 78 static void segkp_badop(void); 79 static void segkp_dump(struct seg *seg); 80 static int segkp_checkprot(struct seg *seg, caddr_t addr, size_t len, 81 uint_t prot); 82 static int segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta); 83 static int segkp_pagelock(struct seg *seg, caddr_t addr, size_t len, 84 struct page ***page, enum lock_type type, 85 enum seg_rw rw); 86 static void segkp_insert(struct seg *seg, struct segkp_data *kpd); 87 static void segkp_delete(struct seg *seg, struct segkp_data *kpd); 88 static caddr_t segkp_get_internal(struct seg *seg, size_t len, uint_t flags, 89 struct segkp_data **tkpd, struct anon_map *amp); 90 static void segkp_release_internal(struct seg *seg, 91 struct segkp_data *kpd, size_t len); 92 static int segkp_unlock(struct hat *hat, struct seg *seg, caddr_t vaddr, 93 size_t len, struct segkp_data *kpd, uint_t flags); 94 static int segkp_load(struct hat *hat, struct seg *seg, caddr_t vaddr, 95 size_t len, struct segkp_data *kpd, uint_t flags); 96 static struct segkp_data *segkp_find(struct seg *seg, caddr_t vaddr); 97 static int segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp); 98 99 /* 100 * Lock used to protect the hash table(s) and caches. 101 */ 102 static kmutex_t segkp_lock; 103 104 /* 105 * The segkp caches 106 */ 107 static struct segkp_cache segkp_cache[SEGKP_MAX_CACHE]; 108 109 #define SEGKP_BADOP(t) (t(*)())segkp_badop 110 111 /* 112 * When there are fewer than red_minavail bytes left on the stack, 113 * segkp_map_red() will map in the redzone (if called). 5000 seems 114 * to work reasonably well... 115 */ 116 long red_minavail = 5000; 117 118 /* 119 * will be set to 1 for 32 bit x86 systems only, in startup.c 120 */ 121 int segkp_fromheap = 0; 122 ulong_t *segkp_bitmap; 123 124 /* 125 * If segkp_map_red() is called with the redzone already mapped and 126 * with less than RED_DEEP_THRESHOLD bytes available on the stack, 127 * then the stack situation has become quite serious; if much more stack 128 * is consumed, we have the potential of scrogging the next thread/LWP 129 * structure. To help debug the "can't happen" panics which may 130 * result from this condition, we record hrestime and the calling thread 131 * in red_deep_hires and red_deep_thread respectively. 132 */ 133 #define RED_DEEP_THRESHOLD 2000 134 135 hrtime_t red_deep_hires; 136 kthread_t *red_deep_thread; 137 138 uint32_t red_nmapped; 139 uint32_t red_closest = UINT_MAX; 140 uint32_t red_ndoubles; 141 142 pgcnt_t anon_segkp_pages_locked; /* See vm/anon.h */ 143 pgcnt_t anon_segkp_pages_resv; /* anon reserved by seg_kp */ 144 145 static struct seg_ops segkp_ops = { 146 .dup = SEGKP_BADOP(int), 147 .unmap = SEGKP_BADOP(int), 148 .free = SEGKP_BADOP(void), 149 .fault = segkp_fault, 150 .faulta = SEGKP_BADOP(faultcode_t), 151 .setprot = SEGKP_BADOP(int), 152 .checkprot = segkp_checkprot, 153 .kluster = segkp_kluster, 154 .swapout = SEGKP_BADOP(size_t), 155 .sync = SEGKP_BADOP(int), 156 .incore = SEGKP_BADOP(size_t), 157 .lockop = SEGKP_BADOP(int), 158 .getprot = SEGKP_BADOP(int), 159 .getoffset = SEGKP_BADOP(u_offset_t), 160 .gettype = SEGKP_BADOP(int), 161 .getvp = SEGKP_BADOP(int), 162 .advise = SEGKP_BADOP(int), 163 .dump = segkp_dump, 164 .pagelock = segkp_pagelock, 165 .setpagesize = SEGKP_BADOP(int), 166 .getmemid = segkp_getmemid, 167 }; 168 169 170 static void 171 segkp_badop(void) 172 { 173 panic("segkp_badop"); 174 /*NOTREACHED*/ 175 } 176 177 static void segkpinit_mem_config(struct seg *); 178 179 static uint32_t segkp_indel; 180 181 /* 182 * Allocate the segment specific private data struct and fill it in 183 * with the per kp segment mutex, anon ptr. array and hash table. 184 */ 185 int 186 segkp_create(struct seg *seg) 187 { 188 struct segkp_segdata *kpsd; 189 size_t np; 190 191 ASSERT(seg != NULL && seg->s_as == &kas); 192 ASSERT(RW_WRITE_HELD(&seg->s_as->a_lock)); 193 194 if (seg->s_size & PAGEOFFSET) { 195 panic("Bad segkp size"); 196 /*NOTREACHED*/ 197 } 198 199 kpsd = kmem_zalloc(sizeof (struct segkp_segdata), KM_SLEEP); 200 201 /* 202 * Allocate the virtual memory for segkp and initialize it 203 */ 204 if (segkp_fromheap) { 205 np = btop(kvseg.s_size); 206 segkp_bitmap = kmem_zalloc(BT_SIZEOFMAP(np), KM_SLEEP); 207 kpsd->kpsd_arena = vmem_create("segkp", NULL, 0, PAGESIZE, 208 vmem_alloc, vmem_free, heap_arena, 5 * PAGESIZE, VM_SLEEP); 209 } else { 210 segkp_bitmap = NULL; 211 np = btop(seg->s_size); 212 kpsd->kpsd_arena = vmem_create("segkp", seg->s_base, 213 seg->s_size, PAGESIZE, NULL, NULL, NULL, 5 * PAGESIZE, 214 VM_SLEEP); 215 } 216 217 kpsd->kpsd_anon = anon_create(np, ANON_SLEEP | ANON_ALLOC_FORCE); 218 219 kpsd->kpsd_hash = kmem_zalloc(SEGKP_HASHSZ * sizeof (struct segkp *), 220 KM_SLEEP); 221 seg->s_data = (void *)kpsd; 222 seg->s_ops = &segkp_ops; 223 segkpinit_mem_config(seg); 224 return (0); 225 } 226 227 228 /* 229 * Find a free 'freelist' and initialize it with the appropriate attributes 230 */ 231 void * 232 segkp_cache_init(struct seg *seg, int maxsize, size_t len, uint_t flags) 233 { 234 int i; 235 236 if ((flags & KPD_NO_ANON) && !(flags & KPD_LOCKED)) 237 return ((void *)-1); 238 239 mutex_enter(&segkp_lock); 240 for (i = 0; i < SEGKP_MAX_CACHE; i++) { 241 if (segkp_cache[i].kpf_inuse) 242 continue; 243 segkp_cache[i].kpf_inuse = 1; 244 segkp_cache[i].kpf_max = maxsize; 245 segkp_cache[i].kpf_flags = flags; 246 segkp_cache[i].kpf_seg = seg; 247 segkp_cache[i].kpf_len = len; 248 mutex_exit(&segkp_lock); 249 return ((void *)(uintptr_t)i); 250 } 251 mutex_exit(&segkp_lock); 252 return ((void *)-1); 253 } 254 255 /* 256 * Free all the cache resources. 257 */ 258 void 259 segkp_cache_free(void) 260 { 261 struct segkp_data *kpd; 262 struct seg *seg; 263 int i; 264 265 mutex_enter(&segkp_lock); 266 for (i = 0; i < SEGKP_MAX_CACHE; i++) { 267 if (!segkp_cache[i].kpf_inuse) 268 continue; 269 /* 270 * Disconnect the freelist and process each element 271 */ 272 kpd = segkp_cache[i].kpf_list; 273 seg = segkp_cache[i].kpf_seg; 274 segkp_cache[i].kpf_list = NULL; 275 segkp_cache[i].kpf_count = 0; 276 mutex_exit(&segkp_lock); 277 278 while (kpd != NULL) { 279 struct segkp_data *next; 280 281 next = kpd->kp_next; 282 segkp_release_internal(seg, kpd, kpd->kp_len); 283 kpd = next; 284 } 285 mutex_enter(&segkp_lock); 286 } 287 mutex_exit(&segkp_lock); 288 } 289 290 /* 291 * There are 2 entries into segkp_get_internal. The first includes a cookie 292 * used to access a pool of cached segkp resources. The second does not 293 * use the cache. 294 */ 295 caddr_t 296 segkp_get(struct seg *seg, size_t len, uint_t flags) 297 { 298 struct segkp_data *kpd = NULL; 299 300 if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) { 301 kpd->kp_cookie = -1; 302 return (stom(kpd->kp_base, flags)); 303 } 304 return (NULL); 305 } 306 307 /* 308 * Return a 'cached' segkp address 309 */ 310 caddr_t 311 segkp_cache_get(void *cookie) 312 { 313 struct segkp_cache *freelist = NULL; 314 struct segkp_data *kpd = NULL; 315 int index = (int)(uintptr_t)cookie; 316 struct seg *seg; 317 size_t len; 318 uint_t flags; 319 320 if (index < 0 || index >= SEGKP_MAX_CACHE) 321 return (NULL); 322 freelist = &segkp_cache[index]; 323 324 mutex_enter(&segkp_lock); 325 seg = freelist->kpf_seg; 326 flags = freelist->kpf_flags; 327 if (freelist->kpf_list != NULL) { 328 kpd = freelist->kpf_list; 329 freelist->kpf_list = kpd->kp_next; 330 freelist->kpf_count--; 331 mutex_exit(&segkp_lock); 332 kpd->kp_next = NULL; 333 segkp_insert(seg, kpd); 334 return (stom(kpd->kp_base, flags)); 335 } 336 len = freelist->kpf_len; 337 mutex_exit(&segkp_lock); 338 if (segkp_get_internal(seg, len, flags, &kpd, NULL) != NULL) { 339 kpd->kp_cookie = index; 340 return (stom(kpd->kp_base, flags)); 341 } 342 return (NULL); 343 } 344 345 caddr_t 346 segkp_get_withanonmap( 347 struct seg *seg, 348 size_t len, 349 uint_t flags, 350 struct anon_map *amp) 351 { 352 struct segkp_data *kpd = NULL; 353 354 ASSERT(amp != NULL); 355 flags |= KPD_HASAMP; 356 if (segkp_get_internal(seg, len, flags, &kpd, amp) != NULL) { 357 kpd->kp_cookie = -1; 358 return (stom(kpd->kp_base, flags)); 359 } 360 return (NULL); 361 } 362 363 /* 364 * This does the real work of segkp allocation. 365 * Return to client base addr. len must be page-aligned. A null value is 366 * returned if there are no more vm resources (e.g. pages, swap). The len 367 * and base recorded in the private data structure include the redzone 368 * and the redzone length (if applicable). If the user requests a redzone 369 * either the first or last page is left unmapped depending whether stacks 370 * grow to low or high memory. 371 * 372 * The client may also specify a no-wait flag. If that is set then the 373 * request will choose a non-blocking path when requesting resources. 374 * The default is make the client wait. 375 */ 376 static caddr_t 377 segkp_get_internal( 378 struct seg *seg, 379 size_t len, 380 uint_t flags, 381 struct segkp_data **tkpd, 382 struct anon_map *amp) 383 { 384 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 385 struct segkp_data *kpd; 386 caddr_t vbase = NULL; /* always first virtual, may not be mapped */ 387 pgcnt_t np = 0; /* number of pages in the resource */ 388 pgcnt_t segkpindex; 389 long i; 390 caddr_t va; 391 pgcnt_t pages = 0; 392 ulong_t anon_idx = 0; 393 int kmflag = (flags & KPD_NOWAIT) ? KM_NOSLEEP : KM_SLEEP; 394 caddr_t s_base = (segkp_fromheap) ? kvseg.s_base : seg->s_base; 395 396 if (len & PAGEOFFSET) { 397 panic("segkp_get: len is not page-aligned"); 398 /*NOTREACHED*/ 399 } 400 401 ASSERT(((flags & KPD_HASAMP) == 0) == (amp == NULL)); 402 403 /* Only allow KPD_NO_ANON if we are going to lock it down */ 404 if ((flags & (KPD_LOCKED|KPD_NO_ANON)) == KPD_NO_ANON) 405 return (NULL); 406 407 if ((kpd = kmem_zalloc(sizeof (struct segkp_data), kmflag)) == NULL) 408 return (NULL); 409 /* 410 * Fix up the len to reflect the REDZONE if applicable 411 */ 412 if (flags & KPD_HASREDZONE) 413 len += PAGESIZE; 414 np = btop(len); 415 416 vbase = vmem_alloc(SEGKP_VMEM(seg), len, kmflag | VM_BESTFIT); 417 if (vbase == NULL) { 418 kmem_free(kpd, sizeof (struct segkp_data)); 419 return (NULL); 420 } 421 422 /* If locking, reserve physical memory */ 423 if (flags & KPD_LOCKED) { 424 pages = btop(SEGKP_MAPLEN(len, flags)); 425 if (page_resv(pages, kmflag) == 0) { 426 vmem_free(SEGKP_VMEM(seg), vbase, len); 427 kmem_free(kpd, sizeof (struct segkp_data)); 428 return (NULL); 429 } 430 if ((flags & KPD_NO_ANON) == 0) 431 atomic_add_long(&anon_segkp_pages_locked, pages); 432 } 433 434 /* 435 * Reserve sufficient swap space for this vm resource. We'll 436 * actually allocate it in the loop below, but reserving it 437 * here allows us to back out more gracefully than if we 438 * had an allocation failure in the body of the loop. 439 * 440 * Note that we don't need swap space for the red zone page. 441 */ 442 if (amp != NULL) { 443 /* 444 * The swap reservation has been done, if required, and the 445 * anon_hdr is separate. 446 */ 447 anon_idx = 0; 448 kpd->kp_anon_idx = anon_idx; 449 kpd->kp_anon = amp->ahp; 450 451 TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u", 452 kpd, vbase, len, flags, 1); 453 454 } else if ((flags & KPD_NO_ANON) == 0) { 455 if (anon_resv_zone(SEGKP_MAPLEN(len, flags), NULL) == 0) { 456 if (flags & KPD_LOCKED) { 457 atomic_add_long(&anon_segkp_pages_locked, 458 -pages); 459 page_unresv(pages); 460 } 461 vmem_free(SEGKP_VMEM(seg), vbase, len); 462 kmem_free(kpd, sizeof (struct segkp_data)); 463 return (NULL); 464 } 465 atomic_add_long(&anon_segkp_pages_resv, 466 btop(SEGKP_MAPLEN(len, flags))); 467 anon_idx = ((uintptr_t)(vbase - s_base)) >> PAGESHIFT; 468 kpd->kp_anon_idx = anon_idx; 469 kpd->kp_anon = kpsd->kpsd_anon; 470 471 TRACE_5(TR_FAC_VM, TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u", 472 kpd, vbase, len, flags, 1); 473 } else { 474 kpd->kp_anon = NULL; 475 kpd->kp_anon_idx = 0; 476 } 477 478 /* 479 * Allocate page and anon resources for the virtual address range 480 * except the redzone 481 */ 482 if (segkp_fromheap) 483 segkpindex = btop((uintptr_t)(vbase - kvseg.s_base)); 484 for (i = 0, va = vbase; i < np; i++, va += PAGESIZE) { 485 page_t *pl[2]; 486 struct vnode *vp; 487 anoff_t off; 488 int err; 489 page_t *pp = NULL; 490 491 /* 492 * Mark this page to be a segkp page in the bitmap. 493 */ 494 if (segkp_fromheap) { 495 BT_ATOMIC_SET(segkp_bitmap, segkpindex); 496 segkpindex++; 497 } 498 499 /* 500 * If this page is the red zone page, we don't need swap 501 * space for it. Note that we skip over the code that 502 * establishes MMU mappings, so that the page remains 503 * invalid. 504 */ 505 if ((flags & KPD_HASREDZONE) && KPD_REDZONE(kpd) == i) 506 continue; 507 508 if (kpd->kp_anon != NULL) { 509 struct anon *ap; 510 511 ASSERT(anon_get_ptr(kpd->kp_anon, anon_idx + i) 512 == NULL); 513 /* 514 * Determine the "vp" and "off" of the anon slot. 515 */ 516 ap = anon_alloc(NULL, 0); 517 if (amp != NULL) 518 ANON_LOCK_ENTER(&->a_rwlock, RW_WRITER); 519 (void) anon_set_ptr(kpd->kp_anon, anon_idx + i, 520 ap, ANON_SLEEP); 521 if (amp != NULL) 522 ANON_LOCK_EXIT(&->a_rwlock); 523 swap_xlate(ap, &vp, &off); 524 525 /* 526 * Create a page with the specified identity. The 527 * page is returned with the "shared" lock held. 528 */ 529 err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE, 530 NULL, pl, PAGESIZE, seg, va, S_CREATE, 531 kcred, NULL); 532 if (err) { 533 /* 534 * XXX - This should not fail. 535 */ 536 panic("segkp_get: no pages"); 537 /*NOTREACHED*/ 538 } 539 pp = pl[0]; 540 } else { 541 ASSERT(page_exists(&kvp, 542 (u_offset_t)(uintptr_t)va) == NULL); 543 544 if ((pp = page_create_va(&kvp, 545 (u_offset_t)(uintptr_t)va, PAGESIZE, 546 (flags & KPD_NOWAIT ? 0 : PG_WAIT) | PG_EXCL | 547 PG_NORELOC, seg, va)) == NULL) { 548 /* 549 * Legitimize resource; then destroy it. 550 * Easier than trying to unwind here. 551 */ 552 kpd->kp_flags = flags; 553 kpd->kp_base = vbase; 554 kpd->kp_len = len; 555 segkp_release_internal(seg, kpd, va - vbase); 556 return (NULL); 557 } 558 page_io_unlock(pp); 559 } 560 561 if (flags & KPD_ZERO) 562 pagezero(pp, 0, PAGESIZE); 563 564 /* 565 * Load and lock an MMU translation for the page. 566 */ 567 hat_memload(seg->s_as->a_hat, va, pp, (PROT_READ|PROT_WRITE), 568 ((flags & KPD_LOCKED) ? HAT_LOAD_LOCK : HAT_LOAD)); 569 570 /* 571 * Now, release lock on the page. 572 */ 573 if (flags & KPD_LOCKED) { 574 /* 575 * Indicate to page_retire framework that this 576 * page can only be retired when it is freed. 577 */ 578 PP_SETRAF(pp); 579 page_downgrade(pp); 580 } else 581 page_unlock(pp); 582 } 583 584 kpd->kp_flags = flags; 585 kpd->kp_base = vbase; 586 kpd->kp_len = len; 587 segkp_insert(seg, kpd); 588 *tkpd = kpd; 589 return (stom(kpd->kp_base, flags)); 590 } 591 592 /* 593 * Release the resource to cache if the pool(designate by the cookie) 594 * has less than the maximum allowable. If inserted in cache, 595 * segkp_delete insures element is taken off of active list. 596 */ 597 void 598 segkp_release(struct seg *seg, caddr_t vaddr) 599 { 600 struct segkp_cache *freelist; 601 struct segkp_data *kpd = NULL; 602 603 if ((kpd = segkp_find(seg, vaddr)) == NULL) { 604 panic("segkp_release: null kpd"); 605 /*NOTREACHED*/ 606 } 607 608 if (kpd->kp_cookie != -1) { 609 freelist = &segkp_cache[kpd->kp_cookie]; 610 mutex_enter(&segkp_lock); 611 if (!segkp_indel && freelist->kpf_count < freelist->kpf_max) { 612 segkp_delete(seg, kpd); 613 kpd->kp_next = freelist->kpf_list; 614 freelist->kpf_list = kpd; 615 freelist->kpf_count++; 616 mutex_exit(&segkp_lock); 617 return; 618 } else { 619 mutex_exit(&segkp_lock); 620 kpd->kp_cookie = -1; 621 } 622 } 623 segkp_release_internal(seg, kpd, kpd->kp_len); 624 } 625 626 /* 627 * Free the entire resource. segkp_unlock gets called with the start of the 628 * mapped portion of the resource. The length is the size of the mapped 629 * portion 630 */ 631 static void 632 segkp_release_internal(struct seg *seg, struct segkp_data *kpd, size_t len) 633 { 634 caddr_t va; 635 long i; 636 long redzone; 637 size_t np; 638 page_t *pp; 639 struct vnode *vp; 640 anoff_t off; 641 struct anon *ap; 642 pgcnt_t segkpindex; 643 644 ASSERT(kpd != NULL); 645 ASSERT((kpd->kp_flags & KPD_HASAMP) == 0 || kpd->kp_cookie == -1); 646 np = btop(len); 647 648 /* Remove from active hash list */ 649 if (kpd->kp_cookie == -1) { 650 mutex_enter(&segkp_lock); 651 segkp_delete(seg, kpd); 652 mutex_exit(&segkp_lock); 653 } 654 655 /* 656 * Precompute redzone page index. 657 */ 658 redzone = -1; 659 if (kpd->kp_flags & KPD_HASREDZONE) 660 redzone = KPD_REDZONE(kpd); 661 662 663 va = kpd->kp_base; 664 665 hat_unload(seg->s_as->a_hat, va, (np << PAGESHIFT), 666 ((kpd->kp_flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD)); 667 /* 668 * Free up those anon resources that are quiescent. 669 */ 670 if (segkp_fromheap) 671 segkpindex = btop((uintptr_t)(va - kvseg.s_base)); 672 for (i = 0; i < np; i++, va += PAGESIZE) { 673 674 /* 675 * Clear the bit for this page from the bitmap. 676 */ 677 if (segkp_fromheap) { 678 BT_ATOMIC_CLEAR(segkp_bitmap, segkpindex); 679 segkpindex++; 680 } 681 682 if (i == redzone) 683 continue; 684 if (kpd->kp_anon) { 685 /* 686 * Free up anon resources and destroy the 687 * associated pages. 688 * 689 * Release the lock if there is one. Have to get the 690 * page to do this, unfortunately. 691 */ 692 if (kpd->kp_flags & KPD_LOCKED) { 693 ap = anon_get_ptr(kpd->kp_anon, 694 kpd->kp_anon_idx + i); 695 swap_xlate(ap, &vp, &off); 696 /* Find the shared-locked page. */ 697 pp = page_find(vp, (u_offset_t)off); 698 if (pp == NULL) { 699 panic("segkp_release: " 700 "kp_anon: no page to unlock "); 701 /*NOTREACHED*/ 702 } 703 if (PP_ISRAF(pp)) 704 PP_CLRRAF(pp); 705 706 page_unlock(pp); 707 } 708 if ((kpd->kp_flags & KPD_HASAMP) == 0) { 709 anon_free(kpd->kp_anon, kpd->kp_anon_idx + i, 710 PAGESIZE); 711 anon_unresv_zone(PAGESIZE, NULL); 712 atomic_dec_ulong(&anon_segkp_pages_resv); 713 } 714 TRACE_5(TR_FAC_VM, 715 TR_ANON_SEGKP, "anon segkp:%p %p %lu %u %u", 716 kpd, va, PAGESIZE, 0, 0); 717 } else { 718 if (kpd->kp_flags & KPD_LOCKED) { 719 pp = page_find(&kvp, (u_offset_t)(uintptr_t)va); 720 if (pp == NULL) { 721 panic("segkp_release: " 722 "no page to unlock"); 723 /*NOTREACHED*/ 724 } 725 if (PP_ISRAF(pp)) 726 PP_CLRRAF(pp); 727 /* 728 * We should just upgrade the lock here 729 * but there is no upgrade that waits. 730 */ 731 page_unlock(pp); 732 } 733 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)va, 734 SE_EXCL); 735 if (pp != NULL) 736 page_destroy(pp, 0); 737 } 738 } 739 740 /* If locked, release physical memory reservation */ 741 if (kpd->kp_flags & KPD_LOCKED) { 742 pgcnt_t pages = btop(SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)); 743 if ((kpd->kp_flags & KPD_NO_ANON) == 0) 744 atomic_add_long(&anon_segkp_pages_locked, -pages); 745 page_unresv(pages); 746 } 747 748 vmem_free(SEGKP_VMEM(seg), kpd->kp_base, kpd->kp_len); 749 kmem_free(kpd, sizeof (struct segkp_data)); 750 } 751 752 /* 753 * segkp_map_red() will check the current frame pointer against the 754 * stack base. If the amount of stack remaining is questionable 755 * (less than red_minavail), then segkp_map_red() will map in the redzone 756 * and return 1. Otherwise, it will return 0. segkp_map_red() can 757 * _only_ be called when: 758 * 759 * - it is safe to sleep on page_create_va(). 760 * - the caller is non-swappable. 761 * 762 * It is up to the caller to remember whether segkp_map_red() successfully 763 * mapped the redzone, and, if so, to call segkp_unmap_red() at a later 764 * time. Note that the caller must _remain_ non-swappable until after 765 * calling segkp_unmap_red(). 766 * 767 * Currently, this routine is only called from pagefault() (which necessarily 768 * satisfies the above conditions). 769 */ 770 #if defined(STACK_GROWTH_DOWN) 771 int 772 segkp_map_red(void) 773 { 774 uintptr_t fp = STACK_BIAS + (uintptr_t)getfp(); 775 #ifndef _LP64 776 caddr_t stkbase; 777 #endif 778 779 ASSERT(curthread->t_schedflag & TS_DONT_SWAP); 780 781 /* 782 * Optimize for the common case where we simply return. 783 */ 784 if ((curthread->t_red_pp == NULL) && 785 (fp - (uintptr_t)curthread->t_stkbase >= red_minavail)) 786 return (0); 787 788 #if defined(_LP64) 789 /* 790 * XXX We probably need something better than this. 791 */ 792 panic("kernel stack overflow"); 793 /*NOTREACHED*/ 794 #else /* _LP64 */ 795 if (curthread->t_red_pp == NULL) { 796 page_t *red_pp; 797 struct seg kseg; 798 799 caddr_t red_va = (caddr_t) 800 (((uintptr_t)curthread->t_stkbase & (uintptr_t)PAGEMASK) - 801 PAGESIZE); 802 803 ASSERT(page_exists(&kvp, (u_offset_t)(uintptr_t)red_va) == 804 NULL); 805 806 /* 807 * Allocate the physical for the red page. 808 */ 809 /* 810 * No PG_NORELOC here to avoid waits. Unlikely to get 811 * a relocate happening in the short time the page exists 812 * and it will be OK anyway. 813 */ 814 815 kseg.s_as = &kas; 816 red_pp = page_create_va(&kvp, (u_offset_t)(uintptr_t)red_va, 817 PAGESIZE, PG_WAIT | PG_EXCL, &kseg, red_va); 818 ASSERT(red_pp != NULL); 819 820 /* 821 * So we now have a page to jam into the redzone... 822 */ 823 page_io_unlock(red_pp); 824 825 hat_memload(kas.a_hat, red_va, red_pp, 826 (PROT_READ|PROT_WRITE), HAT_LOAD_LOCK); 827 page_downgrade(red_pp); 828 829 /* 830 * The page is left SE_SHARED locked so we can hold on to 831 * the page_t pointer. 832 */ 833 curthread->t_red_pp = red_pp; 834 835 atomic_inc_32(&red_nmapped); 836 while (fp - (uintptr_t)curthread->t_stkbase < red_closest) { 837 (void) atomic_cas_32(&red_closest, red_closest, 838 (uint32_t)(fp - (uintptr_t)curthread->t_stkbase)); 839 } 840 return (1); 841 } 842 843 stkbase = (caddr_t)(((uintptr_t)curthread->t_stkbase & 844 (uintptr_t)PAGEMASK) - PAGESIZE); 845 846 atomic_inc_32(&red_ndoubles); 847 848 if (fp - (uintptr_t)stkbase < RED_DEEP_THRESHOLD) { 849 /* 850 * Oh boy. We're already deep within the mapped-in 851 * redzone page, and the caller is trying to prepare 852 * for a deep stack run. We're running without a 853 * redzone right now: if the caller plows off the 854 * end of the stack, it'll plow another thread or 855 * LWP structure. That situation could result in 856 * a very hard-to-debug panic, so, in the spirit of 857 * recording the name of one's killer in one's own 858 * blood, we're going to record hrestime and the calling 859 * thread. 860 */ 861 red_deep_hires = hrestime.tv_nsec; 862 red_deep_thread = curthread; 863 } 864 865 /* 866 * If this is a DEBUG kernel, and we've run too deep for comfort, toss. 867 */ 868 ASSERT(fp - (uintptr_t)stkbase >= RED_DEEP_THRESHOLD); 869 return (0); 870 #endif /* _LP64 */ 871 } 872 873 void 874 segkp_unmap_red(void) 875 { 876 page_t *pp; 877 caddr_t red_va = (caddr_t)(((uintptr_t)curthread->t_stkbase & 878 (uintptr_t)PAGEMASK) - PAGESIZE); 879 880 ASSERT(curthread->t_red_pp != NULL); 881 ASSERT(curthread->t_schedflag & TS_DONT_SWAP); 882 883 /* 884 * Because we locked the mapping down, we can't simply rely 885 * on page_destroy() to clean everything up; we need to call 886 * hat_unload() to explicitly unlock the mapping resources. 887 */ 888 hat_unload(kas.a_hat, red_va, PAGESIZE, HAT_UNLOAD_UNLOCK); 889 890 pp = curthread->t_red_pp; 891 892 ASSERT(pp == page_find(&kvp, (u_offset_t)(uintptr_t)red_va)); 893 894 /* 895 * Need to upgrade the SE_SHARED lock to SE_EXCL. 896 */ 897 if (!page_tryupgrade(pp)) { 898 /* 899 * As there is now wait for upgrade, release the 900 * SE_SHARED lock and wait for SE_EXCL. 901 */ 902 page_unlock(pp); 903 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)red_va, SE_EXCL); 904 /* pp may be NULL here, hence the test below */ 905 } 906 907 /* 908 * Destroy the page, with dontfree set to zero (i.e. free it). 909 */ 910 if (pp != NULL) 911 page_destroy(pp, 0); 912 curthread->t_red_pp = NULL; 913 } 914 #else 915 #error Red stacks only supported with downwards stack growth. 916 #endif 917 918 /* 919 * Handle a fault on an address corresponding to one of the 920 * resources in the segkp segment. 921 */ 922 faultcode_t 923 segkp_fault( 924 struct hat *hat, 925 struct seg *seg, 926 caddr_t vaddr, 927 size_t len, 928 enum fault_type type, 929 enum seg_rw rw) 930 { 931 struct segkp_data *kpd = NULL; 932 int err; 933 934 ASSERT(seg->s_as == &kas && RW_READ_HELD(&seg->s_as->a_lock)); 935 936 /* 937 * Sanity checks. 938 */ 939 if (type == F_PROT) { 940 panic("segkp_fault: unexpected F_PROT fault"); 941 /*NOTREACHED*/ 942 } 943 944 if ((kpd = segkp_find(seg, vaddr)) == NULL) 945 return (FC_NOMAP); 946 947 mutex_enter(&kpd->kp_lock); 948 949 if (type == F_SOFTLOCK) { 950 ASSERT(!(kpd->kp_flags & KPD_LOCKED)); 951 /* 952 * The F_SOFTLOCK case has more stringent 953 * range requirements: the given range must exactly coincide 954 * with the resource's mapped portion. Note reference to 955 * redzone is handled since vaddr would not equal base 956 */ 957 if (vaddr != stom(kpd->kp_base, kpd->kp_flags) || 958 len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) { 959 mutex_exit(&kpd->kp_lock); 960 return (FC_MAKE_ERR(EFAULT)); 961 } 962 963 if ((err = segkp_load(hat, seg, vaddr, len, kpd, KPD_LOCKED))) { 964 mutex_exit(&kpd->kp_lock); 965 return (FC_MAKE_ERR(err)); 966 } 967 kpd->kp_flags |= KPD_LOCKED; 968 mutex_exit(&kpd->kp_lock); 969 return (0); 970 } 971 972 if (type == F_INVAL) { 973 ASSERT(!(kpd->kp_flags & KPD_NO_ANON)); 974 975 /* 976 * Check if we touched the redzone. Somewhat optimistic 977 * here if we are touching the redzone of our own stack 978 * since we wouldn't have a stack to get this far... 979 */ 980 if ((kpd->kp_flags & KPD_HASREDZONE) && 981 btop((uintptr_t)(vaddr - kpd->kp_base)) == KPD_REDZONE(kpd)) 982 panic("segkp_fault: accessing redzone"); 983 984 /* 985 * This fault may occur while the page is being F_SOFTLOCK'ed. 986 * Return since a 2nd segkp_load is unnecessary and also would 987 * result in the page being locked twice and eventually 988 * hang the thread_reaper thread. 989 */ 990 if (kpd->kp_flags & KPD_LOCKED) { 991 mutex_exit(&kpd->kp_lock); 992 return (0); 993 } 994 995 err = segkp_load(hat, seg, vaddr, len, kpd, kpd->kp_flags); 996 mutex_exit(&kpd->kp_lock); 997 return (err ? FC_MAKE_ERR(err) : 0); 998 } 999 1000 if (type == F_SOFTUNLOCK) { 1001 uint_t flags; 1002 1003 /* 1004 * Make sure the addr is LOCKED and it has anon backing 1005 * before unlocking 1006 */ 1007 if ((kpd->kp_flags & (KPD_LOCKED|KPD_NO_ANON)) != KPD_LOCKED) { 1008 panic("segkp_fault: bad unlock"); 1009 /*NOTREACHED*/ 1010 } 1011 1012 if (vaddr != stom(kpd->kp_base, kpd->kp_flags) || 1013 len != SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)) { 1014 panic("segkp_fault: bad range"); 1015 /*NOTREACHED*/ 1016 } 1017 1018 if (rw == S_WRITE) 1019 flags = kpd->kp_flags | KPD_WRITEDIRTY; 1020 else 1021 flags = kpd->kp_flags; 1022 err = segkp_unlock(hat, seg, vaddr, len, kpd, flags); 1023 kpd->kp_flags &= ~KPD_LOCKED; 1024 mutex_exit(&kpd->kp_lock); 1025 return (err ? FC_MAKE_ERR(err) : 0); 1026 } 1027 mutex_exit(&kpd->kp_lock); 1028 panic("segkp_fault: bogus fault type: %d\n", type); 1029 /*NOTREACHED*/ 1030 } 1031 1032 /* 1033 * Check that the given protections suffice over the range specified by 1034 * vaddr and len. For this segment type, the only issue is whether or 1035 * not the range lies completely within the mapped part of an allocated 1036 * resource. 1037 */ 1038 /* ARGSUSED */ 1039 static int 1040 segkp_checkprot(struct seg *seg, caddr_t vaddr, size_t len, uint_t prot) 1041 { 1042 struct segkp_data *kpd = NULL; 1043 caddr_t mbase; 1044 size_t mlen; 1045 1046 if ((kpd = segkp_find(seg, vaddr)) == NULL) 1047 return (EACCES); 1048 1049 mutex_enter(&kpd->kp_lock); 1050 mbase = stom(kpd->kp_base, kpd->kp_flags); 1051 mlen = SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags); 1052 if (len > mlen || vaddr < mbase || 1053 ((vaddr + len) > (mbase + mlen))) { 1054 mutex_exit(&kpd->kp_lock); 1055 return (EACCES); 1056 } 1057 mutex_exit(&kpd->kp_lock); 1058 return (0); 1059 } 1060 1061 1062 /* 1063 * Check to see if it makes sense to do kluster/read ahead to 1064 * addr + delta relative to the mapping at addr. We assume here 1065 * that delta is a signed PAGESIZE'd multiple (which can be negative). 1066 * 1067 * For seg_u we always "approve" of this action from our standpoint. 1068 */ 1069 /*ARGSUSED*/ 1070 static int 1071 segkp_kluster(struct seg *seg, caddr_t addr, ssize_t delta) 1072 { 1073 return (0); 1074 } 1075 1076 /* 1077 * Load and possibly lock intra-slot resources in the range given by 1078 * vaddr and len. 1079 */ 1080 static int 1081 segkp_load( 1082 struct hat *hat, 1083 struct seg *seg, 1084 caddr_t vaddr, 1085 size_t len, 1086 struct segkp_data *kpd, 1087 uint_t flags) 1088 { 1089 caddr_t va; 1090 caddr_t vlim; 1091 ulong_t i; 1092 uint_t lock; 1093 1094 ASSERT(MUTEX_HELD(&kpd->kp_lock)); 1095 1096 len = P2ROUNDUP(len, PAGESIZE); 1097 1098 /* If locking, reserve physical memory */ 1099 if (flags & KPD_LOCKED) { 1100 pgcnt_t pages = btop(len); 1101 if ((kpd->kp_flags & KPD_NO_ANON) == 0) 1102 atomic_add_long(&anon_segkp_pages_locked, pages); 1103 (void) page_resv(pages, KM_SLEEP); 1104 } 1105 1106 /* 1107 * Loop through the pages in the given range. 1108 */ 1109 va = (caddr_t)((uintptr_t)vaddr & (uintptr_t)PAGEMASK); 1110 vaddr = va; 1111 vlim = va + len; 1112 lock = flags & KPD_LOCKED; 1113 i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT; 1114 for (; va < vlim; va += PAGESIZE, i++) { 1115 page_t *pl[2]; /* second element NULL terminator */ 1116 struct vnode *vp; 1117 anoff_t off; 1118 int err; 1119 struct anon *ap; 1120 1121 /* 1122 * Summon the page. If it's not resident, arrange 1123 * for synchronous i/o to pull it in. 1124 */ 1125 ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i); 1126 swap_xlate(ap, &vp, &off); 1127 1128 /* 1129 * The returned page list will have exactly one entry, 1130 * which is returned to us already kept. 1131 */ 1132 err = VOP_GETPAGE(vp, (offset_t)off, PAGESIZE, NULL, 1133 pl, PAGESIZE, seg, va, S_READ, kcred, NULL); 1134 1135 if (err) { 1136 /* 1137 * Back out of what we've done so far. 1138 */ 1139 (void) segkp_unlock(hat, seg, vaddr, 1140 (va - vaddr), kpd, flags); 1141 return (err); 1142 } 1143 1144 /* 1145 * Load an MMU translation for the page. 1146 */ 1147 hat_memload(hat, va, pl[0], (PROT_READ|PROT_WRITE), 1148 lock ? HAT_LOAD_LOCK : HAT_LOAD); 1149 1150 if (!lock) { 1151 /* 1152 * Now, release "shared" lock on the page. 1153 */ 1154 page_unlock(pl[0]); 1155 } 1156 } 1157 return (0); 1158 } 1159 1160 /* 1161 * At the very least unload the mmu-translations and unlock the range if locked 1162 * Can be called with the following flag value KPD_WRITEDIRTY which specifies 1163 * any dirty pages should be written to disk. 1164 */ 1165 static int 1166 segkp_unlock( 1167 struct hat *hat, 1168 struct seg *seg, 1169 caddr_t vaddr, 1170 size_t len, 1171 struct segkp_data *kpd, 1172 uint_t flags) 1173 { 1174 caddr_t va; 1175 caddr_t vlim; 1176 ulong_t i; 1177 struct page *pp; 1178 struct vnode *vp; 1179 anoff_t off; 1180 struct anon *ap; 1181 1182 #ifdef lint 1183 seg = seg; 1184 #endif /* lint */ 1185 1186 ASSERT(MUTEX_HELD(&kpd->kp_lock)); 1187 1188 /* 1189 * Loop through the pages in the given range. It is assumed 1190 * segkp_unlock is called with page aligned base 1191 */ 1192 va = vaddr; 1193 vlim = va + len; 1194 i = ((uintptr_t)(va - kpd->kp_base)) >> PAGESHIFT; 1195 hat_unload(hat, va, len, 1196 ((flags & KPD_LOCKED) ? HAT_UNLOAD_UNLOCK : HAT_UNLOAD)); 1197 for (; va < vlim; va += PAGESIZE, i++) { 1198 /* 1199 * Find the page associated with this part of the 1200 * slot, tracking it down through its associated swap 1201 * space. 1202 */ 1203 ap = anon_get_ptr(kpd->kp_anon, kpd->kp_anon_idx + i); 1204 swap_xlate(ap, &vp, &off); 1205 1206 if (flags & KPD_LOCKED) { 1207 if ((pp = page_find(vp, off)) == NULL) { 1208 if (flags & KPD_LOCKED) { 1209 panic("segkp_softunlock: missing page"); 1210 /*NOTREACHED*/ 1211 } 1212 } 1213 } else { 1214 /* 1215 * Nothing to do if the slot is not locked and the 1216 * page doesn't exist. 1217 */ 1218 if ((pp = page_lookup(vp, off, SE_SHARED)) == NULL) 1219 continue; 1220 } 1221 1222 /* 1223 * If the page doesn't have any translations, is 1224 * dirty and not being shared, then push it out 1225 * asynchronously and avoid waiting for the 1226 * pageout daemon to do it for us. 1227 * 1228 * XXX - Do we really need to get the "exclusive" 1229 * lock via an upgrade? 1230 */ 1231 if ((flags & KPD_WRITEDIRTY) && !hat_page_is_mapped(pp) && 1232 hat_ismod(pp) && page_tryupgrade(pp)) { 1233 /* 1234 * Hold the vnode before releasing the page lock to 1235 * prevent it from being freed and re-used by some 1236 * other thread. 1237 */ 1238 VN_HOLD(vp); 1239 page_unlock(pp); 1240 1241 /* 1242 * Want most powerful credentials we can get so 1243 * use kcred. 1244 */ 1245 (void) VOP_PUTPAGE(vp, (offset_t)off, PAGESIZE, 1246 B_ASYNC | B_FREE, kcred, NULL); 1247 VN_RELE(vp); 1248 } else { 1249 page_unlock(pp); 1250 } 1251 } 1252 1253 /* If unlocking, release physical memory */ 1254 if (flags & KPD_LOCKED) { 1255 pgcnt_t pages = btopr(len); 1256 if ((kpd->kp_flags & KPD_NO_ANON) == 0) 1257 atomic_add_long(&anon_segkp_pages_locked, -pages); 1258 page_unresv(pages); 1259 } 1260 return (0); 1261 } 1262 1263 /* 1264 * Insert the kpd in the hash table. 1265 */ 1266 static void 1267 segkp_insert(struct seg *seg, struct segkp_data *kpd) 1268 { 1269 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 1270 int index; 1271 1272 /* 1273 * Insert the kpd based on the address that will be returned 1274 * via segkp_release. 1275 */ 1276 index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags)); 1277 mutex_enter(&segkp_lock); 1278 kpd->kp_next = kpsd->kpsd_hash[index]; 1279 kpsd->kpsd_hash[index] = kpd; 1280 mutex_exit(&segkp_lock); 1281 } 1282 1283 /* 1284 * Remove kpd from the hash table. 1285 */ 1286 static void 1287 segkp_delete(struct seg *seg, struct segkp_data *kpd) 1288 { 1289 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 1290 struct segkp_data **kpp; 1291 int index; 1292 1293 ASSERT(MUTEX_HELD(&segkp_lock)); 1294 1295 index = SEGKP_HASH(stom(kpd->kp_base, kpd->kp_flags)); 1296 for (kpp = &kpsd->kpsd_hash[index]; 1297 *kpp != NULL; kpp = &((*kpp)->kp_next)) { 1298 if (*kpp == kpd) { 1299 *kpp = kpd->kp_next; 1300 return; 1301 } 1302 } 1303 panic("segkp_delete: unable to find element to delete"); 1304 /*NOTREACHED*/ 1305 } 1306 1307 /* 1308 * Find the kpd associated with a vaddr. 1309 * 1310 * Most of the callers of segkp_find will pass the vaddr that 1311 * hashes to the desired index, but there are cases where 1312 * this is not true in which case we have to (potentially) scan 1313 * the whole table looking for it. This should be very rare 1314 * (e.g. a segkp_fault(F_INVAL) on an address somewhere in the 1315 * middle of the segkp_data region). 1316 */ 1317 static struct segkp_data * 1318 segkp_find(struct seg *seg, caddr_t vaddr) 1319 { 1320 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 1321 struct segkp_data *kpd; 1322 int i; 1323 int stop; 1324 1325 i = stop = SEGKP_HASH(vaddr); 1326 mutex_enter(&segkp_lock); 1327 do { 1328 for (kpd = kpsd->kpsd_hash[i]; kpd != NULL; 1329 kpd = kpd->kp_next) { 1330 if (vaddr >= kpd->kp_base && 1331 vaddr < kpd->kp_base + kpd->kp_len) { 1332 mutex_exit(&segkp_lock); 1333 return (kpd); 1334 } 1335 } 1336 if (--i < 0) 1337 i = SEGKP_HASHSZ - 1; /* Wrap */ 1338 } while (i != stop); 1339 mutex_exit(&segkp_lock); 1340 return (NULL); /* Not found */ 1341 } 1342 1343 /* 1344 * returns size of swappable area. 1345 */ 1346 size_t 1347 swapsize(caddr_t v) 1348 { 1349 struct segkp_data *kpd; 1350 1351 if ((kpd = segkp_find(segkp, v)) != NULL) 1352 return (SEGKP_MAPLEN(kpd->kp_len, kpd->kp_flags)); 1353 else 1354 return (NULL); 1355 } 1356 1357 /* 1358 * Dump out all the active segkp pages 1359 */ 1360 static void 1361 segkp_dump(struct seg *seg) 1362 { 1363 int i; 1364 struct segkp_data *kpd; 1365 struct segkp_segdata *kpsd = (struct segkp_segdata *)seg->s_data; 1366 1367 for (i = 0; i < SEGKP_HASHSZ; i++) { 1368 for (kpd = kpsd->kpsd_hash[i]; 1369 kpd != NULL; kpd = kpd->kp_next) { 1370 pfn_t pfn; 1371 caddr_t addr; 1372 caddr_t eaddr; 1373 1374 addr = kpd->kp_base; 1375 eaddr = addr + kpd->kp_len; 1376 while (addr < eaddr) { 1377 ASSERT(seg->s_as == &kas); 1378 pfn = hat_getpfnum(seg->s_as->a_hat, addr); 1379 if (pfn != PFN_INVALID) 1380 dump_addpage(seg->s_as, addr, pfn); 1381 addr += PAGESIZE; 1382 dump_timeleft = dump_timeout; 1383 } 1384 } 1385 } 1386 } 1387 1388 /*ARGSUSED*/ 1389 static int 1390 segkp_pagelock(struct seg *seg, caddr_t addr, size_t len, 1391 struct page ***ppp, enum lock_type type, enum seg_rw rw) 1392 { 1393 return (ENOTSUP); 1394 } 1395 1396 /*ARGSUSED*/ 1397 static int 1398 segkp_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp) 1399 { 1400 return (ENODEV); 1401 } 1402 1403 #include <sys/mem_config.h> 1404 1405 /*ARGSUSED*/ 1406 static void 1407 segkp_mem_config_post_add(void *arg, pgcnt_t delta_pages) 1408 {} 1409 1410 /* 1411 * During memory delete, turn off caches so that pages are not held. 1412 * A better solution may be to unlock the pages while they are 1413 * in the cache so that they may be collected naturally. 1414 */ 1415 1416 /*ARGSUSED*/ 1417 static int 1418 segkp_mem_config_pre_del(void *arg, pgcnt_t delta_pages) 1419 { 1420 atomic_inc_32(&segkp_indel); 1421 segkp_cache_free(); 1422 return (0); 1423 } 1424 1425 /*ARGSUSED*/ 1426 static void 1427 segkp_mem_config_post_del(void *arg, pgcnt_t delta_pages, int cancelled) 1428 { 1429 atomic_dec_32(&segkp_indel); 1430 } 1431 1432 static kphysm_setup_vector_t segkp_mem_config_vec = { 1433 KPHYSM_SETUP_VECTOR_VERSION, 1434 segkp_mem_config_post_add, 1435 segkp_mem_config_pre_del, 1436 segkp_mem_config_post_del, 1437 }; 1438 1439 static void 1440 segkpinit_mem_config(struct seg *seg) 1441 { 1442 int ret; 1443 1444 ret = kphysm_setup_func_register(&segkp_mem_config_vec, (void *)seg); 1445 ASSERT(ret == 0); 1446 }