1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1998, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 #include <sys/types.h>
26 #include <sys/t_lock.h>
27 #include <sys/param.h>
28 #include <sys/sysmacros.h>
29 #include <sys/tuneable.h>
30 #include <sys/systm.h>
31 #include <sys/vm.h>
32 #include <sys/kmem.h>
33 #include <sys/vmem.h>
34 #include <sys/mman.h>
35 #include <sys/cmn_err.h>
36 #include <sys/debug.h>
37 #include <sys/dumphdr.h>
38 #include <sys/bootconf.h>
39 #include <sys/lgrp.h>
40 #include <vm/seg_kmem.h>
41 #include <vm/hat.h>
42 #include <vm/page.h>
43 #include <vm/vm_dep.h>
44 #include <vm/faultcode.h>
45 #include <sys/promif.h>
46 #include <vm/seg_kp.h>
47 #include <sys/bitmap.h>
48 #include <sys/mem_cage.h>
49
50 #ifdef __sparc
51 #include <sys/ivintr.h>
52 #include <sys/panic.h>
53 #endif
54
55 /*
56 * seg_kmem is the primary kernel memory segment driver. It
57 * maps the kernel heap [kernelheap, ekernelheap), module text,
58 * and all memory which was allocated before the VM was initialized
59 * into kas.
60 *
61 * Pages which belong to seg_kmem are hashed into &kvp vnode at
62 * an offset equal to (u_offset_t)virt_addr, and have p_lckcnt >= 1.
63 * They must never be paged out since segkmem_fault() is a no-op to
64 * prevent recursive faults.
65 *
66 * Currently, seg_kmem pages are sharelocked (p_sharelock == 1) on
67 * __x86 and are unlocked (p_sharelock == 0) on __sparc. Once __x86
68 * supports relocation the #ifdef kludges can be removed.
69 *
70 * seg_kmem pages may be subject to relocation by page_relocate(),
71 * provided that the HAT supports it; if this is so, segkmem_reloc
72 * will be set to a nonzero value. All boot time allocated memory as
73 * well as static memory is considered off limits to relocation.
74 * Pages are "relocatable" if p_state does not have P_NORELOC set, so
75 * we request P_NORELOC pages for memory that isn't safe to relocate.
76 *
77 * The kernel heap is logically divided up into four pieces:
78 *
79 * heap32_arena is for allocations that require 32-bit absolute
80 * virtual addresses (e.g. code that uses 32-bit pointers/offsets).
81 *
82 * heap_core is for allocations that require 2GB *relative*
83 * offsets; in other words all memory from heap_core is within
84 * 2GB of all other memory from the same arena. This is a requirement
85 * of the addressing modes of some processors in supervisor code.
86 *
87 * heap_arena is the general heap arena.
88 *
89 * static_arena is the static memory arena. Allocations from it
90 * are not subject to relocation so it is safe to use the memory
91 * physical address as well as the virtual address (e.g. the VA to
92 * PA translations are static). Caches may import from static_arena;
93 * all other static memory allocations should use static_alloc_arena.
94 *
95 * On some platforms which have limited virtual address space, seg_kmem
96 * may share [kernelheap, ekernelheap) with seg_kp; if this is so,
97 * segkp_bitmap is non-NULL, and each bit represents a page of virtual
98 * address space which is actually seg_kp mapped.
99 */
100
101 extern ulong_t *segkp_bitmap; /* Is set if segkp is from the kernel heap */
102
103 char *kernelheap; /* start of primary kernel heap */
104 char *ekernelheap; /* end of primary kernel heap */
105 struct seg kvseg; /* primary kernel heap segment */
106 struct seg kvseg_core; /* "core" kernel heap segment */
107 struct seg kzioseg; /* Segment for zio mappings */
108 vmem_t *heap_arena; /* primary kernel heap arena */
109 vmem_t *heap_core_arena; /* core kernel heap arena */
110 char *heap_core_base; /* start of core kernel heap arena */
111 char *heap_lp_base; /* start of kernel large page heap arena */
112 char *heap_lp_end; /* end of kernel large page heap arena */
113 vmem_t *hat_memload_arena; /* HAT translation data */
114 struct seg kvseg32; /* 32-bit kernel heap segment */
115 vmem_t *heap32_arena; /* 32-bit kernel heap arena */
116 vmem_t *heaptext_arena; /* heaptext arena */
117 struct as kas; /* kernel address space */
118 int segkmem_reloc; /* enable/disable relocatable segkmem pages */
119 vmem_t *static_arena; /* arena for caches to import static memory */
120 vmem_t *static_alloc_arena; /* arena for allocating static memory */
121 vmem_t *zio_arena = NULL; /* arena for allocating zio memory */
122 vmem_t *zio_alloc_arena = NULL; /* arena for allocating zio memory */
123
124 /*
125 * seg_kmem driver can map part of the kernel heap with large pages.
126 * Currently this functionality is implemented for sparc platforms only.
127 *
128 * The large page size "segkmem_lpsize" for kernel heap is selected in the
129 * platform specific code. It can also be modified via /etc/system file.
130 * Setting segkmem_lpsize to PAGESIZE in /etc/system disables usage of large
131 * pages for kernel heap. "segkmem_lpshift" is adjusted appropriately to
132 * match segkmem_lpsize.
133 *
134 * At boot time we carve from kernel heap arena a range of virtual addresses
135 * that will be used for large page mappings. This range [heap_lp_base,
136 * heap_lp_end) is set up as a separate vmem arena - "heap_lp_arena". We also
137 * create "kmem_lp_arena" that caches memory already backed up by large
138 * pages. kmem_lp_arena imports virtual segments from heap_lp_arena.
139 */
140
141 size_t segkmem_lpsize;
142 static uint_t segkmem_lpshift = PAGESHIFT;
143 int segkmem_lpszc = 0;
144
145 size_t segkmem_kmemlp_quantum = 0x400000; /* 4MB */
146 size_t segkmem_heaplp_quantum;
147 vmem_t *heap_lp_arena;
148 static vmem_t *kmem_lp_arena;
149 static vmem_t *segkmem_ppa_arena;
150 static segkmem_lpcb_t segkmem_lpcb;
151
152 /*
153 * We use "segkmem_kmemlp_max" to limit the total amount of physical memory
154 * consumed by the large page heap. By default this parameter is set to 1/8 of
155 * physmem but can be adjusted through /etc/system either directly or
156 * indirectly by setting "segkmem_kmemlp_pcnt" to the percent of physmem
157 * we allow for large page heap.
158 */
159 size_t segkmem_kmemlp_max;
160 static uint_t segkmem_kmemlp_pcnt;
161
162 /*
163 * Getting large pages for kernel heap could be problematic due to
164 * physical memory fragmentation. That's why we allow to preallocate
165 * "segkmem_kmemlp_min" bytes at boot time.
166 */
167 static size_t segkmem_kmemlp_min;
168
169 /*
170 * Throttling is used to avoid expensive tries to allocate large pages
171 * for kernel heap when a lot of succesive attempts to do so fail.
172 */
173 static ulong_t segkmem_lpthrottle_max = 0x400000;
174 static ulong_t segkmem_lpthrottle_start = 0x40;
175 static ulong_t segkmem_use_lpthrottle = 1;
176
177 /*
178 * Freed pages accumulate on a garbage list until segkmem is ready,
179 * at which point we call segkmem_gc() to free it all.
180 */
181 typedef struct segkmem_gc_list {
182 struct segkmem_gc_list *gc_next;
183 vmem_t *gc_arena;
184 size_t gc_size;
185 } segkmem_gc_list_t;
186
187 static segkmem_gc_list_t *segkmem_gc_list;
188
189 /*
190 * Allocations from the hat_memload arena add VM_MEMLOAD to their
191 * vmflags so that segkmem_xalloc() can inform the hat layer that it needs
192 * to take steps to prevent infinite recursion. HAT allocations also
193 * must be non-relocatable to prevent recursive page faults.
194 */
195 static void *
196 hat_memload_alloc(vmem_t *vmp, size_t size, int flags)
197 {
198 flags |= (VM_MEMLOAD | VM_NORELOC);
199 return (segkmem_alloc(vmp, size, flags));
200 }
201
202 /*
203 * Allocations from static_arena arena (or any other arena that uses
204 * segkmem_alloc_permanent()) require non-relocatable (permanently
205 * wired) memory pages, since these pages are referenced by physical
206 * as well as virtual address.
207 */
208 void *
209 segkmem_alloc_permanent(vmem_t *vmp, size_t size, int flags)
210 {
211 return (segkmem_alloc(vmp, size, flags | VM_NORELOC));
212 }
213
214 /*
215 * Initialize kernel heap boundaries.
216 */
217 void
218 kernelheap_init(
219 void *heap_start,
220 void *heap_end,
221 char *first_avail,
222 void *core_start,
223 void *core_end)
224 {
225 uintptr_t textbase;
226 size_t core_size;
227 size_t heap_size;
228 vmem_t *heaptext_parent;
229 size_t heap_lp_size = 0;
230 #ifdef __sparc
231 size_t kmem64_sz = kmem64_aligned_end - kmem64_base;
232 #endif /* __sparc */
233
234 kernelheap = heap_start;
235 ekernelheap = heap_end;
236
237 #ifdef __sparc
238 heap_lp_size = (((uintptr_t)heap_end - (uintptr_t)heap_start) / 4);
239 /*
240 * Bias heap_lp start address by kmem64_sz to reduce collisions
241 * in 4M kernel TSB between kmem64 area and heap_lp
242 */
243 kmem64_sz = P2ROUNDUP(kmem64_sz, MMU_PAGESIZE256M);
244 if (kmem64_sz <= heap_lp_size / 2)
245 heap_lp_size -= kmem64_sz;
246 heap_lp_base = ekernelheap - heap_lp_size;
247 heap_lp_end = heap_lp_base + heap_lp_size;
248 #endif /* __sparc */
249
250 /*
251 * If this platform has a 'core' heap area, then the space for
252 * overflow module text should be carved out of the end of that
253 * heap. Otherwise, it gets carved out of the general purpose
254 * heap.
255 */
256 core_size = (uintptr_t)core_end - (uintptr_t)core_start;
257 if (core_size > 0) {
258 ASSERT(core_size >= HEAPTEXT_SIZE);
259 textbase = (uintptr_t)core_end - HEAPTEXT_SIZE;
260 core_size -= HEAPTEXT_SIZE;
261 }
262 #ifndef __sparc
263 else {
264 ekernelheap -= HEAPTEXT_SIZE;
265 textbase = (uintptr_t)ekernelheap;
266 }
267 #endif
268
269 heap_size = (uintptr_t)ekernelheap - (uintptr_t)kernelheap;
270 heap_arena = vmem_init("heap", kernelheap, heap_size, PAGESIZE,
271 segkmem_alloc, segkmem_free);
272
273 if (core_size > 0) {
274 heap_core_arena = vmem_create("heap_core", core_start,
275 core_size, PAGESIZE, NULL, NULL, NULL, 0, VM_SLEEP);
276 heap_core_base = core_start;
277 } else {
278 heap_core_arena = heap_arena;
279 heap_core_base = kernelheap;
280 }
281
282 /*
283 * reserve space for the large page heap. If large pages for kernel
284 * heap is enabled large page heap arean will be created later in the
285 * boot sequence in segkmem_heap_lp_init(). Otherwise the allocated
286 * range will be returned back to the heap_arena.
287 */
288 if (heap_lp_size) {
289 (void) vmem_xalloc(heap_arena, heap_lp_size, PAGESIZE, 0, 0,
290 heap_lp_base, heap_lp_end,
291 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
292 }
293
294 /*
295 * Remove the already-spoken-for memory range [kernelheap, first_avail).
296 */
297 (void) vmem_xalloc(heap_arena, first_avail - kernelheap, PAGESIZE,
298 0, 0, kernelheap, first_avail, VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
299
300 #ifdef __sparc
301 heap32_arena = vmem_create("heap32", (void *)SYSBASE32,
302 SYSLIMIT32 - SYSBASE32 - HEAPTEXT_SIZE, PAGESIZE, NULL,
303 NULL, NULL, 0, VM_SLEEP);
304 /*
305 * Prom claims the physical and virtual resources used by panicbuf
306 * and inter_vec_table. So reserve space for panicbuf, intr_vec_table,
307 * reserved interrupt vector data structures from 32-bit heap.
308 */
309 (void) vmem_xalloc(heap32_arena, PANICBUFSIZE, PAGESIZE, 0, 0,
310 panicbuf, panicbuf + PANICBUFSIZE,
311 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
312
313 (void) vmem_xalloc(heap32_arena, IVSIZE, PAGESIZE, 0, 0,
314 intr_vec_table, (caddr_t)intr_vec_table + IVSIZE,
315 VM_NOSLEEP | VM_BESTFIT | VM_PANIC);
316
317 textbase = SYSLIMIT32 - HEAPTEXT_SIZE;
318 heaptext_parent = NULL;
319 #else /* __sparc */
320 heap32_arena = heap_core_arena;
321 heaptext_parent = heap_core_arena;
322 #endif /* __sparc */
323
324 heaptext_arena = vmem_create("heaptext", (void *)textbase,
325 HEAPTEXT_SIZE, PAGESIZE, NULL, NULL, heaptext_parent, 0, VM_SLEEP);
326
327 /*
328 * Create a set of arenas for memory with static translations
329 * (e.g. VA -> PA translations cannot change). Since using
330 * kernel pages by physical address implies it isn't safe to
331 * walk across page boundaries, the static_arena quantum must
332 * be PAGESIZE. Any kmem caches that require static memory
333 * should source from static_arena, while direct allocations
334 * should only use static_alloc_arena.
335 */
336 static_arena = vmem_create("static", NULL, 0, PAGESIZE,
337 segkmem_alloc_permanent, segkmem_free, heap_arena, 0, VM_SLEEP);
338 static_alloc_arena = vmem_create("static_alloc", NULL, 0,
339 sizeof (uint64_t), vmem_alloc, vmem_free, static_arena,
340 0, VM_SLEEP);
341
342 /*
343 * Create an arena for translation data (ptes, hmes, or hblks).
344 * We need an arena for this because hat_memload() is essential
345 * to vmem_populate() (see comments in common/os/vmem.c).
346 *
347 * Note: any kmem cache that allocates from hat_memload_arena
348 * must be created as a KMC_NOHASH cache (i.e. no external slab
349 * and bufctl structures to allocate) so that slab creation doesn't
350 * require anything more than a single vmem_alloc().
351 */
352 hat_memload_arena = vmem_create("hat_memload", NULL, 0, PAGESIZE,
353 hat_memload_alloc, segkmem_free, heap_arena, 0,
354 VM_SLEEP | VMC_POPULATOR | VMC_DUMPSAFE);
355 }
356
357 void
358 boot_mapin(caddr_t addr, size_t size)
359 {
360 caddr_t eaddr;
361 page_t *pp;
362 pfn_t pfnum;
363
364 if (page_resv(btop(size), KM_NOSLEEP) == 0)
365 panic("boot_mapin: page_resv failed");
366
367 for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
368 pfnum = va_to_pfn(addr);
369 if (pfnum == PFN_INVALID)
370 continue;
371 if ((pp = page_numtopp_nolock(pfnum)) == NULL)
372 panic("boot_mapin(): No pp for pfnum = %lx", pfnum);
373
374 /*
375 * must break up any large pages that may have constituent
376 * pages being utilized for BOP_ALLOC()'s before calling
377 * page_numtopp().The locking code (ie. page_reclaim())
378 * can't handle them
379 */
380 if (pp->p_szc != 0)
381 page_boot_demote(pp);
382
383 pp = page_numtopp(pfnum, SE_EXCL);
384 if (pp == NULL || PP_ISFREE(pp))
385 panic("boot_alloc: pp is NULL or free");
386
387 /*
388 * If the cage is on but doesn't yet contain this page,
389 * mark it as non-relocatable.
390 */
391 if (kcage_on && !PP_ISNORELOC(pp)) {
392 PP_SETNORELOC(pp);
393 PLCNT_XFER_NORELOC(pp);
394 }
395
396 (void) page_hashin(pp, &kvp, (u_offset_t)(uintptr_t)addr, NULL);
397 pp->p_lckcnt = 1;
398 #if defined(__x86)
399 page_downgrade(pp);
400 #else
401 page_unlock(pp);
402 #endif
403 }
404 }
405
406 /*
407 * Get pages from boot and hash them into the kernel's vp.
408 * Used after page structs have been allocated, but before segkmem is ready.
409 */
410 void *
411 boot_alloc(void *inaddr, size_t size, uint_t align)
412 {
413 caddr_t addr = inaddr;
414
415 if (bootops == NULL)
416 prom_panic("boot_alloc: attempt to allocate memory after "
417 "BOP_GONE");
418
419 size = ptob(btopr(size));
420 #ifdef __sparc
421 if (bop_alloc_chunk(addr, size, align) != (caddr_t)addr)
422 panic("boot_alloc: bop_alloc_chunk failed");
423 #else
424 if (BOP_ALLOC(bootops, addr, size, align) != addr)
425 panic("boot_alloc: BOP_ALLOC failed");
426 #endif
427 boot_mapin((caddr_t)addr, size);
428 return (addr);
429 }
430
431 static void
432 segkmem_badop()
433 {
434 panic("segkmem_badop");
435 }
436
437 #define SEGKMEM_BADOP(t) (t(*)())segkmem_badop
438
439 /*ARGSUSED*/
440 static faultcode_t
441 segkmem_fault(struct hat *hat, struct seg *seg, caddr_t addr, size_t size,
442 enum fault_type type, enum seg_rw rw)
443 {
444 pgcnt_t npages;
445 spgcnt_t pg;
446 page_t *pp;
447 struct vnode *vp = seg->s_data;
448
449 ASSERT(RW_READ_HELD(&seg->s_as->a_lock));
450
451 if (seg->s_as != &kas || size > seg->s_size ||
452 addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
453 panic("segkmem_fault: bad args");
454
455 /*
456 * If it is one of segkp pages, call segkp_fault.
457 */
458 if (segkp_bitmap && seg == &kvseg &&
459 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
460 return (SEGOP_FAULT(hat, segkp, addr, size, type, rw));
461
462 if (rw != S_READ && rw != S_WRITE && rw != S_OTHER)
463 return (FC_NOSUPPORT);
464
465 npages = btopr(size);
466
467 switch (type) {
468 case F_SOFTLOCK: /* lock down already-loaded translations */
469 for (pg = 0; pg < npages; pg++) {
470 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
471 SE_SHARED);
472 if (pp == NULL) {
473 /*
474 * Hmm, no page. Does a kernel mapping
475 * exist for it?
476 */
477 if (!hat_probe(kas.a_hat, addr)) {
478 addr -= PAGESIZE;
479 while (--pg >= 0) {
480 pp = page_find(vp, (u_offset_t)
481 (uintptr_t)addr);
482 if (pp)
483 page_unlock(pp);
484 addr -= PAGESIZE;
485 }
486 return (FC_NOMAP);
487 }
488 }
489 addr += PAGESIZE;
490 }
491 if (rw == S_OTHER)
492 hat_reserve(seg->s_as, addr, size);
493 return (0);
494 case F_SOFTUNLOCK:
495 while (npages--) {
496 pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
497 if (pp)
498 page_unlock(pp);
499 addr += PAGESIZE;
500 }
501 return (0);
502 default:
503 return (FC_NOSUPPORT);
504 }
505 /*NOTREACHED*/
506 }
507
508 static int
509 segkmem_setprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
510 {
511 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
512
513 if (seg->s_as != &kas || size > seg->s_size ||
514 addr < seg->s_base || addr + size > seg->s_base + seg->s_size)
515 panic("segkmem_setprot: bad args");
516
517 /*
518 * If it is one of segkp pages, call segkp.
519 */
520 if (segkp_bitmap && seg == &kvseg &&
521 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
522 return (SEGOP_SETPROT(segkp, addr, size, prot));
523
524 if (prot == 0)
525 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD);
526 else
527 hat_chgprot(kas.a_hat, addr, size, prot);
528 return (0);
529 }
530
531 /*
532 * This is a dummy segkmem function overloaded to call segkp
533 * when segkp is under the heap.
534 */
535 /* ARGSUSED */
536 static int
537 segkmem_checkprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
538 {
539 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
540
541 if (seg->s_as != &kas)
542 segkmem_badop();
543
544 /*
545 * If it is one of segkp pages, call into segkp.
546 */
547 if (segkp_bitmap && seg == &kvseg &&
548 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
549 return (SEGOP_CHECKPROT(segkp, addr, size, prot));
550
551 segkmem_badop();
552 return (0);
553 }
554
555 /*
556 * This is a dummy segkmem function overloaded to call segkp
557 * when segkp is under the heap.
558 */
559 /* ARGSUSED */
560 static int
561 segkmem_kluster(struct seg *seg, caddr_t addr, ssize_t delta)
562 {
563 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
564
565 if (seg->s_as != &kas)
566 segkmem_badop();
567
568 /*
569 * If it is one of segkp pages, call into segkp.
570 */
571 if (segkp_bitmap && seg == &kvseg &&
572 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
573 return (SEGOP_KLUSTER(segkp, addr, delta));
574
575 segkmem_badop();
576 return (0);
577 }
578
579 static void
580 segkmem_xdump_range(void *arg, void *start, size_t size)
581 {
582 struct as *as = arg;
583 caddr_t addr = start;
584 caddr_t addr_end = addr + size;
585
586 while (addr < addr_end) {
587 pfn_t pfn = hat_getpfnum(kas.a_hat, addr);
588 if (pfn != PFN_INVALID && pfn <= physmax && pf_is_memory(pfn))
589 dump_addpage(as, addr, pfn);
590 addr += PAGESIZE;
591 dump_timeleft = dump_timeout;
592 }
593 }
594
595 static void
596 segkmem_dump_range(void *arg, void *start, size_t size)
597 {
598 caddr_t addr = start;
599 caddr_t addr_end = addr + size;
600
601 /*
602 * If we are about to start dumping the range of addresses we
603 * carved out of the kernel heap for the large page heap walk
604 * heap_lp_arena to find what segments are actually populated
605 */
606 if (SEGKMEM_USE_LARGEPAGES &&
607 addr == heap_lp_base && addr_end == heap_lp_end &&
608 vmem_size(heap_lp_arena, VMEM_ALLOC) < size) {
609 vmem_walk(heap_lp_arena, VMEM_ALLOC | VMEM_REENTRANT,
610 segkmem_xdump_range, arg);
611 } else {
612 segkmem_xdump_range(arg, start, size);
613 }
614 }
615
616 static void
617 segkmem_dump(struct seg *seg)
618 {
619 /*
620 * The kernel's heap_arena (represented by kvseg) is a very large
621 * VA space, most of which is typically unused. To speed up dumping
622 * we use vmem_walk() to quickly find the pieces of heap_arena that
623 * are actually in use. We do the same for heap32_arena and
624 * heap_core.
625 *
626 * We specify VMEM_REENTRANT to vmem_walk() because dump_addpage()
627 * may ultimately need to allocate memory. Reentrant walks are
628 * necessarily imperfect snapshots. The kernel heap continues
629 * to change during a live crash dump, for example. For a normal
630 * crash dump, however, we know that there won't be any other threads
631 * messing with the heap. Therefore, at worst, we may fail to dump
632 * the pages that get allocated by the act of dumping; but we will
633 * always dump every page that was allocated when the walk began.
634 *
635 * The other segkmem segments are dense (fully populated), so there's
636 * no need to use this technique when dumping them.
637 *
638 * Note: when adding special dump handling for any new sparsely-
639 * populated segments, be sure to add similar handling to the ::kgrep
640 * code in mdb.
641 */
642 if (seg == &kvseg) {
643 vmem_walk(heap_arena, VMEM_ALLOC | VMEM_REENTRANT,
644 segkmem_dump_range, seg->s_as);
645 #ifndef __sparc
646 vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
647 segkmem_dump_range, seg->s_as);
648 #endif
649 } else if (seg == &kvseg_core) {
650 vmem_walk(heap_core_arena, VMEM_ALLOC | VMEM_REENTRANT,
651 segkmem_dump_range, seg->s_as);
652 } else if (seg == &kvseg32) {
653 vmem_walk(heap32_arena, VMEM_ALLOC | VMEM_REENTRANT,
654 segkmem_dump_range, seg->s_as);
655 vmem_walk(heaptext_arena, VMEM_ALLOC | VMEM_REENTRANT,
656 segkmem_dump_range, seg->s_as);
657 } else if (seg == &kzioseg) {
658 /*
659 * We don't want to dump pages attached to kzioseg since they
660 * contain file data from ZFS. If this page's segment is
661 * kzioseg return instead of writing it to the dump device.
662 */
663 return;
664 } else {
665 segkmem_dump_range(seg->s_as, seg->s_base, seg->s_size);
666 }
667 }
668
669 /*
670 * lock/unlock kmem pages over a given range [addr, addr+len).
671 * Returns a shadow list of pages in ppp. If there are holes
672 * in the range (e.g. some of the kernel mappings do not have
673 * underlying page_ts) returns ENOTSUP so that as_pagelock()
674 * will handle the range via as_fault(F_SOFTLOCK).
675 */
676 /*ARGSUSED*/
677 static int
678 segkmem_pagelock(struct seg *seg, caddr_t addr, size_t len,
679 page_t ***ppp, enum lock_type type, enum seg_rw rw)
680 {
681 page_t **pplist, *pp;
682 pgcnt_t npages;
683 spgcnt_t pg;
684 size_t nb;
685 struct vnode *vp = seg->s_data;
686
687 ASSERT(ppp != NULL);
688
689 /*
690 * If it is one of segkp pages, call into segkp.
691 */
692 if (segkp_bitmap && seg == &kvseg &&
693 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
694 return (SEGOP_PAGELOCK(segkp, addr, len, ppp, type, rw));
695
696 npages = btopr(len);
697 nb = sizeof (page_t *) * npages;
698
699 if (type == L_PAGEUNLOCK) {
700 pplist = *ppp;
701 ASSERT(pplist != NULL);
702
703 for (pg = 0; pg < npages; pg++) {
704 pp = pplist[pg];
705 page_unlock(pp);
706 }
707 kmem_free(pplist, nb);
708 return (0);
709 }
710
711 ASSERT(type == L_PAGELOCK);
712
713 pplist = kmem_alloc(nb, KM_NOSLEEP);
714 if (pplist == NULL) {
715 *ppp = NULL;
716 return (ENOTSUP); /* take the slow path */
717 }
718
719 for (pg = 0; pg < npages; pg++) {
720 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_SHARED);
721 if (pp == NULL) {
722 while (--pg >= 0)
723 page_unlock(pplist[pg]);
724 kmem_free(pplist, nb);
725 *ppp = NULL;
726 return (ENOTSUP);
727 }
728 pplist[pg] = pp;
729 addr += PAGESIZE;
730 }
731
732 *ppp = pplist;
733 return (0);
734 }
735
736 /*
737 * This is a dummy segkmem function overloaded to call segkp
738 * when segkp is under the heap.
739 */
740 /* ARGSUSED */
741 static int
742 segkmem_getmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
743 {
744 ASSERT(RW_LOCK_HELD(&seg->s_as->a_lock));
745
746 if (seg->s_as != &kas)
747 segkmem_badop();
748
749 /*
750 * If it is one of segkp pages, call into segkp.
751 */
752 if (segkp_bitmap && seg == &kvseg &&
753 BT_TEST(segkp_bitmap, btop((uintptr_t)(addr - seg->s_base))))
754 return (SEGOP_GETMEMID(segkp, addr, memidp));
755
756 segkmem_badop();
757 return (0);
758 }
759
760 /*ARGSUSED*/
761 static lgrp_mem_policy_info_t *
762 segkmem_getpolicy(struct seg *seg, caddr_t addr)
763 {
764 return (NULL);
765 }
766
767 /*ARGSUSED*/
768 static int
769 segkmem_capable(struct seg *seg, segcapability_t capability)
770 {
771 if (capability == S_CAPABILITY_NOMINFLT)
772 return (1);
773 return (0);
774 }
775
776 static struct seg_ops segkmem_ops = {
777 SEGKMEM_BADOP(int), /* dup */
778 SEGKMEM_BADOP(int), /* unmap */
779 SEGKMEM_BADOP(void), /* free */
780 segkmem_fault,
781 SEGKMEM_BADOP(faultcode_t), /* faulta */
782 segkmem_setprot,
783 segkmem_checkprot,
784 segkmem_kluster,
785 SEGKMEM_BADOP(int), /* sync */
786 SEGKMEM_BADOP(size_t), /* incore */
787 SEGKMEM_BADOP(int), /* lockop */
788 SEGKMEM_BADOP(int), /* getprot */
789 SEGKMEM_BADOP(u_offset_t), /* getoffset */
790 SEGKMEM_BADOP(int), /* gettype */
791 SEGKMEM_BADOP(int), /* getvp */
792 SEGKMEM_BADOP(int), /* advise */
793 segkmem_dump,
794 segkmem_pagelock,
795 SEGKMEM_BADOP(int), /* setpgsz */
796 segkmem_getmemid,
797 segkmem_getpolicy, /* getpolicy */
798 segkmem_capable, /* capable */
799 };
800
801 int
802 segkmem_zio_create(struct seg *seg)
803 {
804 ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock));
805 seg->s_ops = &segkmem_ops;
806 seg->s_data = &zvp;
807 kas.a_size += seg->s_size;
808 return (0);
809 }
810
811 int
812 segkmem_create(struct seg *seg)
813 {
814 ASSERT(seg->s_as == &kas && RW_WRITE_HELD(&kas.a_lock));
815 seg->s_ops = &segkmem_ops;
816 seg->s_data = &kvp;
817 kas.a_size += seg->s_size;
818 return (0);
819 }
820
821 /*ARGSUSED*/
822 page_t *
823 segkmem_page_create(void *addr, size_t size, int vmflag, void *arg)
824 {
825 struct seg kseg;
826 int pgflags;
827 struct vnode *vp = arg;
828
829 if (vp == NULL)
830 vp = &kvp;
831
832 kseg.s_as = &kas;
833 pgflags = PG_EXCL;
834
835 if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
836 pgflags |= PG_NORELOC;
837 if ((vmflag & VM_NOSLEEP) == 0)
838 pgflags |= PG_WAIT;
839 if (vmflag & VM_PANIC)
840 pgflags |= PG_PANIC;
841 if (vmflag & VM_PUSHPAGE)
842 pgflags |= PG_PUSHPAGE;
843 if (vmflag & VM_NORMALPRI) {
844 ASSERT(vmflag & VM_NOSLEEP);
845 pgflags |= PG_NORMALPRI;
846 }
847
848 return (page_create_va(vp, (u_offset_t)(uintptr_t)addr, size,
849 pgflags, &kseg, addr));
850 }
851
852 /*
853 * Allocate pages to back the virtual address range [addr, addr + size).
854 * If addr is NULL, allocate the virtual address space as well.
855 */
856 void *
857 segkmem_xalloc(vmem_t *vmp, void *inaddr, size_t size, int vmflag, uint_t attr,
858 page_t *(*page_create_func)(void *, size_t, int, void *), void *pcarg)
859 {
860 page_t *ppl;
861 caddr_t addr = inaddr;
862 pgcnt_t npages = btopr(size);
863 int allocflag;
864
865 if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
866 return (NULL);
867
868 ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
869
870 if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
871 if (inaddr == NULL)
872 vmem_free(vmp, addr, size);
873 return (NULL);
874 }
875
876 ppl = page_create_func(addr, size, vmflag, pcarg);
877 if (ppl == NULL) {
878 if (inaddr == NULL)
879 vmem_free(vmp, addr, size);
880 page_unresv(npages);
881 return (NULL);
882 }
883
884 /*
885 * Under certain conditions, we need to let the HAT layer know
886 * that it cannot safely allocate memory. Allocations from
887 * the hat_memload vmem arena always need this, to prevent
888 * infinite recursion.
889 *
890 * In addition, the x86 hat cannot safely do memory
891 * allocations while in vmem_populate(), because there
892 * is no simple bound on its usage.
893 */
894 if (vmflag & VM_MEMLOAD)
895 allocflag = HAT_NO_KALLOC;
896 #if defined(__x86)
897 else if (vmem_is_populator())
898 allocflag = HAT_NO_KALLOC;
899 #endif
900 else
901 allocflag = 0;
902
903 while (ppl != NULL) {
904 page_t *pp = ppl;
905 page_sub(&ppl, pp);
906 ASSERT(page_iolock_assert(pp));
907 ASSERT(PAGE_EXCL(pp));
908 page_io_unlock(pp);
909 hat_memload(kas.a_hat, (caddr_t)(uintptr_t)pp->p_offset, pp,
910 (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
911 HAT_LOAD_LOCK | allocflag);
912 pp->p_lckcnt = 1;
913 #if defined(__x86)
914 page_downgrade(pp);
915 #else
916 if (vmflag & SEGKMEM_SHARELOCKED)
917 page_downgrade(pp);
918 else
919 page_unlock(pp);
920 #endif
921 }
922
923 return (addr);
924 }
925
926 static void *
927 segkmem_alloc_vn(vmem_t *vmp, size_t size, int vmflag, struct vnode *vp)
928 {
929 void *addr;
930 segkmem_gc_list_t *gcp, **prev_gcpp;
931
932 ASSERT(vp != NULL);
933
934 if (kvseg.s_base == NULL) {
935 #ifndef __sparc
936 if (bootops->bsys_alloc == NULL)
937 halt("Memory allocation between bop_alloc() and "
938 "kmem_alloc().\n");
939 #endif
940
941 /*
942 * There's not a lot of memory to go around during boot,
943 * so recycle it if we can.
944 */
945 for (prev_gcpp = &segkmem_gc_list; (gcp = *prev_gcpp) != NULL;
946 prev_gcpp = &gcp->gc_next) {
947 if (gcp->gc_arena == vmp && gcp->gc_size == size) {
948 *prev_gcpp = gcp->gc_next;
949 return (gcp);
950 }
951 }
952
953 addr = vmem_alloc(vmp, size, vmflag | VM_PANIC);
954 if (boot_alloc(addr, size, BO_NO_ALIGN) != addr)
955 panic("segkmem_alloc: boot_alloc failed");
956 return (addr);
957 }
958 return (segkmem_xalloc(vmp, NULL, size, vmflag, 0,
959 segkmem_page_create, vp));
960 }
961
962 void *
963 segkmem_alloc(vmem_t *vmp, size_t size, int vmflag)
964 {
965 return (segkmem_alloc_vn(vmp, size, vmflag, &kvp));
966 }
967
968 void *
969 segkmem_zio_alloc(vmem_t *vmp, size_t size, int vmflag)
970 {
971 return (segkmem_alloc_vn(vmp, size, vmflag, &zvp));
972 }
973
974 /*
975 * Any changes to this routine must also be carried over to
976 * devmap_free_pages() in the seg_dev driver. This is because
977 * we currently don't have a special kernel segment for non-paged
978 * kernel memory that is exported by drivers to user space.
979 */
980 static void
981 segkmem_free_vn(vmem_t *vmp, void *inaddr, size_t size, struct vnode *vp,
982 void (*func)(page_t *))
983 {
984 page_t *pp;
985 caddr_t addr = inaddr;
986 caddr_t eaddr;
987 pgcnt_t npages = btopr(size);
988
989 ASSERT(((uintptr_t)addr & PAGEOFFSET) == 0);
990 ASSERT(vp != NULL);
991
992 if (kvseg.s_base == NULL) {
993 segkmem_gc_list_t *gc = inaddr;
994 gc->gc_arena = vmp;
995 gc->gc_size = size;
996 gc->gc_next = segkmem_gc_list;
997 segkmem_gc_list = gc;
998 return;
999 }
1000
1001 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1002
1003 for (eaddr = addr + size; addr < eaddr; addr += PAGESIZE) {
1004 #if defined(__x86)
1005 pp = page_find(vp, (u_offset_t)(uintptr_t)addr);
1006 if (pp == NULL)
1007 panic("segkmem_free: page not found");
1008 if (!page_tryupgrade(pp)) {
1009 /*
1010 * Some other thread has a sharelock. Wait for
1011 * it to drop the lock so we can free this page.
1012 */
1013 page_unlock(pp);
1014 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr,
1015 SE_EXCL);
1016 }
1017 #else
1018 pp = page_lookup(vp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
1019 #endif
1020 if (pp == NULL)
1021 panic("segkmem_free: page not found");
1022 /* Clear p_lckcnt so page_destroy() doesn't update availrmem */
1023 pp->p_lckcnt = 0;
1024 if (func)
1025 func(pp);
1026 else
1027 page_destroy(pp, 0);
1028 }
1029 if (func == NULL)
1030 page_unresv(npages);
1031
1032 if (vmp != NULL)
1033 vmem_free(vmp, inaddr, size);
1034
1035 }
1036
1037 void
1038 segkmem_xfree(vmem_t *vmp, void *inaddr, size_t size, void (*func)(page_t *))
1039 {
1040 segkmem_free_vn(vmp, inaddr, size, &kvp, func);
1041 }
1042
1043 void
1044 segkmem_free(vmem_t *vmp, void *inaddr, size_t size)
1045 {
1046 segkmem_free_vn(vmp, inaddr, size, &kvp, NULL);
1047 }
1048
1049 void
1050 segkmem_zio_free(vmem_t *vmp, void *inaddr, size_t size)
1051 {
1052 segkmem_free_vn(vmp, inaddr, size, &zvp, NULL);
1053 }
1054
1055 void
1056 segkmem_gc(void)
1057 {
1058 ASSERT(kvseg.s_base != NULL);
1059 while (segkmem_gc_list != NULL) {
1060 segkmem_gc_list_t *gc = segkmem_gc_list;
1061 segkmem_gc_list = gc->gc_next;
1062 segkmem_free(gc->gc_arena, gc, gc->gc_size);
1063 }
1064 }
1065
1066 /*
1067 * Legacy entry points from here to end of file.
1068 */
1069 void
1070 segkmem_mapin(struct seg *seg, void *addr, size_t size, uint_t vprot,
1071 pfn_t pfn, uint_t flags)
1072 {
1073 hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1074 hat_devload(seg->s_as->a_hat, addr, size, pfn, vprot,
1075 flags | HAT_LOAD_LOCK);
1076 }
1077
1078 void
1079 segkmem_mapout(struct seg *seg, void *addr, size_t size)
1080 {
1081 hat_unload(seg->s_as->a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1082 }
1083
1084 void *
1085 kmem_getpages(pgcnt_t npages, int kmflag)
1086 {
1087 return (kmem_alloc(ptob(npages), kmflag));
1088 }
1089
1090 void
1091 kmem_freepages(void *addr, pgcnt_t npages)
1092 {
1093 kmem_free(addr, ptob(npages));
1094 }
1095
1096 /*
1097 * segkmem_page_create_large() allocates a large page to be used for the kmem
1098 * caches. If kpr is enabled we ask for a relocatable page unless requested
1099 * otherwise. If kpr is disabled we have to ask for a non-reloc page
1100 */
1101 static page_t *
1102 segkmem_page_create_large(void *addr, size_t size, int vmflag, void *arg)
1103 {
1104 int pgflags;
1105
1106 pgflags = PG_EXCL;
1107
1108 if (segkmem_reloc == 0 || (vmflag & VM_NORELOC))
1109 pgflags |= PG_NORELOC;
1110 if (!(vmflag & VM_NOSLEEP))
1111 pgflags |= PG_WAIT;
1112 if (vmflag & VM_PUSHPAGE)
1113 pgflags |= PG_PUSHPAGE;
1114 if (vmflag & VM_NORMALPRI)
1115 pgflags |= PG_NORMALPRI;
1116
1117 return (page_create_va_large(&kvp, (u_offset_t)(uintptr_t)addr, size,
1118 pgflags, &kvseg, addr, arg));
1119 }
1120
1121 /*
1122 * Allocate a large page to back the virtual address range
1123 * [addr, addr + size). If addr is NULL, allocate the virtual address
1124 * space as well.
1125 */
1126 static void *
1127 segkmem_xalloc_lp(vmem_t *vmp, void *inaddr, size_t size, int vmflag,
1128 uint_t attr, page_t *(*page_create_func)(void *, size_t, int, void *),
1129 void *pcarg)
1130 {
1131 caddr_t addr = inaddr, pa;
1132 size_t lpsize = segkmem_lpsize;
1133 pgcnt_t npages = btopr(size);
1134 pgcnt_t nbpages = btop(lpsize);
1135 pgcnt_t nlpages = size >> segkmem_lpshift;
1136 size_t ppasize = nbpages * sizeof (page_t *);
1137 page_t *pp, *rootpp, **ppa, *pplist = NULL;
1138 int i;
1139
1140 vmflag |= VM_NOSLEEP;
1141
1142 if (page_resv(npages, vmflag & VM_KMFLAGS) == 0) {
1143 return (NULL);
1144 }
1145
1146 /*
1147 * allocate an array we need for hat_memload_array.
1148 * we use a separate arena to avoid recursion.
1149 * we will not need this array when hat_memload_array learns pp++
1150 */
1151 if ((ppa = vmem_alloc(segkmem_ppa_arena, ppasize, vmflag)) == NULL) {
1152 goto fail_array_alloc;
1153 }
1154
1155 if (inaddr == NULL && (addr = vmem_alloc(vmp, size, vmflag)) == NULL)
1156 goto fail_vmem_alloc;
1157
1158 ASSERT(((uintptr_t)addr & (lpsize - 1)) == 0);
1159
1160 /* create all the pages */
1161 for (pa = addr, i = 0; i < nlpages; i++, pa += lpsize) {
1162 if ((pp = page_create_func(pa, lpsize, vmflag, pcarg)) == NULL)
1163 goto fail_page_create;
1164 page_list_concat(&pplist, &pp);
1165 }
1166
1167 /* at this point we have all the resource to complete the request */
1168 while ((rootpp = pplist) != NULL) {
1169 for (i = 0; i < nbpages; i++) {
1170 ASSERT(pplist != NULL);
1171 pp = pplist;
1172 page_sub(&pplist, pp);
1173 ASSERT(page_iolock_assert(pp));
1174 page_io_unlock(pp);
1175 ppa[i] = pp;
1176 }
1177 /*
1178 * Load the locked entry. It's OK to preload the entry into the
1179 * TSB since we now support large mappings in the kernel TSB.
1180 */
1181 hat_memload_array(kas.a_hat,
1182 (caddr_t)(uintptr_t)rootpp->p_offset, lpsize,
1183 ppa, (PROT_ALL & ~PROT_USER) | HAT_NOSYNC | attr,
1184 HAT_LOAD_LOCK);
1185
1186 for (--i; i >= 0; --i) {
1187 ppa[i]->p_lckcnt = 1;
1188 page_unlock(ppa[i]);
1189 }
1190 }
1191
1192 vmem_free(segkmem_ppa_arena, ppa, ppasize);
1193 return (addr);
1194
1195 fail_page_create:
1196 while ((rootpp = pplist) != NULL) {
1197 for (i = 0, pp = pplist; i < nbpages; i++, pp = pplist) {
1198 ASSERT(pp != NULL);
1199 page_sub(&pplist, pp);
1200 ASSERT(page_iolock_assert(pp));
1201 page_io_unlock(pp);
1202 }
1203 page_destroy_pages(rootpp);
1204 }
1205
1206 if (inaddr == NULL)
1207 vmem_free(vmp, addr, size);
1208
1209 fail_vmem_alloc:
1210 vmem_free(segkmem_ppa_arena, ppa, ppasize);
1211
1212 fail_array_alloc:
1213 page_unresv(npages);
1214
1215 return (NULL);
1216 }
1217
1218 static void
1219 segkmem_free_one_lp(caddr_t addr, size_t size)
1220 {
1221 page_t *pp, *rootpp = NULL;
1222 pgcnt_t pgs_left = btopr(size);
1223
1224 ASSERT(size == segkmem_lpsize);
1225
1226 hat_unload(kas.a_hat, addr, size, HAT_UNLOAD_UNLOCK);
1227
1228 for (; pgs_left > 0; addr += PAGESIZE, pgs_left--) {
1229 pp = page_lookup(&kvp, (u_offset_t)(uintptr_t)addr, SE_EXCL);
1230 if (pp == NULL)
1231 panic("segkmem_free_one_lp: page not found");
1232 ASSERT(PAGE_EXCL(pp));
1233 pp->p_lckcnt = 0;
1234 if (rootpp == NULL)
1235 rootpp = pp;
1236 }
1237 ASSERT(rootpp != NULL);
1238 page_destroy_pages(rootpp);
1239
1240 /* page_unresv() is done by the caller */
1241 }
1242
1243 /*
1244 * This function is called to import new spans into the vmem arenas like
1245 * kmem_default_arena and kmem_oversize_arena. It first tries to import
1246 * spans from large page arena - kmem_lp_arena. In order to do this it might
1247 * have to "upgrade the requested size" to kmem_lp_arena quantum. If
1248 * it was not able to satisfy the upgraded request it then calls regular
1249 * segkmem_alloc() that satisfies the request by importing from "*vmp" arena
1250 */
1251 /*ARGSUSED*/
1252 void *
1253 segkmem_alloc_lp(vmem_t *vmp, size_t *sizep, size_t align, int vmflag)
1254 {
1255 size_t size;
1256 kthread_t *t = curthread;
1257 segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1258
1259 ASSERT(sizep != NULL);
1260
1261 size = *sizep;
1262
1263 if (lpcb->lp_uselp && !(t->t_flag & T_PANIC) &&
1264 !(vmflag & SEGKMEM_SHARELOCKED)) {
1265
1266 size_t kmemlp_qnt = segkmem_kmemlp_quantum;
1267 size_t asize = P2ROUNDUP(size, kmemlp_qnt);
1268 void *addr = NULL;
1269 ulong_t *lpthrtp = &lpcb->lp_throttle;
1270 ulong_t lpthrt = *lpthrtp;
1271 int dowakeup = 0;
1272 int doalloc = 1;
1273
1274 ASSERT(kmem_lp_arena != NULL);
1275 ASSERT(asize >= size);
1276
1277 if (lpthrt != 0) {
1278 /* try to update the throttle value */
1279 lpthrt = atomic_add_long_nv(lpthrtp, 1);
1280 if (lpthrt >= segkmem_lpthrottle_max) {
1281 lpthrt = atomic_cas_ulong(lpthrtp, lpthrt,
1282 segkmem_lpthrottle_max / 4);
1283 }
1284
1285 /*
1286 * when we get above throttle start do an exponential
1287 * backoff at trying large pages and reaping
1288 */
1289 if (lpthrt > segkmem_lpthrottle_start &&
1290 (lpthrt & (lpthrt - 1))) {
1291 lpcb->allocs_throttled++;
1292 lpthrt--;
1293 if ((lpthrt & (lpthrt - 1)) == 0)
1294 kmem_reap();
1295 return (segkmem_alloc(vmp, size, vmflag));
1296 }
1297 }
1298
1299 if (!(vmflag & VM_NOSLEEP) &&
1300 segkmem_heaplp_quantum >= (8 * kmemlp_qnt) &&
1301 vmem_size(kmem_lp_arena, VMEM_FREE) <= kmemlp_qnt &&
1302 asize < (segkmem_heaplp_quantum - kmemlp_qnt)) {
1303
1304 /*
1305 * we are low on free memory in kmem_lp_arena
1306 * we let only one guy to allocate heap_lp
1307 * quantum size chunk that everybody is going to
1308 * share
1309 */
1310 mutex_enter(&lpcb->lp_lock);
1311
1312 if (lpcb->lp_wait) {
1313
1314 /* we are not the first one - wait */
1315 cv_wait(&lpcb->lp_cv, &lpcb->lp_lock);
1316 if (vmem_size(kmem_lp_arena, VMEM_FREE) <
1317 kmemlp_qnt) {
1318 doalloc = 0;
1319 }
1320 } else if (vmem_size(kmem_lp_arena, VMEM_FREE) <=
1321 kmemlp_qnt) {
1322
1323 /*
1324 * we are the first one, make sure we import
1325 * a large page
1326 */
1327 if (asize == kmemlp_qnt)
1328 asize += kmemlp_qnt;
1329 dowakeup = 1;
1330 lpcb->lp_wait = 1;
1331 }
1332
1333 mutex_exit(&lpcb->lp_lock);
1334 }
1335
1336 /*
1337 * VM_ABORT flag prevents sleeps in vmem_xalloc when
1338 * large pages are not available. In that case this allocation
1339 * attempt will fail and we will retry allocation with small
1340 * pages. We also do not want to panic if this allocation fails
1341 * because we are going to retry.
1342 */
1343 if (doalloc) {
1344 addr = vmem_alloc(kmem_lp_arena, asize,
1345 (vmflag | VM_ABORT) & ~VM_PANIC);
1346
1347 if (dowakeup) {
1348 mutex_enter(&lpcb->lp_lock);
1349 ASSERT(lpcb->lp_wait != 0);
1350 lpcb->lp_wait = 0;
1351 cv_broadcast(&lpcb->lp_cv);
1352 mutex_exit(&lpcb->lp_lock);
1353 }
1354 }
1355
1356 if (addr != NULL) {
1357 *sizep = asize;
1358 *lpthrtp = 0;
1359 return (addr);
1360 }
1361
1362 if (vmflag & VM_NOSLEEP)
1363 lpcb->nosleep_allocs_failed++;
1364 else
1365 lpcb->sleep_allocs_failed++;
1366 lpcb->alloc_bytes_failed += size;
1367
1368 /* if large page throttling is not started yet do it */
1369 if (segkmem_use_lpthrottle && lpthrt == 0) {
1370 lpthrt = atomic_cas_ulong(lpthrtp, lpthrt, 1);
1371 }
1372 }
1373 return (segkmem_alloc(vmp, size, vmflag));
1374 }
1375
1376 void
1377 segkmem_free_lp(vmem_t *vmp, void *inaddr, size_t size)
1378 {
1379 if (kmem_lp_arena == NULL || !IS_KMEM_VA_LARGEPAGE((caddr_t)inaddr)) {
1380 segkmem_free(vmp, inaddr, size);
1381 } else {
1382 vmem_free(kmem_lp_arena, inaddr, size);
1383 }
1384 }
1385
1386 /*
1387 * segkmem_alloc_lpi() imports virtual memory from large page heap arena
1388 * into kmem_lp arena. In the process it maps the imported segment with
1389 * large pages
1390 */
1391 static void *
1392 segkmem_alloc_lpi(vmem_t *vmp, size_t size, int vmflag)
1393 {
1394 segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1395 void *addr;
1396
1397 ASSERT(size != 0);
1398 ASSERT(vmp == heap_lp_arena);
1399
1400 /* do not allow large page heap grow beyound limits */
1401 if (vmem_size(vmp, VMEM_ALLOC) >= segkmem_kmemlp_max) {
1402 lpcb->allocs_limited++;
1403 return (NULL);
1404 }
1405
1406 addr = segkmem_xalloc_lp(vmp, NULL, size, vmflag, 0,
1407 segkmem_page_create_large, NULL);
1408 return (addr);
1409 }
1410
1411 /*
1412 * segkmem_free_lpi() returns virtual memory back into large page heap arena
1413 * from kmem_lp arena. Beore doing this it unmaps the segment and frees
1414 * large pages used to map it.
1415 */
1416 static void
1417 segkmem_free_lpi(vmem_t *vmp, void *inaddr, size_t size)
1418 {
1419 pgcnt_t nlpages = size >> segkmem_lpshift;
1420 size_t lpsize = segkmem_lpsize;
1421 caddr_t addr = inaddr;
1422 pgcnt_t npages = btopr(size);
1423 int i;
1424
1425 ASSERT(vmp == heap_lp_arena);
1426 ASSERT(IS_KMEM_VA_LARGEPAGE(addr));
1427 ASSERT(((uintptr_t)inaddr & (lpsize - 1)) == 0);
1428
1429 for (i = 0; i < nlpages; i++) {
1430 segkmem_free_one_lp(addr, lpsize);
1431 addr += lpsize;
1432 }
1433
1434 page_unresv(npages);
1435
1436 vmem_free(vmp, inaddr, size);
1437 }
1438
1439 /*
1440 * This function is called at system boot time by kmem_init right after
1441 * /etc/system file has been read. It checks based on hardware configuration
1442 * and /etc/system settings if system is going to use large pages. The
1443 * initialiazation necessary to actually start using large pages
1444 * happens later in the process after segkmem_heap_lp_init() is called.
1445 */
1446 int
1447 segkmem_lpsetup()
1448 {
1449 int use_large_pages = 0;
1450
1451 #ifdef __sparc
1452
1453 size_t memtotal = physmem * PAGESIZE;
1454
1455 if (heap_lp_base == NULL) {
1456 segkmem_lpsize = PAGESIZE;
1457 return (0);
1458 }
1459
1460 /* get a platform dependent value of large page size for kernel heap */
1461 segkmem_lpsize = get_segkmem_lpsize(segkmem_lpsize);
1462
1463 if (segkmem_lpsize <= PAGESIZE) {
1464 /*
1465 * put virtual space reserved for the large page kernel
1466 * back to the regular heap
1467 */
1468 vmem_xfree(heap_arena, heap_lp_base,
1469 heap_lp_end - heap_lp_base);
1470 heap_lp_base = NULL;
1471 heap_lp_end = NULL;
1472 segkmem_lpsize = PAGESIZE;
1473 return (0);
1474 }
1475
1476 /* set heap_lp quantum if necessary */
1477 if (segkmem_heaplp_quantum == 0 ||
1478 (segkmem_heaplp_quantum & (segkmem_heaplp_quantum - 1)) ||
1479 P2PHASE(segkmem_heaplp_quantum, segkmem_lpsize)) {
1480 segkmem_heaplp_quantum = segkmem_lpsize;
1481 }
1482
1483 /* set kmem_lp quantum if necessary */
1484 if (segkmem_kmemlp_quantum == 0 ||
1485 (segkmem_kmemlp_quantum & (segkmem_kmemlp_quantum - 1)) ||
1486 segkmem_kmemlp_quantum > segkmem_heaplp_quantum) {
1487 segkmem_kmemlp_quantum = segkmem_heaplp_quantum;
1488 }
1489
1490 /* set total amount of memory allowed for large page kernel heap */
1491 if (segkmem_kmemlp_max == 0) {
1492 if (segkmem_kmemlp_pcnt == 0 || segkmem_kmemlp_pcnt > 100)
1493 segkmem_kmemlp_pcnt = 12;
1494 segkmem_kmemlp_max = (memtotal * segkmem_kmemlp_pcnt) / 100;
1495 }
1496 segkmem_kmemlp_max = P2ROUNDUP(segkmem_kmemlp_max,
1497 segkmem_heaplp_quantum);
1498
1499 /* fix lp kmem preallocation request if necesssary */
1500 if (segkmem_kmemlp_min) {
1501 segkmem_kmemlp_min = P2ROUNDUP(segkmem_kmemlp_min,
1502 segkmem_heaplp_quantum);
1503 if (segkmem_kmemlp_min > segkmem_kmemlp_max)
1504 segkmem_kmemlp_min = segkmem_kmemlp_max;
1505 }
1506
1507 use_large_pages = 1;
1508 segkmem_lpszc = page_szc(segkmem_lpsize);
1509 segkmem_lpshift = page_get_shift(segkmem_lpszc);
1510
1511 #endif
1512 return (use_large_pages);
1513 }
1514
1515 void
1516 segkmem_zio_init(void *zio_mem_base, size_t zio_mem_size)
1517 {
1518 ASSERT(zio_mem_base != NULL);
1519 ASSERT(zio_mem_size != 0);
1520
1521 /*
1522 * To reduce VA space fragmentation, we set up quantum caches for the
1523 * smaller sizes; we chose 32k because that translates to 128k VA
1524 * slabs, which matches nicely with the common 128k zio_data bufs.
1525 */
1526 zio_arena = vmem_create("zfs_file_data", zio_mem_base, zio_mem_size,
1527 PAGESIZE, NULL, NULL, NULL, 32 * 1024, VM_SLEEP);
1528
1529 zio_alloc_arena = vmem_create("zfs_file_data_buf", NULL, 0, PAGESIZE,
1530 segkmem_zio_alloc, segkmem_zio_free, zio_arena, 0, VM_SLEEP);
1531
1532 ASSERT(zio_arena != NULL);
1533 ASSERT(zio_alloc_arena != NULL);
1534 }
1535
1536 #ifdef __sparc
1537
1538
1539 static void *
1540 segkmem_alloc_ppa(vmem_t *vmp, size_t size, int vmflag)
1541 {
1542 size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1543 void *addr;
1544
1545 if (ppaquantum <= PAGESIZE)
1546 return (segkmem_alloc(vmp, size, vmflag));
1547
1548 ASSERT((size & (ppaquantum - 1)) == 0);
1549
1550 addr = vmem_xalloc(vmp, size, ppaquantum, 0, 0, NULL, NULL, vmflag);
1551 if (addr != NULL && segkmem_xalloc(vmp, addr, size, vmflag, 0,
1552 segkmem_page_create, NULL) == NULL) {
1553 vmem_xfree(vmp, addr, size);
1554 addr = NULL;
1555 }
1556
1557 return (addr);
1558 }
1559
1560 static void
1561 segkmem_free_ppa(vmem_t *vmp, void *addr, size_t size)
1562 {
1563 size_t ppaquantum = btopr(segkmem_lpsize) * sizeof (page_t *);
1564
1565 ASSERT(addr != NULL);
1566
1567 if (ppaquantum <= PAGESIZE) {
1568 segkmem_free(vmp, addr, size);
1569 } else {
1570 segkmem_free(NULL, addr, size);
1571 vmem_xfree(vmp, addr, size);
1572 }
1573 }
1574
1575 void
1576 segkmem_heap_lp_init()
1577 {
1578 segkmem_lpcb_t *lpcb = &segkmem_lpcb;
1579 size_t heap_lp_size = heap_lp_end - heap_lp_base;
1580 size_t lpsize = segkmem_lpsize;
1581 size_t ppaquantum;
1582 void *addr;
1583
1584 if (segkmem_lpsize <= PAGESIZE) {
1585 ASSERT(heap_lp_base == NULL);
1586 ASSERT(heap_lp_end == NULL);
1587 return;
1588 }
1589
1590 ASSERT(segkmem_heaplp_quantum >= lpsize);
1591 ASSERT((segkmem_heaplp_quantum & (lpsize - 1)) == 0);
1592 ASSERT(lpcb->lp_uselp == 0);
1593 ASSERT(heap_lp_base != NULL);
1594 ASSERT(heap_lp_end != NULL);
1595 ASSERT(heap_lp_base < heap_lp_end);
1596 ASSERT(heap_lp_arena == NULL);
1597 ASSERT(((uintptr_t)heap_lp_base & (lpsize - 1)) == 0);
1598 ASSERT(((uintptr_t)heap_lp_end & (lpsize - 1)) == 0);
1599
1600 /* create large page heap arena */
1601 heap_lp_arena = vmem_create("heap_lp", heap_lp_base, heap_lp_size,
1602 segkmem_heaplp_quantum, NULL, NULL, NULL, 0, VM_SLEEP);
1603
1604 ASSERT(heap_lp_arena != NULL);
1605
1606 /* This arena caches memory already mapped by large pages */
1607 kmem_lp_arena = vmem_create("kmem_lp", NULL, 0, segkmem_kmemlp_quantum,
1608 segkmem_alloc_lpi, segkmem_free_lpi, heap_lp_arena, 0, VM_SLEEP);
1609
1610 ASSERT(kmem_lp_arena != NULL);
1611
1612 mutex_init(&lpcb->lp_lock, NULL, MUTEX_DEFAULT, NULL);
1613 cv_init(&lpcb->lp_cv, NULL, CV_DEFAULT, NULL);
1614
1615 /*
1616 * this arena is used for the array of page_t pointers necessary
1617 * to call hat_mem_load_array
1618 */
1619 ppaquantum = btopr(lpsize) * sizeof (page_t *);
1620 segkmem_ppa_arena = vmem_create("segkmem_ppa", NULL, 0, ppaquantum,
1621 segkmem_alloc_ppa, segkmem_free_ppa, heap_arena, ppaquantum,
1622 VM_SLEEP);
1623
1624 ASSERT(segkmem_ppa_arena != NULL);
1625
1626 /* prealloacate some memory for the lp kernel heap */
1627 if (segkmem_kmemlp_min) {
1628
1629 ASSERT(P2PHASE(segkmem_kmemlp_min,
1630 segkmem_heaplp_quantum) == 0);
1631
1632 if ((addr = segkmem_alloc_lpi(heap_lp_arena,
1633 segkmem_kmemlp_min, VM_SLEEP)) != NULL) {
1634
1635 addr = vmem_add(kmem_lp_arena, addr,
1636 segkmem_kmemlp_min, VM_SLEEP);
1637 ASSERT(addr != NULL);
1638 }
1639 }
1640
1641 lpcb->lp_uselp = 1;
1642 }
1643
1644 #endif