1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21 /*
22 * Copyright (c) 1986, 2010, Oracle and/or its affiliates. All rights reserved.
23 */
24
25 /* Copyright (c) 1984, 1986, 1987, 1988, 1989 AT&T */
26 /* All Rights Reserved */
27
28 /*
29 * University Copyright- Copyright (c) 1982, 1986, 1988
30 * The Regents of the University of California
31 * All Rights Reserved
32 *
33 * University Acknowledgment- Portions of this document are derived from
34 * software developed by the University of California, Berkeley, and its
35 * contributors.
36 */
37
38 #ifndef _VM_ANON_H
39 #define _VM_ANON_H
40
41 #include <sys/cred.h>
42 #include <sys/zone.h>
43 #include <vm/seg.h>
44 #include <vm/vpage.h>
45
46 #ifdef __cplusplus
47 extern "C" {
48 #endif
49
50 /*
51 * VM - Anonymous pages.
52 */
53
54 typedef unsigned long anoff_t; /* anon offsets */
55
56 /*
57 * Each anonymous page, either in memory or in swap, has an anon structure.
58 * The structure (slot) provides a level of indirection between anonymous pages
59 * and their backing store.
60 *
61 * (an_vp, an_off) names the vnode of the anonymous page for this slot.
62 *
63 * (an_pvp, an_poff) names the location of the physical backing store
64 * for the page this slot represents. If the name is null there is no
65 * associated physical store. The physical backing store location can
66 * change while the slot is in use.
67 *
68 * an_hash is a hash list of anon slots. The list is hashed by
69 * (an_vp, an_off) of the associated anonymous page and provides a
70 * method of going from the name of an anonymous page to its
71 * associated anon slot.
72 *
73 * an_refcnt holds a reference count which is the number of separate
74 * copies that will need to be created in case of copy-on-write.
75 * A refcnt > 0 protects the existence of the slot. The refcnt is
76 * initialized to 1 when the anon slot is created in anon_alloc().
77 * If a client obtains an anon slot and allows multiple threads to
78 * share it, then it is the client's responsibility to insure that
79 * it does not allow one thread to try to reference the slot at the
80 * same time as another is trying to decrement the last count and
81 * destroy the anon slot. E.g., the seg_vn segment type protects
82 * against this with higher level locks.
83 */
84
85 struct anon {
86 struct vnode *an_vp; /* vnode of anon page */
87 struct vnode *an_pvp; /* vnode of physical backing store */
88 anoff_t an_off; /* offset of anon page */
89 anoff_t an_poff; /* offset in vnode */
90 struct anon *an_hash; /* hash table of anon slots */
91 int an_refcnt; /* # of people sharing slot */
92 };
93
94 #define AN_CACHE_ALIGN_LOG2 4 /* log2(AN_CACHE_ALIGN) */
95 #define AN_CACHE_ALIGN (1U << AN_CACHE_ALIGN_LOG2) /* anon address aligned */
96 /* 16 bytes */
97
98
99 #ifdef _KERNEL
100 /*
101 * The swapinfo_lock protects:
102 * swapinfo list
103 * individual swapinfo structures
104 *
105 * The anoninfo_lock protects:
106 * anoninfo counters
107 *
108 * The anonhash_lock protects:
109 * anon hash lists
110 * anon slot fields
111 *
112 * Fields in the anon slot which are read-only for the life of the slot
113 * (an_vp, an_off) do not require the anonhash_lock be held to access them.
114 * If you access a field without the anonhash_lock held you must be holding
115 * the slot with an_refcnt to make sure it isn't destroyed.
116 * To write (an_pvp, an_poff) in a given slot you must also hold the
117 * p_iolock of the anonymous page for slot.
118 */
119 extern kmutex_t anoninfo_lock;
120 extern kmutex_t swapinfo_lock;
121 extern pad_mutex_t *anonhash_lock;
122 extern pad_mutex_t anon_array_lock[];
123 extern kcondvar_t anon_array_cv[];
124
125 /*
126 * Global hash table to provide a function from (vp, off) -> ap
127 */
128 extern size_t anon_hash_size;
129 extern unsigned int anon_hash_shift;
130 extern struct anon **anon_hash;
131 #define ANON_HASH_SIZE anon_hash_size
132 #define ANON_HASHAVELEN 4
133 /*
134 * Try to use as many bits of randomness from both vp and off as we can.
135 * This should help spreading evenly for a variety of workloads. See comments
136 * for PAGE_HASH_FUNC for more explanation.
137 */
138 #define ANON_HASH(vp, off) \
139 (((((uintptr_t)(off) >> PAGESHIFT) ^ \
140 ((uintptr_t)(off) >> (PAGESHIFT + anon_hash_shift))) ^ \
141 (((uintptr_t)(vp) >> 3) ^ \
142 ((uintptr_t)(vp) >> (3 + anon_hash_shift)) ^ \
143 ((uintptr_t)(vp) >> (3 + 2 * anon_hash_shift)) ^ \
144 ((uintptr_t)(vp) << \
145 (anon_hash_shift - AN_VPSHIFT - VNODE_ALIGN_LOG2)))) & \
146 (anon_hash_size - 1))
147
148 #define AH_LOCK_SIZE (2 << NCPU_LOG2)
149
150 #define AH_MUTEX(vp, off) \
151 (&anonhash_lock[(ANON_HASH((vp), (off)) & \
152 (AH_LOCK_SIZE - 1))].pad_mutex)
153
154 #endif /* _KERNEL */
155
156 /*
157 * Declaration for the Global counters to accurately
158 * track the kernel foot print in memory.
159 */
160 extern pgcnt_t pages_locked;
161 extern pgcnt_t pages_claimed;
162 extern pgcnt_t pages_useclaim;
163 extern pgcnt_t obp_pages;
164
165 /*
166 * Anonymous backing store accounting structure for swapctl.
167 *
168 * ani_max = maximum amount of swap space
169 * (including potentially available physical memory)
170 * ani_free = amount of unallocated anonymous memory
171 * (some of which might be reserved and including
172 * potentially available physical memory)
173 * ani_resv = amount of claimed (reserved) anonymous memory
174 *
175 * The swap data can be aquired more efficiently through the
176 * kstats interface.
177 * Total slots currently available for reservation =
178 * MAX(ani_max - ani_resv, 0) + (availrmem - swapfs_minfree)
179 */
180 struct anoninfo {
181 pgcnt_t ani_max;
182 pgcnt_t ani_free;
183 pgcnt_t ani_resv;
184 };
185
186 #ifdef _SYSCALL32
187 struct anoninfo32 {
188 size32_t ani_max;
189 size32_t ani_free;
190 size32_t ani_resv;
191 };
192 #endif /* _SYSCALL32 */
193
194 /*
195 * Define the NCPU pool of the ani_free counters. Update the counter
196 * of the cpu on which the thread is running and in every clock intr
197 * sync anoninfo.ani_free with the current total off all the NCPU entries.
198 */
199
200 typedef struct ani_free {
201 pgcnt_t ani_count;
202 uchar_t pad[64 - sizeof (pgcnt_t)];
203 /* XXX 64 = cacheline size */
204 } ani_free_t;
205
206 #define ANI_MAX_POOL (NCPU_P2)
207 extern ani_free_t *ani_free_pool;
208
209 /*
210 * Since each CPU has its own bucket in ani_free_pool, there should be no
211 * contention here.
212 */
213 #define ANI_ADD(inc) { \
214 pgcnt_t *ani_countp; \
215 int index; \
216 index = (CPU->cpu_seqid & (ANI_MAX_POOL - 1)); \
217 ani_countp = &ani_free_pool[index].ani_count; \
218 atomic_add_long(ani_countp, inc); \
219 }
220
221 extern void set_anoninfo(void);
222
223 /*
224 * Anon array pointers are allocated in chunks. Each chunk
225 * has PAGESIZE/sizeof(u_long *) of anon pointers.
226 * There are two levels of arrays for anon array pointers larger
227 * than a chunk. The first level points to anon array chunks.
228 * The second level consists of chunks of anon pointers.
229 *
230 * If anon array is smaller than a chunk then the whole anon array
231 * is created (memory is allocated for whole anon array).
232 * If anon array is larger than a chunk only first level array is
233 * allocated. Then other arrays (chunks) are allocated only when
234 * they are initialized with anon pointers.
235 */
236 struct anon_hdr {
237 kmutex_t serial_lock; /* serialize array chunk allocation */
238 pgcnt_t size; /* number of pointers to (anon) pages */
239 void **array_chunk; /* pointers to anon pointers or chunks of */
240 /* anon pointers */
241 int flags; /* ANON_ALLOC_FORCE force preallocation of */
242 /* whole anon array */
243 };
244
245 #ifdef _LP64
246 #define ANON_PTRSHIFT 3
247 #define ANON_PTRMASK ~7
248 #else
249 #define ANON_PTRSHIFT 2
250 #define ANON_PTRMASK ~3
251 #endif
252
253 #define ANON_CHUNK_SIZE (PAGESIZE >> ANON_PTRSHIFT)
254 #define ANON_CHUNK_SHIFT (PAGESHIFT - ANON_PTRSHIFT)
255 #define ANON_CHUNK_OFF (ANON_CHUNK_SIZE - 1)
256
257 /*
258 * Anon flags.
259 */
260 #define ANON_SLEEP 0x0 /* ok to block */
261 #define ANON_NOSLEEP 0x1 /* non-blocking call */
262 #define ANON_ALLOC_FORCE 0x2 /* force single level anon array */
263 #define ANON_GROWDOWN 0x4 /* anon array should grow downward */
264
265 struct kshmid;
266
267 /*
268 * The anon_map structure is used by various clients of the anon layer to
269 * manage anonymous memory. When anonymous memory is shared,
270 * then the different clients sharing it will point to the
271 * same anon_map structure. Also, if a segment is unmapped
272 * in the middle where an anon_map structure exists, the
273 * newly created segment will also share the anon_map structure,
274 * although the two segments will use different ranges of the
275 * anon array. When mappings are private (or shared with
276 * a reference count of 1), an unmap operation will free up
277 * a range of anon slots in the array given by the anon_map
278 * structure. Because of fragmentation due to this unmapping,
279 * we have to store the size of the anon array in the anon_map
280 * structure so that we can free everything when the referernce
281 * count goes to zero.
282 *
283 * A new rangelock scheme is introduced to make the anon layer scale.
284 * A reader/writer lock per anon_amp and an array of system-wide hash
285 * locks, anon_array_lock[] are introduced to replace serial_lock and
286 * anonmap lock. The writer lock is held when we want to singlethreaD
287 * the reference to the anon array pointers or when references to
288 * anon_map's members, whereas reader lock and anon_array_lock are
289 * held to allows multiple threads to reference different part of
290 * anon array. A global set of condition variables, anon_array_cv,
291 * are used with anon_array_lock[] to make the hold time of the locks
292 * short.
293 *
294 * szc is used to calculate the index of hash locks and cv's. We
295 * could've just used seg->s_szc if not for the possible sharing of
296 * anon_amp between SYSV shared memory and ISM, so now we introduce
297 * szc in the anon_map structure. For MAP_SHARED, the amp->szc is either
298 * 0 (base page size) or page_num_pagesizes() - 1, while MAP_PRIVATE
299 * the amp->szc could be anything in [0, page_num_pagesizes() - 1].
300 */
301 typedef struct anon_map {
302 krwlock_t a_rwlock; /* protect anon_map and anon array */
303 size_t size; /* size in bytes mapped by the anon array */
304 struct anon_hdr *ahp; /* anon array header pointer, containing */
305 /* anon pointer array(s) */
306 size_t swresv; /* swap space reserved for this anon_map */
307 ulong_t refcnt; /* reference count on this structure */
308 ushort_t a_szc; /* max szc among shared processes */
309 void *locality; /* lgroup locality info */
310 struct kshmid *a_sp; /* kshmid if amp backs sysV, or NULL */
311 int a_purgewait; /* somebody waits for slocks to go away */
312 kcondvar_t a_purgecv; /* cv for waiting for slocks to go away */
313 kmutex_t a_purgemtx; /* mutex for anonmap_purge() */
314 spgcnt_t a_softlockcnt; /* number of pages locked in pcache */
315 kmutex_t a_pmtx; /* protects amp's pcache list */
316 pcache_link_t a_phead; /* head of amp's pcache list */
317 } amp_t;
318
319 #ifdef _KERNEL
320
321 #define ANON_BUSY 0x1
322 #define ANON_ISBUSY(slot) (*(slot) & ANON_BUSY)
323 #define ANON_SETBUSY(slot) (*(slot) |= ANON_BUSY)
324 #define ANON_CLRBUSY(slot) (*(slot) &= ~ANON_BUSY)
325
326 #define ANON_MAP_SHIFT 6 /* log2(sizeof (struct anon_map)) */
327 #define ANON_ARRAY_SHIFT 7 /* log2(ANON_LOCKSIZE) */
328 #define ANON_LOCKSIZE 128
329
330 #define ANON_LOCK_ENTER(lock, type) rw_enter((lock), (type))
331 #define ANON_LOCK_EXIT(lock) rw_exit((lock))
332 #define ANON_LOCK_HELD(lock) RW_LOCK_HELD((lock))
333 #define ANON_READ_HELD(lock) RW_READ_HELD((lock))
334 #define ANON_WRITE_HELD(lock) RW_WRITE_HELD((lock))
335
336 #define ANON_ARRAY_HASH(amp, idx)\
337 ((((idx) + ((idx) >> ANON_ARRAY_SHIFT) +\
338 ((idx) >> (ANON_ARRAY_SHIFT << 1)) +\
339 ((idx) >> (ANON_ARRAY_SHIFT + (ANON_ARRAY_SHIFT << 1)))) ^\
340 ((uintptr_t)(amp) >> ANON_MAP_SHIFT)) & (ANON_LOCKSIZE - 1))
341
342 typedef struct anon_sync_obj {
343 kmutex_t *sync_mutex;
344 kcondvar_t *sync_cv;
345 ulong_t *sync_data;
346 } anon_sync_obj_t;
347
348 /*
349 * Anonymous backing store accounting structure for kernel.
350 * ani_max = total reservable slots on physical (disk-backed) swap
351 * ani_phys_resv = total phys slots reserved for use by clients
352 * ani_mem_resv = total mem slots reserved for use by clients
353 * ani_free = # unallocated physical slots + # of reserved unallocated
354 * memory slots
355 */
356
357 /*
358 * Initial total swap slots available for reservation
359 */
360 #define TOTAL_AVAILABLE_SWAP \
361 (k_anoninfo.ani_max + MAX((spgcnt_t)(availrmem - swapfs_minfree), 0))
362
363 /*
364 * Swap slots currently available for reservation
365 */
366 #define CURRENT_TOTAL_AVAILABLE_SWAP \
367 ((k_anoninfo.ani_max - k_anoninfo.ani_phys_resv) + \
368 MAX((spgcnt_t)(availrmem - swapfs_minfree), 0))
369
370 struct k_anoninfo {
371 pgcnt_t ani_max; /* total reservable slots on phys */
372 /* (disk) swap */
373 pgcnt_t ani_free; /* # of unallocated phys and mem slots */
374 pgcnt_t ani_phys_resv; /* # of reserved phys (disk) slots */
375 pgcnt_t ani_mem_resv; /* # of reserved mem slots */
376 pgcnt_t ani_locked_swap; /* # of swap slots locked in reserved */
377 /* mem swap */
378 };
379
380 extern struct k_anoninfo k_anoninfo;
381
382 extern void anon_init(void);
383 extern struct anon *anon_alloc(struct vnode *, anoff_t);
384 extern void anon_dup(struct anon_hdr *, ulong_t,
385 struct anon_hdr *, ulong_t, size_t);
386 extern void anon_dup_fill_holes(struct anon_hdr *, ulong_t,
387 struct anon_hdr *, ulong_t, size_t, uint_t, int);
388 extern int anon_fill_cow_holes(struct seg *, caddr_t, struct anon_hdr *,
389 ulong_t, struct vnode *, u_offset_t, size_t, uint_t,
390 uint_t, struct vpage [], struct cred *);
391 extern void anon_free(struct anon_hdr *, ulong_t, size_t);
392 extern void anon_free_pages(struct anon_hdr *, ulong_t, size_t, uint_t);
393 extern void anon_disclaim(struct anon_map *, ulong_t, size_t);
394 extern int anon_getpage(struct anon **, uint_t *, struct page **,
395 size_t, struct seg *, caddr_t, enum seg_rw, struct cred *);
396 extern int swap_getconpage(struct vnode *, u_offset_t, size_t,
397 uint_t *, page_t *[], size_t, page_t *, uint_t *,
398 spgcnt_t *, struct seg *, caddr_t,
399 enum seg_rw, struct cred *);
400 extern int anon_map_getpages(struct anon_map *, ulong_t,
401 uint_t, struct seg *, caddr_t, uint_t,
402 uint_t *, page_t *[], uint_t *,
403 struct vpage [], enum seg_rw, int, int, int, struct cred *);
404 extern int anon_map_privatepages(struct anon_map *, ulong_t,
405 uint_t, struct seg *, caddr_t, uint_t,
406 page_t *[], struct vpage [], int, int, struct cred *);
407 extern struct page *anon_private(struct anon **, struct seg *,
408 caddr_t, uint_t, struct page *,
409 int, struct cred *);
410 extern struct page *anon_zero(struct seg *, caddr_t,
411 struct anon **, struct cred *);
412 extern int anon_map_createpages(struct anon_map *, ulong_t,
413 size_t, struct page **,
414 struct seg *, caddr_t,
415 enum seg_rw, struct cred *);
416 extern int anon_map_demotepages(struct anon_map *, ulong_t,
417 struct seg *, caddr_t, uint_t,
418 struct vpage [], struct cred *);
419 extern void anon_shmap_free_pages(struct anon_map *, ulong_t, size_t);
420 extern int anon_resvmem(size_t, boolean_t, zone_t *, int);
421 extern void anon_unresvmem(size_t, zone_t *);
422 extern struct anon_map *anonmap_alloc(size_t, size_t, int);
423 extern void anonmap_free(struct anon_map *);
424 extern void anonmap_purge(struct anon_map *);
425 extern void anon_swap_free(struct anon *, struct page *);
426 extern void anon_decref(struct anon *);
427 extern int non_anon(struct anon_hdr *, ulong_t, u_offset_t *, size_t *);
428 extern pgcnt_t anon_pages(struct anon_hdr *, ulong_t, pgcnt_t);
429 extern int anon_swap_adjust(pgcnt_t);
430 extern void anon_swap_restore(pgcnt_t);
431 extern struct anon_hdr *anon_create(pgcnt_t, int);
432 extern void anon_release(struct anon_hdr *, pgcnt_t);
433 extern struct anon *anon_get_ptr(struct anon_hdr *, ulong_t);
434 extern ulong_t *anon_get_slot(struct anon_hdr *, ulong_t);
435 extern struct anon *anon_get_next_ptr(struct anon_hdr *, ulong_t *);
436 extern int anon_set_ptr(struct anon_hdr *, ulong_t, struct anon *, int);
437 extern int anon_copy_ptr(struct anon_hdr *, ulong_t,
438 struct anon_hdr *, ulong_t, pgcnt_t, int);
439 extern pgcnt_t anon_grow(struct anon_hdr *, ulong_t *, pgcnt_t, pgcnt_t, int);
440 extern void anon_array_enter(struct anon_map *, ulong_t,
441 anon_sync_obj_t *);
442 extern int anon_array_try_enter(struct anon_map *, ulong_t,
443 anon_sync_obj_t *);
444 extern void anon_array_exit(anon_sync_obj_t *);
445
446 /*
447 * anon_resv checks to see if there is enough swap space to fulfill a
448 * request and if so, reserves the appropriate anonymous memory resources.
449 * anon_checkspace just checks to see if there is space to fulfill the request,
450 * without taking any resources. Both return 1 if successful and 0 if not.
451 *
452 * Macros are provided as anon reservation is usually charged to the zone of
453 * the current process. In some cases (such as anon reserved by tmpfs), a
454 * zone pointer is needed to charge the appropriate zone.
455 */
456 #define anon_unresv(size) anon_unresvmem(size, curproc->p_zone)
457 #define anon_unresv_zone(size, zone) anon_unresvmem(size, zone)
458 #define anon_resv(size) \
459 anon_resvmem((size), 1, curproc->p_zone, 1)
460 #define anon_resv_zone(size, zone) anon_resvmem((size), 1, zone, 1)
461 #define anon_checkspace(size, zone) anon_resvmem((size), 0, zone, 0)
462 #define anon_try_resv_zone(size, zone) anon_resvmem((size), 1, zone, 0)
463
464 /*
465 * Flags to anon_private
466 */
467 #define STEAL_PAGE 0x1 /* page can be stolen */
468 #define LOCK_PAGE 0x2 /* page must be ``logically'' locked */
469
470 /*
471 * SEGKP ANON pages that are locked are assumed to be LWP stack pages
472 * and thus count towards the user pages locked count.
473 * This value is protected by the same lock as availrmem.
474 */
475 extern pgcnt_t anon_segkp_pages_locked;
476
477 extern int anon_debug;
478
479 #ifdef ANON_DEBUG
480
481 #define A_ANON 0x01
482 #define A_RESV 0x02
483 #define A_MRESV 0x04
484
485 /* vararg-like debugging macro. */
486 #define ANON_PRINT(f, printf_args) \
487 if (anon_debug & f) \
488 printf printf_args
489
490 #else /* ANON_DEBUG */
491
492 #define ANON_PRINT(f, printf_args)
493
494 #endif /* ANON_DEBUG */
495
496 #endif /* _KERNEL */
497
498 #ifdef __cplusplus
499 }
500 #endif
501
502 #endif /* _VM_ANON_H */