1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 1993, 2010, Oracle and/or its affiliates. All rights reserved.
  23  */
  24 
  25 #include <sys/param.h>
  26 #include <sys/user.h>
  27 #include <sys/mman.h>
  28 #include <sys/kmem.h>
  29 #include <sys/sysmacros.h>
  30 #include <sys/cmn_err.h>
  31 #include <sys/systm.h>
  32 #include <sys/tuneable.h>
  33 #include <vm/hat.h>
  34 #include <vm/seg.h>
  35 #include <vm/as.h>
  36 #include <vm/anon.h>
  37 #include <vm/page.h>
  38 #include <sys/buf.h>
  39 #include <sys/swap.h>
  40 #include <sys/atomic.h>
  41 #include <vm/seg_spt.h>
  42 #include <sys/debug.h>
  43 #include <sys/vtrace.h>
  44 #include <sys/shm.h>
  45 #include <sys/shm_impl.h>
  46 #include <sys/lgrp.h>
  47 #include <sys/vmsystm.h>
  48 #include <sys/policy.h>
  49 #include <sys/project.h>
  50 #include <sys/tnf_probe.h>
  51 #include <sys/zone.h>
  52 
  53 #define SEGSPTADDR      (caddr_t)0x0
  54 
  55 /*
  56  * # pages used for spt
  57  */
  58 size_t  spt_used;
  59 
  60 /*
  61  * segspt_minfree is the memory left for system after ISM
  62  * locked its pages; it is set up to 5% of availrmem in
  63  * sptcreate when ISM is created.  ISM should not use more
  64  * than ~90% of availrmem; if it does, then the performance
  65  * of the system may decrease. Machines with large memories may
  66  * be able to use up more memory for ISM so we set the default
  67  * segspt_minfree to 5% (which gives ISM max 95% of availrmem.
  68  * If somebody wants even more memory for ISM (risking hanging
  69  * the system) they can patch the segspt_minfree to smaller number.
  70  */
  71 pgcnt_t segspt_minfree = 0;
  72 
  73 static int segspt_create(struct seg *seg, caddr_t argsp);
  74 static int segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize);
  75 static void segspt_free(struct seg *seg);
  76 static void segspt_free_pages(struct seg *seg, caddr_t addr, size_t len);
  77 static lgrp_mem_policy_info_t *segspt_getpolicy(struct seg *seg, caddr_t addr);
  78 
  79 static void
  80 segspt_badop()
  81 {
  82         panic("segspt_badop called");
  83         /*NOTREACHED*/
  84 }
  85 
  86 #define SEGSPT_BADOP(t) (t(*)())segspt_badop
  87 
  88 static const struct seg_ops segspt_ops = {
  89         .dup            = SEGSPT_BADOP(int),
  90         .unmap          = segspt_unmap,
  91         .free           = segspt_free,
  92         .fault          = SEGSPT_BADOP(int),
  93         .faulta         = SEGSPT_BADOP(faultcode_t),
  94         .setprot        = SEGSPT_BADOP(int),
  95         .checkprot      = SEGSPT_BADOP(int),
  96         .kluster        = SEGSPT_BADOP(int),
  97         .swapout        = SEGSPT_BADOP(size_t),
  98         .sync           = SEGSPT_BADOP(int),
  99         .incore         = SEGSPT_BADOP(size_t),
 100         .lockop         = SEGSPT_BADOP(int),
 101         .getprot        = SEGSPT_BADOP(int),
 102         .getoffset      = SEGSPT_BADOP(u_offset_t),
 103         .gettype        = SEGSPT_BADOP(int),
 104         .getvp          = SEGSPT_BADOP(int),
 105         .advise         = SEGSPT_BADOP(int),
 106         .dump           = SEGSPT_BADOP(void),
 107         .pagelock       = SEGSPT_BADOP(int),
 108         .setpagesize    = SEGSPT_BADOP(int),
 109         .getmemid       = SEGSPT_BADOP(int),
 110         .getpolicy      = segspt_getpolicy,
 111         .capable        = SEGSPT_BADOP(int),
 112 };
 113 
 114 static int segspt_shmdup(struct seg *seg, struct seg *newseg);
 115 static int segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize);
 116 static void segspt_shmfree(struct seg *seg);
 117 static faultcode_t segspt_shmfault(struct hat *hat, struct seg *seg,
 118                 caddr_t addr, size_t len, enum fault_type type, enum seg_rw rw);
 119 static faultcode_t segspt_shmfaulta(struct seg *seg, caddr_t addr);
 120 static int segspt_shmsetprot(register struct seg *seg, register caddr_t addr,
 121                         register size_t len, register uint_t prot);
 122 static int segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size,
 123                         uint_t prot);
 124 static int      segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta);
 125 static size_t   segspt_shmswapout(struct seg *seg);
 126 static size_t segspt_shmincore(struct seg *seg, caddr_t addr, size_t len,
 127                         register char *vec);
 128 static int segspt_shmsync(struct seg *seg, register caddr_t addr, size_t len,
 129                         int attr, uint_t flags);
 130 static int segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len,
 131                         int attr, int op, ulong_t *lockmap, size_t pos);
 132 static int segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len,
 133                         uint_t *protv);
 134 static u_offset_t segspt_shmgetoffset(struct seg *seg, caddr_t addr);
 135 static int segspt_shmgettype(struct seg *seg, caddr_t addr);
 136 static int segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp);
 137 static int segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len,
 138                         uint_t behav);
 139 static int segspt_shmpagelock(struct seg *, caddr_t, size_t,
 140                         struct page ***, enum lock_type, enum seg_rw);
 141 static int segspt_shmgetmemid(struct seg *, caddr_t, memid_t *);
 142 static lgrp_mem_policy_info_t *segspt_shmgetpolicy(struct seg *, caddr_t);
 143 
 144 const struct seg_ops segspt_shmops = {
 145         .dup            = segspt_shmdup,
 146         .unmap          = segspt_shmunmap,
 147         .free           = segspt_shmfree,
 148         .fault          = segspt_shmfault,
 149         .faulta         = segspt_shmfaulta,
 150         .setprot        = segspt_shmsetprot,
 151         .checkprot      = segspt_shmcheckprot,
 152         .kluster        = segspt_shmkluster,
 153         .swapout        = segspt_shmswapout,
 154         .sync           = segspt_shmsync,
 155         .incore         = segspt_shmincore,
 156         .lockop         = segspt_shmlockop,
 157         .getprot        = segspt_shmgetprot,
 158         .getoffset      = segspt_shmgetoffset,
 159         .gettype        = segspt_shmgettype,
 160         .getvp          = segspt_shmgetvp,
 161         .advise         = segspt_shmadvise,
 162         .pagelock       = segspt_shmpagelock,
 163         .getmemid       = segspt_shmgetmemid,
 164         .getpolicy      = segspt_shmgetpolicy,
 165 };
 166 
 167 static void segspt_purge(struct seg *seg);
 168 static int segspt_reclaim(void *, caddr_t, size_t, struct page **,
 169                 enum seg_rw, int);
 170 static int spt_anon_getpages(struct seg *seg, caddr_t addr, size_t len,
 171                 page_t **ppa);
 172 
 173 
 174 
 175 /*ARGSUSED*/
 176 int
 177 sptcreate(size_t size, struct seg **sptseg, struct anon_map *amp,
 178         uint_t prot, uint_t flags, uint_t share_szc)
 179 {
 180         int     err;
 181         struct  as      *newas;
 182         struct  segspt_crargs sptcargs;
 183 
 184 #ifdef DEBUG
 185         TNF_PROBE_1(sptcreate, "spt", /* CSTYLED */,
 186                         tnf_ulong, size, size );
 187 #endif
 188         if (segspt_minfree == 0)        /* leave min 5% of availrmem for */
 189                 segspt_minfree = availrmem/20;  /* for the system */
 190 
 191         if (!hat_supported(HAT_SHARED_PT, (void *)0))
 192                 return (EINVAL);
 193 
 194         /*
 195          * get a new as for this shared memory segment
 196          */
 197         newas = as_alloc();
 198         newas->a_proc = NULL;
 199         sptcargs.amp = amp;
 200         sptcargs.prot = prot;
 201         sptcargs.flags = flags;
 202         sptcargs.szc = share_szc;
 203         /*
 204          * create a shared page table (spt) segment
 205          */
 206 
 207         if (err = as_map(newas, SEGSPTADDR, size, segspt_create, &sptcargs)) {
 208                 as_free(newas);
 209                 return (err);
 210         }
 211         *sptseg = sptcargs.seg_spt;
 212         return (0);
 213 }
 214 
 215 void
 216 sptdestroy(struct as *as, struct anon_map *amp)
 217 {
 218 
 219 #ifdef DEBUG
 220         TNF_PROBE_0(sptdestroy, "spt", /* CSTYLED */);
 221 #endif
 222         (void) as_unmap(as, SEGSPTADDR, amp->size);
 223         as_free(as);
 224 }
 225 
 226 /*
 227  * called from seg_free().
 228  * free (i.e., unlock, unmap, return to free list)
 229  *  all the pages in the given seg.
 230  */
 231 void
 232 segspt_free(struct seg  *seg)
 233 {
 234         struct spt_data *sptd = (struct spt_data *)seg->s_data;
 235 
 236         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
 237 
 238         if (sptd != NULL) {
 239                 if (sptd->spt_realsize)
 240                         segspt_free_pages(seg, seg->s_base, sptd->spt_realsize);
 241 
 242         if (sptd->spt_ppa_lckcnt)
 243                 kmem_free(sptd->spt_ppa_lckcnt,
 244                     sizeof (*sptd->spt_ppa_lckcnt)
 245                     * btopr(sptd->spt_amp->size));
 246                 kmem_free(sptd->spt_vp, sizeof (*sptd->spt_vp));
 247                 cv_destroy(&sptd->spt_cv);
 248                 mutex_destroy(&sptd->spt_lock);
 249                 kmem_free(sptd, sizeof (*sptd));
 250         }
 251 }
 252 
 253 /*ARGSUSED*/
 254 static int
 255 segspt_shmsync(struct seg *seg, caddr_t addr, size_t len, int attr,
 256         uint_t flags)
 257 {
 258         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
 259 
 260         return (0);
 261 }
 262 
 263 /*ARGSUSED*/
 264 static size_t
 265 segspt_shmincore(struct seg *seg, caddr_t addr, size_t len, char *vec)
 266 {
 267         caddr_t eo_seg;
 268         pgcnt_t npages;
 269         struct shm_data *shmd = (struct shm_data *)seg->s_data;
 270         struct seg      *sptseg;
 271         struct spt_data *sptd;
 272 
 273         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
 274 #ifdef lint
 275         seg = seg;
 276 #endif
 277         sptseg = shmd->shm_sptseg;
 278         sptd = sptseg->s_data;
 279 
 280         if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
 281                 eo_seg = addr + len;
 282                 while (addr < eo_seg) {
 283                         /* page exists, and it's locked. */
 284                         *vec++ = SEG_PAGE_INCORE | SEG_PAGE_LOCKED |
 285                             SEG_PAGE_ANON;
 286                         addr += PAGESIZE;
 287                 }
 288                 return (len);
 289         } else {
 290                 struct  anon_map *amp = shmd->shm_amp;
 291                 struct  anon    *ap;
 292                 page_t          *pp;
 293                 pgcnt_t         anon_index;
 294                 struct vnode    *vp;
 295                 u_offset_t      off;
 296                 ulong_t         i;
 297                 int             ret;
 298                 anon_sync_obj_t cookie;
 299 
 300                 addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
 301                 anon_index = seg_page(seg, addr);
 302                 npages = btopr(len);
 303                 if (anon_index + npages > btopr(shmd->shm_amp->size)) {
 304                         return (EINVAL);
 305                 }
 306                 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
 307                 for (i = 0; i < npages; i++, anon_index++) {
 308                         ret = 0;
 309                         anon_array_enter(amp, anon_index, &cookie);
 310                         ap = anon_get_ptr(amp->ahp, anon_index);
 311                         if (ap != NULL) {
 312                                 swap_xlate(ap, &vp, &off);
 313                                 anon_array_exit(&cookie);
 314                                 pp = page_lookup_nowait(vp, off, SE_SHARED);
 315                                 if (pp != NULL) {
 316                                         ret |= SEG_PAGE_INCORE | SEG_PAGE_ANON;
 317                                         page_unlock(pp);
 318                                 }
 319                         } else {
 320                                 anon_array_exit(&cookie);
 321                         }
 322                         if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) {
 323                                 ret |= SEG_PAGE_LOCKED;
 324                         }
 325                         *vec++ = (char)ret;
 326                 }
 327                 ANON_LOCK_EXIT(&amp->a_rwlock);
 328                 return (len);
 329         }
 330 }
 331 
 332 static int
 333 segspt_unmap(struct seg *seg, caddr_t raddr, size_t ssize)
 334 {
 335         size_t share_size;
 336 
 337         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
 338 
 339         /*
 340          * seg.s_size may have been rounded up to the largest page size
 341          * in shmat().
 342          * XXX This should be cleanedup. sptdestroy should take a length
 343          * argument which should be the same as sptcreate. Then
 344          * this rounding would not be needed (or is done in shm.c)
 345          * Only the check for full segment will be needed.
 346          *
 347          * XXX -- shouldn't raddr == 0 always? These tests don't seem
 348          * to be useful at all.
 349          */
 350         share_size = page_get_pagesize(seg->s_szc);
 351         ssize = P2ROUNDUP(ssize, share_size);
 352 
 353         if (raddr == seg->s_base && ssize == seg->s_size) {
 354                 seg_free(seg);
 355                 return (0);
 356         } else
 357                 return (EINVAL);
 358 }
 359 
 360 int
 361 segspt_create(struct seg *seg, caddr_t argsp)
 362 {
 363         int             err;
 364         caddr_t         addr = seg->s_base;
 365         struct spt_data *sptd;
 366         struct  segspt_crargs *sptcargs = (struct segspt_crargs *)argsp;
 367         struct anon_map *amp = sptcargs->amp;
 368         struct kshmid   *sp = amp->a_sp;
 369         struct  cred    *cred = CRED();
 370         ulong_t         i, j, anon_index = 0;
 371         pgcnt_t         npages = btopr(amp->size);
 372         struct vnode    *vp;
 373         page_t          **ppa;
 374         uint_t          hat_flags;
 375         size_t          pgsz;
 376         pgcnt_t         pgcnt;
 377         caddr_t         a;
 378         pgcnt_t         pidx;
 379         size_t          sz;
 380         proc_t          *procp = curproc;
 381         rctl_qty_t      lockedbytes = 0;
 382         kproject_t      *proj;
 383 
 384         /*
 385          * We are holding the a_lock on the underlying dummy as,
 386          * so we can make calls to the HAT layer.
 387          */
 388         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
 389         ASSERT(sp != NULL);
 390 
 391 #ifdef DEBUG
 392         TNF_PROBE_2(segspt_create, "spt", /* CSTYLED */,
 393             tnf_opaque, addr, addr, tnf_ulong, len, seg->s_size);
 394 #endif
 395         if ((sptcargs->flags & SHM_PAGEABLE) == 0) {
 396                 if (err = anon_swap_adjust(npages))
 397                         return (err);
 398         }
 399         err = ENOMEM;
 400 
 401         if ((sptd = kmem_zalloc(sizeof (*sptd), KM_NOSLEEP)) == NULL)
 402                 goto out1;
 403 
 404         if ((sptcargs->flags & SHM_PAGEABLE) == 0) {
 405                 if ((ppa = kmem_zalloc(((sizeof (page_t *)) * npages),
 406                     KM_NOSLEEP)) == NULL)
 407                         goto out2;
 408         }
 409 
 410         mutex_init(&sptd->spt_lock, NULL, MUTEX_DEFAULT, NULL);
 411 
 412         if ((vp = kmem_zalloc(sizeof (*vp), KM_NOSLEEP)) == NULL)
 413                 goto out3;
 414 
 415         seg->s_ops = &segspt_ops;
 416         sptd->spt_vp = vp;
 417         sptd->spt_amp = amp;
 418         sptd->spt_prot = sptcargs->prot;
 419         sptd->spt_flags = sptcargs->flags;
 420         seg->s_data = (caddr_t)sptd;
 421         sptd->spt_ppa = NULL;
 422         sptd->spt_ppa_lckcnt = NULL;
 423         seg->s_szc = sptcargs->szc;
 424         cv_init(&sptd->spt_cv, NULL, CV_DEFAULT, NULL);
 425         sptd->spt_gen = 0;
 426 
 427         ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
 428         if (seg->s_szc > amp->a_szc) {
 429                 amp->a_szc = seg->s_szc;
 430         }
 431         ANON_LOCK_EXIT(&amp->a_rwlock);
 432 
 433         /*
 434          * Set policy to affect initial allocation of pages in
 435          * anon_map_createpages()
 436          */
 437         (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, amp, anon_index,
 438             NULL, 0, ptob(npages));
 439 
 440         if (sptcargs->flags & SHM_PAGEABLE) {
 441                 size_t  share_sz;
 442                 pgcnt_t new_npgs, more_pgs;
 443                 struct anon_hdr *nahp;
 444                 zone_t *zone;
 445 
 446                 share_sz = page_get_pagesize(seg->s_szc);
 447                 if (!IS_P2ALIGNED(amp->size, share_sz)) {
 448                         /*
 449                          * We are rounding up the size of the anon array
 450                          * on 4 M boundary because we always create 4 M
 451                          * of page(s) when locking, faulting pages and we
 452                          * don't have to check for all corner cases e.g.
 453                          * if there is enough space to allocate 4 M
 454                          * page.
 455                          */
 456                         new_npgs = btop(P2ROUNDUP(amp->size, share_sz));
 457                         more_pgs = new_npgs - npages;
 458 
 459                         /*
 460                          * The zone will never be NULL, as a fully created
 461                          * shm always has an owning zone.
 462                          */
 463                         zone = sp->shm_perm.ipc_zone_ref.zref_zone;
 464                         ASSERT(zone != NULL);
 465                         if (anon_resv_zone(ptob(more_pgs), zone) == 0) {
 466                                 err = ENOMEM;
 467                                 goto out4;
 468                         }
 469 
 470                         nahp = anon_create(new_npgs, ANON_SLEEP);
 471                         ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
 472                         (void) anon_copy_ptr(amp->ahp, 0, nahp, 0, npages,
 473                             ANON_SLEEP);
 474                         anon_release(amp->ahp, npages);
 475                         amp->ahp = nahp;
 476                         ASSERT(amp->swresv == ptob(npages));
 477                         amp->swresv = amp->size = ptob(new_npgs);
 478                         ANON_LOCK_EXIT(&amp->a_rwlock);
 479                         npages = new_npgs;
 480                 }
 481 
 482                 sptd->spt_ppa_lckcnt = kmem_zalloc(npages *
 483                     sizeof (*sptd->spt_ppa_lckcnt), KM_SLEEP);
 484                 sptd->spt_pcachecnt = 0;
 485                 sptd->spt_realsize = ptob(npages);
 486                 sptcargs->seg_spt = seg;
 487                 return (0);
 488         }
 489 
 490         /*
 491          * get array of pages for each anon slot in amp
 492          */
 493         if ((err = anon_map_createpages(amp, anon_index, ptob(npages), ppa,
 494             seg, addr, S_CREATE, cred)) != 0)
 495                 goto out4;
 496 
 497         mutex_enter(&sp->shm_mlock);
 498 
 499         /* May be partially locked, so, count bytes to charge for locking */
 500         for (i = 0; i < npages; i++)
 501                 if (ppa[i]->p_lckcnt == 0)
 502                         lockedbytes += PAGESIZE;
 503 
 504         proj = sp->shm_perm.ipc_proj;
 505 
 506         if (lockedbytes > 0) {
 507                 mutex_enter(&procp->p_lock);
 508                 if (rctl_incr_locked_mem(procp, proj, lockedbytes, 0)) {
 509                         mutex_exit(&procp->p_lock);
 510                         mutex_exit(&sp->shm_mlock);
 511                         for (i = 0; i < npages; i++)
 512                                 page_unlock(ppa[i]);
 513                         err = ENOMEM;
 514                         goto out4;
 515                 }
 516                 mutex_exit(&procp->p_lock);
 517         }
 518 
 519         /*
 520          * addr is initial address corresponding to the first page on ppa list
 521          */
 522         for (i = 0; i < npages; i++) {
 523                 /* attempt to lock all pages */
 524                 if (page_pp_lock(ppa[i], 0, 1) == 0) {
 525                         /*
 526                          * if unable to lock any page, unlock all
 527                          * of them and return error
 528                          */
 529                         for (j = 0; j < i; j++)
 530                                 page_pp_unlock(ppa[j], 0, 1);
 531                         for (i = 0; i < npages; i++)
 532                                 page_unlock(ppa[i]);
 533                         rctl_decr_locked_mem(NULL, proj, lockedbytes, 0);
 534                         mutex_exit(&sp->shm_mlock);
 535                         err = ENOMEM;
 536                         goto out4;
 537                 }
 538         }
 539         mutex_exit(&sp->shm_mlock);
 540 
 541         /*
 542          * Some platforms assume that ISM mappings are HAT_LOAD_LOCK
 543          * for the entire life of the segment. For example platforms
 544          * that do not support Dynamic Reconfiguration.
 545          */
 546         hat_flags = HAT_LOAD_SHARE;
 547         if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, NULL))
 548                 hat_flags |= HAT_LOAD_LOCK;
 549 
 550         /*
 551          * Load translations one lare page at a time
 552          * to make sure we don't create mappings bigger than
 553          * segment's size code in case underlying pages
 554          * are shared with segvn's segment that uses bigger
 555          * size code than we do.
 556          */
 557         pgsz = page_get_pagesize(seg->s_szc);
 558         pgcnt = page_get_pagecnt(seg->s_szc);
 559         for (a = addr, pidx = 0; pidx < npages; a += pgsz, pidx += pgcnt) {
 560                 sz = MIN(pgsz, ptob(npages - pidx));
 561                 hat_memload_array(seg->s_as->a_hat, a, sz,
 562                     &ppa[pidx], sptd->spt_prot, hat_flags);
 563         }
 564 
 565         /*
 566          * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP,
 567          * we will leave the pages locked SE_SHARED for the life
 568          * of the ISM segment. This will prevent any calls to
 569          * hat_pageunload() on this ISM segment for those platforms.
 570          */
 571         if (!(hat_flags & HAT_LOAD_LOCK)) {
 572                 /*
 573                  * On platforms that support HAT_DYNAMIC_ISM_UNMAP,
 574                  * we no longer need to hold the SE_SHARED lock on the pages,
 575                  * since L_PAGELOCK and F_SOFTLOCK calls will grab the
 576                  * SE_SHARED lock on the pages as necessary.
 577                  */
 578                 for (i = 0; i < npages; i++)
 579                         page_unlock(ppa[i]);
 580         }
 581         sptd->spt_pcachecnt = 0;
 582         kmem_free(ppa, ((sizeof (page_t *)) * npages));
 583         sptd->spt_realsize = ptob(npages);
 584         atomic_add_long(&spt_used, npages);
 585         sptcargs->seg_spt = seg;
 586         return (0);
 587 
 588 out4:
 589         seg->s_data = NULL;
 590         kmem_free(vp, sizeof (*vp));
 591         cv_destroy(&sptd->spt_cv);
 592 out3:
 593         mutex_destroy(&sptd->spt_lock);
 594         if ((sptcargs->flags & SHM_PAGEABLE) == 0)
 595                 kmem_free(ppa, (sizeof (*ppa) * npages));
 596 out2:
 597         kmem_free(sptd, sizeof (*sptd));
 598 out1:
 599         if ((sptcargs->flags & SHM_PAGEABLE) == 0)
 600                 anon_swap_restore(npages);
 601         return (err);
 602 }
 603 
 604 /*ARGSUSED*/
 605 void
 606 segspt_free_pages(struct seg *seg, caddr_t addr, size_t len)
 607 {
 608         struct page     *pp;
 609         struct spt_data *sptd = (struct spt_data *)seg->s_data;
 610         pgcnt_t         npages;
 611         ulong_t         anon_idx;
 612         struct anon_map *amp;
 613         struct anon     *ap;
 614         struct vnode    *vp;
 615         u_offset_t      off;
 616         uint_t          hat_flags;
 617         int             root = 0;
 618         pgcnt_t         pgs, curnpgs = 0;
 619         page_t          *rootpp;
 620         rctl_qty_t      unlocked_bytes = 0;
 621         kproject_t      *proj;
 622         kshmid_t        *sp;
 623 
 624         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
 625 
 626         len = P2ROUNDUP(len, PAGESIZE);
 627 
 628         npages = btop(len);
 629 
 630         hat_flags = HAT_UNLOAD_UNLOCK | HAT_UNLOAD_UNMAP;
 631         if ((hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) ||
 632             (sptd->spt_flags & SHM_PAGEABLE)) {
 633                 hat_flags = HAT_UNLOAD_UNMAP;
 634         }
 635 
 636         hat_unload(seg->s_as->a_hat, addr, len, hat_flags);
 637 
 638         amp = sptd->spt_amp;
 639         if (sptd->spt_flags & SHM_PAGEABLE)
 640                 npages = btop(amp->size);
 641 
 642         ASSERT(amp != NULL);
 643 
 644         if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
 645                 sp = amp->a_sp;
 646                 proj = sp->shm_perm.ipc_proj;
 647                 mutex_enter(&sp->shm_mlock);
 648         }
 649         for (anon_idx = 0; anon_idx < npages; anon_idx++) {
 650                 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
 651                         if ((ap = anon_get_ptr(amp->ahp, anon_idx)) == NULL) {
 652                                 panic("segspt_free_pages: null app");
 653                                 /*NOTREACHED*/
 654                         }
 655                 } else {
 656                         if ((ap = anon_get_next_ptr(amp->ahp, &anon_idx))
 657                             == NULL)
 658                                 continue;
 659                 }
 660                 ASSERT(ANON_ISBUSY(anon_get_slot(amp->ahp, anon_idx)) == 0);
 661                 swap_xlate(ap, &vp, &off);
 662 
 663                 /*
 664                  * If this platform supports HAT_DYNAMIC_ISM_UNMAP,
 665                  * the pages won't be having SE_SHARED lock at this
 666                  * point.
 667                  *
 668                  * On platforms that do not support HAT_DYNAMIC_ISM_UNMAP,
 669                  * the pages are still held SE_SHARED locked from the
 670                  * original segspt_create()
 671                  *
 672                  * Our goal is to get SE_EXCL lock on each page, remove
 673                  * permanent lock on it and invalidate the page.
 674                  */
 675                 if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
 676                         if (hat_flags == HAT_UNLOAD_UNMAP)
 677                                 pp = page_lookup(vp, off, SE_EXCL);
 678                         else {
 679                                 if ((pp = page_find(vp, off)) == NULL) {
 680                                         panic("segspt_free_pages: "
 681                                             "page not locked");
 682                                         /*NOTREACHED*/
 683                                 }
 684                                 if (!page_tryupgrade(pp)) {
 685                                         page_unlock(pp);
 686                                         pp = page_lookup(vp, off, SE_EXCL);
 687                                 }
 688                         }
 689                         if (pp == NULL) {
 690                                 panic("segspt_free_pages: "
 691                                     "page not in the system");
 692                                 /*NOTREACHED*/
 693                         }
 694                         ASSERT(pp->p_lckcnt > 0);
 695                         page_pp_unlock(pp, 0, 1);
 696                         if (pp->p_lckcnt == 0)
 697                                 unlocked_bytes += PAGESIZE;
 698                 } else {
 699                         if ((pp = page_lookup(vp, off, SE_EXCL)) == NULL)
 700                                 continue;
 701                 }
 702                 /*
 703                  * It's logical to invalidate the pages here as in most cases
 704                  * these were created by segspt.
 705                  */
 706                 if (pp->p_szc != 0) {
 707                         if (root == 0) {
 708                                 ASSERT(curnpgs == 0);
 709                                 root = 1;
 710                                 rootpp = pp;
 711                                 pgs = curnpgs = page_get_pagecnt(pp->p_szc);
 712                                 ASSERT(pgs > 1);
 713                                 ASSERT(IS_P2ALIGNED(pgs, pgs));
 714                                 ASSERT(!(page_pptonum(pp) & (pgs - 1)));
 715                                 curnpgs--;
 716                         } else if ((page_pptonum(pp) & (pgs - 1)) == pgs - 1) {
 717                                 ASSERT(curnpgs == 1);
 718                                 ASSERT(page_pptonum(pp) ==
 719                                     page_pptonum(rootpp) + (pgs - 1));
 720                                 page_destroy_pages(rootpp);
 721                                 root = 0;
 722                                 curnpgs = 0;
 723                         } else {
 724                                 ASSERT(curnpgs > 1);
 725                                 ASSERT(page_pptonum(pp) ==
 726                                     page_pptonum(rootpp) + (pgs - curnpgs));
 727                                 curnpgs--;
 728                         }
 729                 } else {
 730                         if (root != 0 || curnpgs != 0) {
 731                                 panic("segspt_free_pages: bad large page");
 732                                 /*NOTREACHED*/
 733                         }
 734                         /*
 735                          * Before destroying the pages, we need to take care
 736                          * of the rctl locked memory accounting. For that
 737                          * we need to calculte the unlocked_bytes.
 738                          */
 739                         if (pp->p_lckcnt > 0)
 740                                 unlocked_bytes += PAGESIZE;
 741                         /*LINTED: constant in conditional context */
 742                         VN_DISPOSE(pp, B_INVAL, 0, kcred);
 743                 }
 744         }
 745         if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
 746                 if (unlocked_bytes > 0)
 747                         rctl_decr_locked_mem(NULL, proj, unlocked_bytes, 0);
 748                 mutex_exit(&sp->shm_mlock);
 749         }
 750         if (root != 0 || curnpgs != 0) {
 751                 panic("segspt_free_pages: bad large page");
 752                 /*NOTREACHED*/
 753         }
 754 
 755         /*
 756          * mark that pages have been released
 757          */
 758         sptd->spt_realsize = 0;
 759 
 760         if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
 761                 atomic_add_long(&spt_used, -npages);
 762                 anon_swap_restore(npages);
 763         }
 764 }
 765 
 766 /*
 767  * Get memory allocation policy info for specified address in given segment
 768  */
 769 static lgrp_mem_policy_info_t *
 770 segspt_getpolicy(struct seg *seg, caddr_t addr)
 771 {
 772         struct anon_map         *amp;
 773         ulong_t                 anon_index;
 774         lgrp_mem_policy_info_t  *policy_info;
 775         struct spt_data         *spt_data;
 776 
 777         ASSERT(seg != NULL);
 778 
 779         /*
 780          * Get anon_map from segspt
 781          *
 782          * Assume that no lock needs to be held on anon_map, since
 783          * it should be protected by its reference count which must be
 784          * nonzero for an existing segment
 785          * Need to grab readers lock on policy tree though
 786          */
 787         spt_data = (struct spt_data *)seg->s_data;
 788         if (spt_data == NULL)
 789                 return (NULL);
 790         amp = spt_data->spt_amp;
 791         ASSERT(amp->refcnt != 0);
 792 
 793         /*
 794          * Get policy info
 795          *
 796          * Assume starting anon index of 0
 797          */
 798         anon_index = seg_page(seg, addr);
 799         policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0);
 800 
 801         return (policy_info);
 802 }
 803 
 804 /*
 805  * DISM only.
 806  * Return locked pages over a given range.
 807  *
 808  * We will cache all DISM locked pages and save the pplist for the
 809  * entire segment in the ppa field of the underlying DISM segment structure.
 810  * Later, during a call to segspt_reclaim() we will use this ppa array
 811  * to page_unlock() all of the pages and then we will free this ppa list.
 812  */
 813 /*ARGSUSED*/
 814 static int
 815 segspt_dismpagelock(struct seg *seg, caddr_t addr, size_t len,
 816     struct page ***ppp, enum lock_type type, enum seg_rw rw)
 817 {
 818         struct  shm_data *shmd = (struct shm_data *)seg->s_data;
 819         struct  seg     *sptseg = shmd->shm_sptseg;
 820         struct  spt_data *sptd = sptseg->s_data;
 821         pgcnt_t pg_idx, npages, tot_npages, npgs;
 822         struct  page **pplist, **pl, **ppa, *pp;
 823         struct  anon_map *amp;
 824         spgcnt_t        an_idx;
 825         int     ret = ENOTSUP;
 826         uint_t  pl_built = 0;
 827         struct  anon *ap;
 828         struct  vnode *vp;
 829         u_offset_t off;
 830         pgcnt_t claim_availrmem = 0;
 831         uint_t  szc;
 832 
 833         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
 834         ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK);
 835 
 836         /*
 837          * We want to lock/unlock the entire ISM segment. Therefore,
 838          * we will be using the underlying sptseg and it's base address
 839          * and length for the caching arguments.
 840          */
 841         ASSERT(sptseg);
 842         ASSERT(sptd);
 843 
 844         pg_idx = seg_page(seg, addr);
 845         npages = btopr(len);
 846 
 847         /*
 848          * check if the request is larger than number of pages covered
 849          * by amp
 850          */
 851         if (pg_idx + npages > btopr(sptd->spt_amp->size)) {
 852                 *ppp = NULL;
 853                 return (ENOTSUP);
 854         }
 855 
 856         if (type == L_PAGEUNLOCK) {
 857                 ASSERT(sptd->spt_ppa != NULL);
 858 
 859                 seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size,
 860                     sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
 861 
 862                 /*
 863                  * If someone is blocked while unmapping, we purge
 864                  * segment page cache and thus reclaim pplist synchronously
 865                  * without waiting for seg_pasync_thread. This speeds up
 866                  * unmapping in cases where munmap(2) is called, while
 867                  * raw async i/o is still in progress or where a thread
 868                  * exits on data fault in a multithreaded application.
 869                  */
 870                 if ((sptd->spt_flags & DISM_PPA_CHANGED) ||
 871                     (AS_ISUNMAPWAIT(seg->s_as) &&
 872                     shmd->shm_softlockcnt > 0)) {
 873                         segspt_purge(seg);
 874                 }
 875                 return (0);
 876         }
 877 
 878         /* The L_PAGELOCK case ... */
 879 
 880         if (sptd->spt_flags & DISM_PPA_CHANGED) {
 881                 segspt_purge(seg);
 882                 /*
 883                  * for DISM ppa needs to be rebuild since
 884                  * number of locked pages could be changed
 885                  */
 886                 *ppp = NULL;
 887                 return (ENOTSUP);
 888         }
 889 
 890         /*
 891          * First try to find pages in segment page cache, without
 892          * holding the segment lock.
 893          */
 894         pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
 895             S_WRITE, SEGP_FORCE_WIRED);
 896         if (pplist != NULL) {
 897                 ASSERT(sptd->spt_ppa != NULL);
 898                 ASSERT(sptd->spt_ppa == pplist);
 899                 ppa = sptd->spt_ppa;
 900                 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
 901                         if (ppa[an_idx] == NULL) {
 902                                 seg_pinactive(seg, NULL, seg->s_base,
 903                                     sptd->spt_amp->size, ppa,
 904                                     S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
 905                                 *ppp = NULL;
 906                                 return (ENOTSUP);
 907                         }
 908                         if ((szc = ppa[an_idx]->p_szc) != 0) {
 909                                 npgs = page_get_pagecnt(szc);
 910                                 an_idx = P2ROUNDUP(an_idx + 1, npgs);
 911                         } else {
 912                                 an_idx++;
 913                         }
 914                 }
 915                 /*
 916                  * Since we cache the entire DISM segment, we want to
 917                  * set ppp to point to the first slot that corresponds
 918                  * to the requested addr, i.e. pg_idx.
 919                  */
 920                 *ppp = &(sptd->spt_ppa[pg_idx]);
 921                 return (0);
 922         }
 923 
 924         mutex_enter(&sptd->spt_lock);
 925         /*
 926          * try to find pages in segment page cache with mutex
 927          */
 928         pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
 929             S_WRITE, SEGP_FORCE_WIRED);
 930         if (pplist != NULL) {
 931                 ASSERT(sptd->spt_ppa != NULL);
 932                 ASSERT(sptd->spt_ppa == pplist);
 933                 ppa = sptd->spt_ppa;
 934                 for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
 935                         if (ppa[an_idx] == NULL) {
 936                                 mutex_exit(&sptd->spt_lock);
 937                                 seg_pinactive(seg, NULL, seg->s_base,
 938                                     sptd->spt_amp->size, ppa,
 939                                     S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
 940                                 *ppp = NULL;
 941                                 return (ENOTSUP);
 942                         }
 943                         if ((szc = ppa[an_idx]->p_szc) != 0) {
 944                                 npgs = page_get_pagecnt(szc);
 945                                 an_idx = P2ROUNDUP(an_idx + 1, npgs);
 946                         } else {
 947                                 an_idx++;
 948                         }
 949                 }
 950                 /*
 951                  * Since we cache the entire DISM segment, we want to
 952                  * set ppp to point to the first slot that corresponds
 953                  * to the requested addr, i.e. pg_idx.
 954                  */
 955                 mutex_exit(&sptd->spt_lock);
 956                 *ppp = &(sptd->spt_ppa[pg_idx]);
 957                 return (0);
 958         }
 959         if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size,
 960             SEGP_FORCE_WIRED) == SEGP_FAIL) {
 961                 mutex_exit(&sptd->spt_lock);
 962                 *ppp = NULL;
 963                 return (ENOTSUP);
 964         }
 965 
 966         /*
 967          * No need to worry about protections because DISM pages are always rw.
 968          */
 969         pl = pplist = NULL;
 970         amp = sptd->spt_amp;
 971 
 972         /*
 973          * Do we need to build the ppa array?
 974          */
 975         if (sptd->spt_ppa == NULL) {
 976                 pgcnt_t lpg_cnt = 0;
 977 
 978                 pl_built = 1;
 979                 tot_npages = btopr(sptd->spt_amp->size);
 980 
 981                 ASSERT(sptd->spt_pcachecnt == 0);
 982                 pplist = kmem_zalloc(sizeof (page_t *) * tot_npages, KM_SLEEP);
 983                 pl = pplist;
 984 
 985                 ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
 986                 for (an_idx = 0; an_idx < tot_npages; ) {
 987                         ap = anon_get_ptr(amp->ahp, an_idx);
 988                         /*
 989                          * Cache only mlocked pages. For large pages
 990                          * if one (constituent) page is mlocked
 991                          * all pages for that large page
 992                          * are cached also. This is for quick
 993                          * lookups of ppa array;
 994                          */
 995                         if ((ap != NULL) && (lpg_cnt != 0 ||
 996                             (sptd->spt_ppa_lckcnt[an_idx] != 0))) {
 997 
 998                                 swap_xlate(ap, &vp, &off);
 999                                 pp = page_lookup(vp, off, SE_SHARED);
1000                                 ASSERT(pp != NULL);
1001                                 if (lpg_cnt == 0) {
1002                                         lpg_cnt++;
1003                                         /*
1004                                          * For a small page, we are done --
1005                                          * lpg_count is reset to 0 below.
1006                                          *
1007                                          * For a large page, we are guaranteed
1008                                          * to find the anon structures of all
1009                                          * constituent pages and a non-zero
1010                                          * lpg_cnt ensures that we don't test
1011                                          * for mlock for these. We are done
1012                                          * when lpg_count reaches (npgs + 1).
1013                                          * If we are not the first constituent
1014                                          * page, restart at the first one.
1015                                          */
1016                                         npgs = page_get_pagecnt(pp->p_szc);
1017                                         if (!IS_P2ALIGNED(an_idx, npgs)) {
1018                                                 an_idx = P2ALIGN(an_idx, npgs);
1019                                                 page_unlock(pp);
1020                                                 continue;
1021                                         }
1022                                 }
1023                                 if (++lpg_cnt > npgs)
1024                                         lpg_cnt = 0;
1025 
1026                                 /*
1027                                  * availrmem is decremented only
1028                                  * for unlocked pages
1029                                  */
1030                                 if (sptd->spt_ppa_lckcnt[an_idx] == 0)
1031                                         claim_availrmem++;
1032                                 pplist[an_idx] = pp;
1033                         }
1034                         an_idx++;
1035                 }
1036                 ANON_LOCK_EXIT(&amp->a_rwlock);
1037 
1038                 if (claim_availrmem) {
1039                         mutex_enter(&freemem_lock);
1040                         if (availrmem < tune.t_minarmem + claim_availrmem) {
1041                                 mutex_exit(&freemem_lock);
1042                                 ret = ENOTSUP;
1043                                 claim_availrmem = 0;
1044                                 goto insert_fail;
1045                         } else {
1046                                 availrmem -= claim_availrmem;
1047                         }
1048                         mutex_exit(&freemem_lock);
1049                 }
1050 
1051                 sptd->spt_ppa = pl;
1052         } else {
1053                 /*
1054                  * We already have a valid ppa[].
1055                  */
1056                 pl = sptd->spt_ppa;
1057         }
1058 
1059         ASSERT(pl != NULL);
1060 
1061         ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size,
1062             sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED,
1063             segspt_reclaim);
1064         if (ret == SEGP_FAIL) {
1065                 /*
1066                  * seg_pinsert failed. We return
1067                  * ENOTSUP, so that the as_pagelock() code will
1068                  * then try the slower F_SOFTLOCK path.
1069                  */
1070                 if (pl_built) {
1071                         /*
1072                          * No one else has referenced the ppa[].
1073                          * We created it and we need to destroy it.
1074                          */
1075                         sptd->spt_ppa = NULL;
1076                 }
1077                 ret = ENOTSUP;
1078                 goto insert_fail;
1079         }
1080 
1081         /*
1082          * In either case, we increment softlockcnt on the 'real' segment.
1083          */
1084         sptd->spt_pcachecnt++;
1085         atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
1086 
1087         ppa = sptd->spt_ppa;
1088         for (an_idx = pg_idx; an_idx < pg_idx + npages; ) {
1089                 if (ppa[an_idx] == NULL) {
1090                         mutex_exit(&sptd->spt_lock);
1091                         seg_pinactive(seg, NULL, seg->s_base,
1092                             sptd->spt_amp->size,
1093                             pl, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
1094                         *ppp = NULL;
1095                         return (ENOTSUP);
1096                 }
1097                 if ((szc = ppa[an_idx]->p_szc) != 0) {
1098                         npgs = page_get_pagecnt(szc);
1099                         an_idx = P2ROUNDUP(an_idx + 1, npgs);
1100                 } else {
1101                         an_idx++;
1102                 }
1103         }
1104         /*
1105          * We can now drop the sptd->spt_lock since the ppa[]
1106          * exists and he have incremented pacachecnt.
1107          */
1108         mutex_exit(&sptd->spt_lock);
1109 
1110         /*
1111          * Since we cache the entire segment, we want to
1112          * set ppp to point to the first slot that corresponds
1113          * to the requested addr, i.e. pg_idx.
1114          */
1115         *ppp = &(sptd->spt_ppa[pg_idx]);
1116         return (0);
1117 
1118 insert_fail:
1119         /*
1120          * We will only reach this code if we tried and failed.
1121          *
1122          * And we can drop the lock on the dummy seg, once we've failed
1123          * to set up a new ppa[].
1124          */
1125         mutex_exit(&sptd->spt_lock);
1126 
1127         if (pl_built) {
1128                 if (claim_availrmem) {
1129                         mutex_enter(&freemem_lock);
1130                         availrmem += claim_availrmem;
1131                         mutex_exit(&freemem_lock);
1132                 }
1133 
1134                 /*
1135                  * We created pl and we need to destroy it.
1136                  */
1137                 pplist = pl;
1138                 for (an_idx = 0; an_idx < tot_npages; an_idx++) {
1139                         if (pplist[an_idx] != NULL)
1140                                 page_unlock(pplist[an_idx]);
1141                 }
1142                 kmem_free(pl, sizeof (page_t *) * tot_npages);
1143         }
1144 
1145         if (shmd->shm_softlockcnt <= 0) {
1146                 if (AS_ISUNMAPWAIT(seg->s_as)) {
1147                         mutex_enter(&seg->s_as->a_contents);
1148                         if (AS_ISUNMAPWAIT(seg->s_as)) {
1149                                 AS_CLRUNMAPWAIT(seg->s_as);
1150                                 cv_broadcast(&seg->s_as->a_cv);
1151                         }
1152                         mutex_exit(&seg->s_as->a_contents);
1153                 }
1154         }
1155         *ppp = NULL;
1156         return (ret);
1157 }
1158 
1159 
1160 
1161 /*
1162  * return locked pages over a given range.
1163  *
1164  * We will cache the entire ISM segment and save the pplist for the
1165  * entire segment in the ppa field of the underlying ISM segment structure.
1166  * Later, during a call to segspt_reclaim() we will use this ppa array
1167  * to page_unlock() all of the pages and then we will free this ppa list.
1168  */
1169 /*ARGSUSED*/
1170 static int
1171 segspt_shmpagelock(struct seg *seg, caddr_t addr, size_t len,
1172     struct page ***ppp, enum lock_type type, enum seg_rw rw)
1173 {
1174         struct shm_data *shmd = (struct shm_data *)seg->s_data;
1175         struct seg      *sptseg = shmd->shm_sptseg;
1176         struct spt_data *sptd = sptseg->s_data;
1177         pgcnt_t np, page_index, npages;
1178         caddr_t a, spt_base;
1179         struct page **pplist, **pl, *pp;
1180         struct anon_map *amp;
1181         ulong_t anon_index;
1182         int ret = ENOTSUP;
1183         uint_t  pl_built = 0;
1184         struct anon *ap;
1185         struct vnode *vp;
1186         u_offset_t off;
1187 
1188         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
1189         ASSERT(type == L_PAGELOCK || type == L_PAGEUNLOCK);
1190 
1191 
1192         /*
1193          * We want to lock/unlock the entire ISM segment. Therefore,
1194          * we will be using the underlying sptseg and it's base address
1195          * and length for the caching arguments.
1196          */
1197         ASSERT(sptseg);
1198         ASSERT(sptd);
1199 
1200         if (sptd->spt_flags & SHM_PAGEABLE) {
1201                 return (segspt_dismpagelock(seg, addr, len, ppp, type, rw));
1202         }
1203 
1204         page_index = seg_page(seg, addr);
1205         npages = btopr(len);
1206 
1207         /*
1208          * check if the request is larger than number of pages covered
1209          * by amp
1210          */
1211         if (page_index + npages > btopr(sptd->spt_amp->size)) {
1212                 *ppp = NULL;
1213                 return (ENOTSUP);
1214         }
1215 
1216         if (type == L_PAGEUNLOCK) {
1217 
1218                 ASSERT(sptd->spt_ppa != NULL);
1219 
1220                 seg_pinactive(seg, NULL, seg->s_base, sptd->spt_amp->size,
1221                     sptd->spt_ppa, S_WRITE, SEGP_FORCE_WIRED, segspt_reclaim);
1222 
1223                 /*
1224                  * If someone is blocked while unmapping, we purge
1225                  * segment page cache and thus reclaim pplist synchronously
1226                  * without waiting for seg_pasync_thread. This speeds up
1227                  * unmapping in cases where munmap(2) is called, while
1228                  * raw async i/o is still in progress or where a thread
1229                  * exits on data fault in a multithreaded application.
1230                  */
1231                 if (AS_ISUNMAPWAIT(seg->s_as) && (shmd->shm_softlockcnt > 0)) {
1232                         segspt_purge(seg);
1233                 }
1234                 return (0);
1235         }
1236 
1237         /* The L_PAGELOCK case... */
1238 
1239         /*
1240          * First try to find pages in segment page cache, without
1241          * holding the segment lock.
1242          */
1243         pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
1244             S_WRITE, SEGP_FORCE_WIRED);
1245         if (pplist != NULL) {
1246                 ASSERT(sptd->spt_ppa == pplist);
1247                 ASSERT(sptd->spt_ppa[page_index]);
1248                 /*
1249                  * Since we cache the entire ISM segment, we want to
1250                  * set ppp to point to the first slot that corresponds
1251                  * to the requested addr, i.e. page_index.
1252                  */
1253                 *ppp = &(sptd->spt_ppa[page_index]);
1254                 return (0);
1255         }
1256 
1257         mutex_enter(&sptd->spt_lock);
1258 
1259         /*
1260          * try to find pages in segment page cache
1261          */
1262         pplist = seg_plookup(seg, NULL, seg->s_base, sptd->spt_amp->size,
1263             S_WRITE, SEGP_FORCE_WIRED);
1264         if (pplist != NULL) {
1265                 ASSERT(sptd->spt_ppa == pplist);
1266                 /*
1267                  * Since we cache the entire segment, we want to
1268                  * set ppp to point to the first slot that corresponds
1269                  * to the requested addr, i.e. page_index.
1270                  */
1271                 mutex_exit(&sptd->spt_lock);
1272                 *ppp = &(sptd->spt_ppa[page_index]);
1273                 return (0);
1274         }
1275 
1276         if (seg_pinsert_check(seg, NULL, seg->s_base, sptd->spt_amp->size,
1277             SEGP_FORCE_WIRED) == SEGP_FAIL) {
1278                 mutex_exit(&sptd->spt_lock);
1279                 *ppp = NULL;
1280                 return (ENOTSUP);
1281         }
1282 
1283         /*
1284          * No need to worry about protections because ISM pages
1285          * are always rw.
1286          */
1287         pl = pplist = NULL;
1288 
1289         /*
1290          * Do we need to build the ppa array?
1291          */
1292         if (sptd->spt_ppa == NULL) {
1293                 ASSERT(sptd->spt_ppa == pplist);
1294 
1295                 spt_base = sptseg->s_base;
1296                 pl_built = 1;
1297 
1298                 /*
1299                  * availrmem is decremented once during anon_swap_adjust()
1300                  * and is incremented during the anon_unresv(), which is
1301                  * called from shm_rm_amp() when the segment is destroyed.
1302                  */
1303                 amp = sptd->spt_amp;
1304                 ASSERT(amp != NULL);
1305 
1306                 /* pcachecnt is protected by sptd->spt_lock */
1307                 ASSERT(sptd->spt_pcachecnt == 0);
1308                 pplist = kmem_zalloc(sizeof (page_t *)
1309                     * btopr(sptd->spt_amp->size), KM_SLEEP);
1310                 pl = pplist;
1311 
1312                 anon_index = seg_page(sptseg, spt_base);
1313 
1314                 ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
1315                 for (a = spt_base; a < (spt_base + sptd->spt_amp->size);
1316                     a += PAGESIZE, anon_index++, pplist++) {
1317                         ap = anon_get_ptr(amp->ahp, anon_index);
1318                         ASSERT(ap != NULL);
1319                         swap_xlate(ap, &vp, &off);
1320                         pp = page_lookup(vp, off, SE_SHARED);
1321                         ASSERT(pp != NULL);
1322                         *pplist = pp;
1323                 }
1324                 ANON_LOCK_EXIT(&amp->a_rwlock);
1325 
1326                 if (a < (spt_base + sptd->spt_amp->size)) {
1327                         ret = ENOTSUP;
1328                         goto insert_fail;
1329                 }
1330                 sptd->spt_ppa = pl;
1331         } else {
1332                 /*
1333                  * We already have a valid ppa[].
1334                  */
1335                 pl = sptd->spt_ppa;
1336         }
1337 
1338         ASSERT(pl != NULL);
1339 
1340         ret = seg_pinsert(seg, NULL, seg->s_base, sptd->spt_amp->size,
1341             sptd->spt_amp->size, pl, S_WRITE, SEGP_FORCE_WIRED,
1342             segspt_reclaim);
1343         if (ret == SEGP_FAIL) {
1344                 /*
1345                  * seg_pinsert failed. We return
1346                  * ENOTSUP, so that the as_pagelock() code will
1347                  * then try the slower F_SOFTLOCK path.
1348                  */
1349                 if (pl_built) {
1350                         /*
1351                          * No one else has referenced the ppa[].
1352                          * We created it and we need to destroy it.
1353                          */
1354                         sptd->spt_ppa = NULL;
1355                 }
1356                 ret = ENOTSUP;
1357                 goto insert_fail;
1358         }
1359 
1360         /*
1361          * In either case, we increment softlockcnt on the 'real' segment.
1362          */
1363         sptd->spt_pcachecnt++;
1364         atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
1365 
1366         /*
1367          * We can now drop the sptd->spt_lock since the ppa[]
1368          * exists and he have incremented pacachecnt.
1369          */
1370         mutex_exit(&sptd->spt_lock);
1371 
1372         /*
1373          * Since we cache the entire segment, we want to
1374          * set ppp to point to the first slot that corresponds
1375          * to the requested addr, i.e. page_index.
1376          */
1377         *ppp = &(sptd->spt_ppa[page_index]);
1378         return (0);
1379 
1380 insert_fail:
1381         /*
1382          * We will only reach this code if we tried and failed.
1383          *
1384          * And we can drop the lock on the dummy seg, once we've failed
1385          * to set up a new ppa[].
1386          */
1387         mutex_exit(&sptd->spt_lock);
1388 
1389         if (pl_built) {
1390                 /*
1391                  * We created pl and we need to destroy it.
1392                  */
1393                 pplist = pl;
1394                 np = (((uintptr_t)(a - spt_base)) >> PAGESHIFT);
1395                 while (np) {
1396                         page_unlock(*pplist);
1397                         np--;
1398                         pplist++;
1399                 }
1400                 kmem_free(pl, sizeof (page_t *) * btopr(sptd->spt_amp->size));
1401         }
1402         if (shmd->shm_softlockcnt <= 0) {
1403                 if (AS_ISUNMAPWAIT(seg->s_as)) {
1404                         mutex_enter(&seg->s_as->a_contents);
1405                         if (AS_ISUNMAPWAIT(seg->s_as)) {
1406                                 AS_CLRUNMAPWAIT(seg->s_as);
1407                                 cv_broadcast(&seg->s_as->a_cv);
1408                         }
1409                         mutex_exit(&seg->s_as->a_contents);
1410                 }
1411         }
1412         *ppp = NULL;
1413         return (ret);
1414 }
1415 
1416 /*
1417  * purge any cached pages in the I/O page cache
1418  */
1419 static void
1420 segspt_purge(struct seg *seg)
1421 {
1422         seg_ppurge(seg, NULL, SEGP_FORCE_WIRED);
1423 }
1424 
1425 static int
1426 segspt_reclaim(void *ptag, caddr_t addr, size_t len, struct page **pplist,
1427         enum seg_rw rw, int async)
1428 {
1429         struct seg *seg = (struct seg *)ptag;
1430         struct  shm_data *shmd = (struct shm_data *)seg->s_data;
1431         struct  seg     *sptseg;
1432         struct  spt_data *sptd;
1433         pgcnt_t npages, i, free_availrmem = 0;
1434         int     done = 0;
1435 
1436 #ifdef lint
1437         addr = addr;
1438 #endif
1439         sptseg = shmd->shm_sptseg;
1440         sptd = sptseg->s_data;
1441         npages = (len >> PAGESHIFT);
1442         ASSERT(npages);
1443         ASSERT(sptd->spt_pcachecnt != 0);
1444         ASSERT(sptd->spt_ppa == pplist);
1445         ASSERT(npages == btopr(sptd->spt_amp->size));
1446         ASSERT(async || AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
1447 
1448         /*
1449          * Acquire the lock on the dummy seg and destroy the
1450          * ppa array IF this is the last pcachecnt.
1451          */
1452         mutex_enter(&sptd->spt_lock);
1453         if (--sptd->spt_pcachecnt == 0) {
1454                 for (i = 0; i < npages; i++) {
1455                         if (pplist[i] == NULL) {
1456                                 continue;
1457                         }
1458                         if (rw == S_WRITE) {
1459                                 hat_setrefmod(pplist[i]);
1460                         } else {
1461                                 hat_setref(pplist[i]);
1462                         }
1463                         if ((sptd->spt_flags & SHM_PAGEABLE) &&
1464                             (sptd->spt_ppa_lckcnt[i] == 0))
1465                                 free_availrmem++;
1466                         page_unlock(pplist[i]);
1467                 }
1468                 if ((sptd->spt_flags & SHM_PAGEABLE) && free_availrmem) {
1469                         mutex_enter(&freemem_lock);
1470                         availrmem += free_availrmem;
1471                         mutex_exit(&freemem_lock);
1472                 }
1473                 /*
1474                  * Since we want to cach/uncache the entire ISM segment,
1475                  * we will track the pplist in a segspt specific field
1476                  * ppa, that is initialized at the time we add an entry to
1477                  * the cache.
1478                  */
1479                 ASSERT(sptd->spt_pcachecnt == 0);
1480                 kmem_free(pplist, sizeof (page_t *) * npages);
1481                 sptd->spt_ppa = NULL;
1482                 sptd->spt_flags &= ~DISM_PPA_CHANGED;
1483                 sptd->spt_gen++;
1484                 cv_broadcast(&sptd->spt_cv);
1485                 done = 1;
1486         }
1487         mutex_exit(&sptd->spt_lock);
1488 
1489         /*
1490          * If we are pcache async thread or called via seg_ppurge_wiredpp() we
1491          * may not hold AS lock (in this case async argument is not 0). This
1492          * means if softlockcnt drops to 0 after the decrement below address
1493          * space may get freed. We can't allow it since after softlock
1494          * derement to 0 we still need to access as structure for possible
1495          * wakeup of unmap waiters. To prevent the disappearance of as we take
1496          * this segment's shm_segfree_syncmtx. segspt_shmfree() also takes
1497          * this mutex as a barrier to make sure this routine completes before
1498          * segment is freed.
1499          *
1500          * The second complication we have to deal with in async case is a
1501          * possibility of missed wake up of unmap wait thread. When we don't
1502          * hold as lock here we may take a_contents lock before unmap wait
1503          * thread that was first to see softlockcnt was still not 0. As a
1504          * result we'll fail to wake up an unmap wait thread. To avoid this
1505          * race we set nounmapwait flag in as structure if we drop softlockcnt
1506          * to 0 if async is not 0.  unmapwait thread
1507          * will not block if this flag is set.
1508          */
1509         if (async)
1510                 mutex_enter(&shmd->shm_segfree_syncmtx);
1511 
1512         /*
1513          * Now decrement softlockcnt.
1514          */
1515         ASSERT(shmd->shm_softlockcnt > 0);
1516         atomic_dec_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
1517 
1518         if (shmd->shm_softlockcnt <= 0) {
1519                 if (async || AS_ISUNMAPWAIT(seg->s_as)) {
1520                         mutex_enter(&seg->s_as->a_contents);
1521                         if (async)
1522                                 AS_SETNOUNMAPWAIT(seg->s_as);
1523                         if (AS_ISUNMAPWAIT(seg->s_as)) {
1524                                 AS_CLRUNMAPWAIT(seg->s_as);
1525                                 cv_broadcast(&seg->s_as->a_cv);
1526                         }
1527                         mutex_exit(&seg->s_as->a_contents);
1528                 }
1529         }
1530 
1531         if (async)
1532                 mutex_exit(&shmd->shm_segfree_syncmtx);
1533 
1534         return (done);
1535 }
1536 
1537 /*
1538  * Do a F_SOFTUNLOCK call over the range requested.
1539  * The range must have already been F_SOFTLOCK'ed.
1540  *
1541  * The calls to acquire and release the anon map lock mutex were
1542  * removed in order to avoid a deadly embrace during a DR
1543  * memory delete operation.  (Eg. DR blocks while waiting for a
1544  * exclusive lock on a page that is being used for kaio; the
1545  * thread that will complete the kaio and call segspt_softunlock
1546  * blocks on the anon map lock; another thread holding the anon
1547  * map lock blocks on another page lock via the segspt_shmfault
1548  * -> page_lookup -> page_lookup_create -> page_lock_es code flow.)
1549  *
1550  * The appropriateness of the removal is based upon the following:
1551  * 1. If we are holding a segment's reader lock and the page is held
1552  * shared, then the corresponding element in anonmap which points to
1553  * anon struct cannot change and there is no need to acquire the
1554  * anonymous map lock.
1555  * 2. Threads in segspt_softunlock have a reader lock on the segment
1556  * and already have the shared page lock, so we are guaranteed that
1557  * the anon map slot cannot change and therefore can call anon_get_ptr()
1558  * without grabbing the anonymous map lock.
1559  * 3. Threads that softlock a shared page break copy-on-write, even if
1560  * its a read.  Thus cow faults can be ignored with respect to soft
1561  * unlocking, since the breaking of cow means that the anon slot(s) will
1562  * not be shared.
1563  */
1564 static void
1565 segspt_softunlock(struct seg *seg, caddr_t sptseg_addr,
1566         size_t len, enum seg_rw rw)
1567 {
1568         struct shm_data *shmd = (struct shm_data *)seg->s_data;
1569         struct seg      *sptseg;
1570         struct spt_data *sptd;
1571         page_t *pp;
1572         caddr_t adr;
1573         struct vnode *vp;
1574         u_offset_t offset;
1575         ulong_t anon_index;
1576         struct anon_map *amp;           /* XXX - for locknest */
1577         struct anon *ap = NULL;
1578         pgcnt_t npages;
1579 
1580         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
1581 
1582         sptseg = shmd->shm_sptseg;
1583         sptd = sptseg->s_data;
1584 
1585         /*
1586          * Some platforms assume that ISM mappings are HAT_LOAD_LOCK
1587          * and therefore their pages are SE_SHARED locked
1588          * for the entire life of the segment.
1589          */
1590         if ((!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) &&
1591             ((sptd->spt_flags & SHM_PAGEABLE) == 0)) {
1592                 goto softlock_decrement;
1593         }
1594 
1595         /*
1596          * Any thread is free to do a page_find and
1597          * page_unlock() on the pages within this seg.
1598          *
1599          * We are already holding the as->a_lock on the user's
1600          * real segment, but we need to hold the a_lock on the
1601          * underlying dummy as. This is mostly to satisfy the
1602          * underlying HAT layer.
1603          */
1604         AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER);
1605         hat_unlock(sptseg->s_as->a_hat, sptseg_addr, len);
1606         AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock);
1607 
1608         amp = sptd->spt_amp;
1609         ASSERT(amp != NULL);
1610         anon_index = seg_page(sptseg, sptseg_addr);
1611 
1612         for (adr = sptseg_addr; adr < sptseg_addr + len; adr += PAGESIZE) {
1613                 ap = anon_get_ptr(amp->ahp, anon_index++);
1614                 ASSERT(ap != NULL);
1615                 swap_xlate(ap, &vp, &offset);
1616 
1617                 /*
1618                  * Use page_find() instead of page_lookup() to
1619                  * find the page since we know that it has a
1620                  * "shared" lock.
1621                  */
1622                 pp = page_find(vp, offset);
1623                 ASSERT(ap == anon_get_ptr(amp->ahp, anon_index - 1));
1624                 if (pp == NULL) {
1625                         panic("segspt_softunlock: "
1626                             "addr %p, ap %p, vp %p, off %llx",
1627                             (void *)adr, (void *)ap, (void *)vp, offset);
1628                         /*NOTREACHED*/
1629                 }
1630 
1631                 if (rw == S_WRITE) {
1632                         hat_setrefmod(pp);
1633                 } else if (rw != S_OTHER) {
1634                         hat_setref(pp);
1635                 }
1636                 page_unlock(pp);
1637         }
1638 
1639 softlock_decrement:
1640         npages = btopr(len);
1641         ASSERT(shmd->shm_softlockcnt >= npages);
1642         atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), -npages);
1643         if (shmd->shm_softlockcnt == 0) {
1644                 /*
1645                  * All SOFTLOCKS are gone. Wakeup any waiting
1646                  * unmappers so they can try again to unmap.
1647                  * Check for waiters first without the mutex
1648                  * held so we don't always grab the mutex on
1649                  * softunlocks.
1650                  */
1651                 if (AS_ISUNMAPWAIT(seg->s_as)) {
1652                         mutex_enter(&seg->s_as->a_contents);
1653                         if (AS_ISUNMAPWAIT(seg->s_as)) {
1654                                 AS_CLRUNMAPWAIT(seg->s_as);
1655                                 cv_broadcast(&seg->s_as->a_cv);
1656                         }
1657                         mutex_exit(&seg->s_as->a_contents);
1658                 }
1659         }
1660 }
1661 
1662 int
1663 segspt_shmattach(struct seg *seg, caddr_t *argsp)
1664 {
1665         struct shm_data *shmd_arg = (struct shm_data *)argsp;
1666         struct shm_data *shmd;
1667         struct anon_map *shm_amp = shmd_arg->shm_amp;
1668         struct spt_data *sptd;
1669         int error = 0;
1670 
1671         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
1672 
1673         shmd = kmem_zalloc((sizeof (*shmd)), KM_NOSLEEP);
1674         if (shmd == NULL)
1675                 return (ENOMEM);
1676 
1677         shmd->shm_sptas = shmd_arg->shm_sptas;
1678         shmd->shm_amp = shm_amp;
1679         shmd->shm_sptseg = shmd_arg->shm_sptseg;
1680 
1681         (void) lgrp_shm_policy_set(LGRP_MEM_POLICY_DEFAULT, shm_amp, 0,
1682             NULL, 0, seg->s_size);
1683 
1684         mutex_init(&shmd->shm_segfree_syncmtx, NULL, MUTEX_DEFAULT, NULL);
1685 
1686         seg->s_data = (void *)shmd;
1687         seg->s_ops = &segspt_shmops;
1688         seg->s_szc = shmd->shm_sptseg->s_szc;
1689         sptd = shmd->shm_sptseg->s_data;
1690 
1691         if (sptd->spt_flags & SHM_PAGEABLE) {
1692                 if ((shmd->shm_vpage = kmem_zalloc(btopr(shm_amp->size),
1693                     KM_NOSLEEP)) == NULL) {
1694                         seg->s_data = (void *)NULL;
1695                         kmem_free(shmd, (sizeof (*shmd)));
1696                         return (ENOMEM);
1697                 }
1698                 shmd->shm_lckpgs = 0;
1699                 if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
1700                         if ((error = hat_share(seg->s_as->a_hat, seg->s_base,
1701                             shmd_arg->shm_sptas->a_hat, SEGSPTADDR,
1702                             seg->s_size, seg->s_szc)) != 0) {
1703                                 kmem_free(shmd->shm_vpage,
1704                                     btopr(shm_amp->size));
1705                         }
1706                 }
1707         } else {
1708                 error = hat_share(seg->s_as->a_hat, seg->s_base,
1709                     shmd_arg->shm_sptas->a_hat, SEGSPTADDR,
1710                     seg->s_size, seg->s_szc);
1711         }
1712         if (error) {
1713                 seg->s_szc = 0;
1714                 seg->s_data = (void *)NULL;
1715                 kmem_free(shmd, (sizeof (*shmd)));
1716         } else {
1717                 ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER);
1718                 shm_amp->refcnt++;
1719                 ANON_LOCK_EXIT(&shm_amp->a_rwlock);
1720         }
1721         return (error);
1722 }
1723 
1724 int
1725 segspt_shmunmap(struct seg *seg, caddr_t raddr, size_t ssize)
1726 {
1727         struct shm_data *shmd = (struct shm_data *)seg->s_data;
1728         int reclaim = 1;
1729 
1730         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
1731 retry:
1732         if (shmd->shm_softlockcnt > 0) {
1733                 if (reclaim == 1) {
1734                         segspt_purge(seg);
1735                         reclaim = 0;
1736                         goto retry;
1737                 }
1738                 return (EAGAIN);
1739         }
1740 
1741         if (ssize != seg->s_size) {
1742 #ifdef DEBUG
1743                 cmn_err(CE_WARN, "Incompatible ssize %lx s_size %lx\n",
1744                     ssize, seg->s_size);
1745 #endif
1746                 return (EINVAL);
1747         }
1748 
1749         (void) segspt_shmlockop(seg, raddr, shmd->shm_amp->size, 0, MC_UNLOCK,
1750             NULL, 0);
1751         hat_unshare(seg->s_as->a_hat, raddr, ssize, seg->s_szc);
1752 
1753         seg_free(seg);
1754 
1755         return (0);
1756 }
1757 
1758 void
1759 segspt_shmfree(struct seg *seg)
1760 {
1761         struct shm_data *shmd = (struct shm_data *)seg->s_data;
1762         struct anon_map *shm_amp = shmd->shm_amp;
1763 
1764         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
1765 
1766         (void) segspt_shmlockop(seg, seg->s_base, shm_amp->size, 0,
1767             MC_UNLOCK, NULL, 0);
1768 
1769         /*
1770          * Need to increment refcnt when attaching
1771          * and decrement when detaching because of dup().
1772          */
1773         ANON_LOCK_ENTER(&shm_amp->a_rwlock, RW_WRITER);
1774         shm_amp->refcnt--;
1775         ANON_LOCK_EXIT(&shm_amp->a_rwlock);
1776 
1777         if (shmd->shm_vpage) {       /* only for DISM */
1778                 kmem_free(shmd->shm_vpage, btopr(shm_amp->size));
1779                 shmd->shm_vpage = NULL;
1780         }
1781 
1782         /*
1783          * Take shm_segfree_syncmtx lock to let segspt_reclaim() finish if it's
1784          * still working with this segment without holding as lock.
1785          */
1786         ASSERT(shmd->shm_softlockcnt == 0);
1787         mutex_enter(&shmd->shm_segfree_syncmtx);
1788         mutex_destroy(&shmd->shm_segfree_syncmtx);
1789 
1790         kmem_free(shmd, sizeof (*shmd));
1791 }
1792 
1793 /*ARGSUSED*/
1794 int
1795 segspt_shmsetprot(struct seg *seg, caddr_t addr, size_t len, uint_t prot)
1796 {
1797         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
1798 
1799         /*
1800          * Shared page table is more than shared mapping.
1801          *  Individual process sharing page tables can't change prot
1802          *  because there is only one set of page tables.
1803          *  This will be allowed after private page table is
1804          *  supported.
1805          */
1806 /* need to return correct status error? */
1807         return (0);
1808 }
1809 
1810 
1811 faultcode_t
1812 segspt_dismfault(struct hat *hat, struct seg *seg, caddr_t addr,
1813     size_t len, enum fault_type type, enum seg_rw rw)
1814 {
1815         struct  shm_data        *shmd = (struct shm_data *)seg->s_data;
1816         struct  seg             *sptseg = shmd->shm_sptseg;
1817         struct  as              *curspt = shmd->shm_sptas;
1818         struct  spt_data        *sptd = sptseg->s_data;
1819         pgcnt_t npages;
1820         size_t  size;
1821         caddr_t segspt_addr, shm_addr;
1822         page_t  **ppa;
1823         int     i;
1824         ulong_t an_idx = 0;
1825         int     err = 0;
1826         int     dyn_ism_unmap = hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0);
1827         size_t  pgsz;
1828         pgcnt_t pgcnt;
1829         caddr_t a;
1830         pgcnt_t pidx;
1831 
1832 #ifdef lint
1833         hat = hat;
1834 #endif
1835         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
1836 
1837         /*
1838          * Because of the way spt is implemented
1839          * the realsize of the segment does not have to be
1840          * equal to the segment size itself. The segment size is
1841          * often in multiples of a page size larger than PAGESIZE.
1842          * The realsize is rounded up to the nearest PAGESIZE
1843          * based on what the user requested. This is a bit of
1844          * ungliness that is historical but not easily fixed
1845          * without re-designing the higher levels of ISM.
1846          */
1847         ASSERT(addr >= seg->s_base);
1848         if (((addr + len) - seg->s_base) > sptd->spt_realsize)
1849                 return (FC_NOMAP);
1850         /*
1851          * For all of the following cases except F_PROT, we need to
1852          * make any necessary adjustments to addr and len
1853          * and get all of the necessary page_t's into an array called ppa[].
1854          *
1855          * The code in shmat() forces base addr and len of ISM segment
1856          * to be aligned to largest page size supported. Therefore,
1857          * we are able to handle F_SOFTLOCK and F_INVAL calls in "large
1858          * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK
1859          * in large pagesize chunks, or else we will screw up the HAT
1860          * layer by calling hat_memload_array() with differing page sizes
1861          * over a given virtual range.
1862          */
1863         pgsz = page_get_pagesize(sptseg->s_szc);
1864         pgcnt = page_get_pagecnt(sptseg->s_szc);
1865         shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz);
1866         size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz);
1867         npages = btopr(size);
1868 
1869         /*
1870          * Now we need to convert from addr in segshm to addr in segspt.
1871          */
1872         an_idx = seg_page(seg, shm_addr);
1873         segspt_addr = sptseg->s_base + ptob(an_idx);
1874 
1875         ASSERT((segspt_addr + ptob(npages)) <=
1876             (sptseg->s_base + sptd->spt_realsize));
1877         ASSERT(segspt_addr < (sptseg->s_base + sptseg->s_size));
1878 
1879         switch (type) {
1880 
1881         case F_SOFTLOCK:
1882 
1883                 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages);
1884                 /*
1885                  * Fall through to the F_INVAL case to load up the hat layer
1886                  * entries with the HAT_LOAD_LOCK flag.
1887                  */
1888                 /* FALLTHRU */
1889         case F_INVAL:
1890 
1891                 if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC))
1892                         return (FC_NOMAP);
1893 
1894                 ppa = kmem_zalloc(npages * sizeof (page_t *), KM_SLEEP);
1895 
1896                 err = spt_anon_getpages(sptseg, segspt_addr, size, ppa);
1897                 if (err != 0) {
1898                         if (type == F_SOFTLOCK) {
1899                                 atomic_add_long((ulong_t *)(
1900                                     &(shmd->shm_softlockcnt)), -npages);
1901                         }
1902                         goto dism_err;
1903                 }
1904                 AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER);
1905                 a = segspt_addr;
1906                 pidx = 0;
1907                 if (type == F_SOFTLOCK) {
1908 
1909                         /*
1910                          * Load up the translation keeping it
1911                          * locked and don't unlock the page.
1912                          */
1913                         for (; pidx < npages; a += pgsz, pidx += pgcnt) {
1914                                 hat_memload_array(sptseg->s_as->a_hat,
1915                                     a, pgsz, &ppa[pidx], sptd->spt_prot,
1916                                     HAT_LOAD_LOCK | HAT_LOAD_SHARE);
1917                         }
1918                 } else {
1919                         if (hat == seg->s_as->a_hat) {
1920 
1921                                 /*
1922                                  * Migrate pages marked for migration
1923                                  */
1924                                 if (lgrp_optimizations())
1925                                         page_migrate(seg, shm_addr, ppa,
1926                                             npages);
1927 
1928                                 /* CPU HAT */
1929                                 for (; pidx < npages;
1930                                     a += pgsz, pidx += pgcnt) {
1931                                         hat_memload_array(sptseg->s_as->a_hat,
1932                                             a, pgsz, &ppa[pidx],
1933                                             sptd->spt_prot,
1934                                             HAT_LOAD_SHARE);
1935                                 }
1936                         } else {
1937                                 /* XHAT. Pass real address */
1938                                 hat_memload_array(hat, shm_addr,
1939                                     size, ppa, sptd->spt_prot, HAT_LOAD_SHARE);
1940                         }
1941 
1942                         /*
1943                          * And now drop the SE_SHARED lock(s).
1944                          */
1945                         if (dyn_ism_unmap) {
1946                                 for (i = 0; i < npages; i++) {
1947                                         page_unlock(ppa[i]);
1948                                 }
1949                         }
1950                 }
1951 
1952                 if (!dyn_ism_unmap) {
1953                         if (hat_share(seg->s_as->a_hat, shm_addr,
1954                             curspt->a_hat, segspt_addr, ptob(npages),
1955                             seg->s_szc) != 0) {
1956                                 panic("hat_share err in DISM fault");
1957                                 /* NOTREACHED */
1958                         }
1959                         if (type == F_INVAL) {
1960                                 for (i = 0; i < npages; i++) {
1961                                         page_unlock(ppa[i]);
1962                                 }
1963                         }
1964                 }
1965                 AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock);
1966 dism_err:
1967                 kmem_free(ppa, npages * sizeof (page_t *));
1968                 return (err);
1969 
1970         case F_SOFTUNLOCK:
1971 
1972                 /*
1973                  * This is a bit ugly, we pass in the real seg pointer,
1974                  * but the segspt_addr is the virtual address within the
1975                  * dummy seg.
1976                  */
1977                 segspt_softunlock(seg, segspt_addr, size, rw);
1978                 return (0);
1979 
1980         case F_PROT:
1981 
1982                 /*
1983                  * This takes care of the unusual case where a user
1984                  * allocates a stack in shared memory and a register
1985                  * window overflow is written to that stack page before
1986                  * it is otherwise modified.
1987                  *
1988                  * We can get away with this because ISM segments are
1989                  * always rw. Other than this unusual case, there
1990                  * should be no instances of protection violations.
1991                  */
1992                 return (0);
1993 
1994         default:
1995 #ifdef DEBUG
1996                 panic("segspt_dismfault default type?");
1997 #else
1998                 return (FC_NOMAP);
1999 #endif
2000         }
2001 }
2002 
2003 
2004 faultcode_t
2005 segspt_shmfault(struct hat *hat, struct seg *seg, caddr_t addr,
2006     size_t len, enum fault_type type, enum seg_rw rw)
2007 {
2008         struct shm_data         *shmd = (struct shm_data *)seg->s_data;
2009         struct seg              *sptseg = shmd->shm_sptseg;
2010         struct as               *curspt = shmd->shm_sptas;
2011         struct spt_data         *sptd   = sptseg->s_data;
2012         pgcnt_t npages;
2013         size_t size;
2014         caddr_t sptseg_addr, shm_addr;
2015         page_t *pp, **ppa;
2016         int     i;
2017         u_offset_t offset;
2018         ulong_t anon_index = 0;
2019         struct vnode *vp;
2020         struct anon_map *amp;           /* XXX - for locknest */
2021         struct anon *ap = NULL;
2022         size_t          pgsz;
2023         pgcnt_t         pgcnt;
2024         caddr_t         a;
2025         pgcnt_t         pidx;
2026         size_t          sz;
2027 
2028 #ifdef lint
2029         hat = hat;
2030 #endif
2031 
2032         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2033 
2034         if (sptd->spt_flags & SHM_PAGEABLE) {
2035                 return (segspt_dismfault(hat, seg, addr, len, type, rw));
2036         }
2037 
2038         /*
2039          * Because of the way spt is implemented
2040          * the realsize of the segment does not have to be
2041          * equal to the segment size itself. The segment size is
2042          * often in multiples of a page size larger than PAGESIZE.
2043          * The realsize is rounded up to the nearest PAGESIZE
2044          * based on what the user requested. This is a bit of
2045          * ungliness that is historical but not easily fixed
2046          * without re-designing the higher levels of ISM.
2047          */
2048         ASSERT(addr >= seg->s_base);
2049         if (((addr + len) - seg->s_base) > sptd->spt_realsize)
2050                 return (FC_NOMAP);
2051         /*
2052          * For all of the following cases except F_PROT, we need to
2053          * make any necessary adjustments to addr and len
2054          * and get all of the necessary page_t's into an array called ppa[].
2055          *
2056          * The code in shmat() forces base addr and len of ISM segment
2057          * to be aligned to largest page size supported. Therefore,
2058          * we are able to handle F_SOFTLOCK and F_INVAL calls in "large
2059          * pagesize" chunks. We want to make sure that we HAT_LOAD_LOCK
2060          * in large pagesize chunks, or else we will screw up the HAT
2061          * layer by calling hat_memload_array() with differing page sizes
2062          * over a given virtual range.
2063          */
2064         pgsz = page_get_pagesize(sptseg->s_szc);
2065         pgcnt = page_get_pagecnt(sptseg->s_szc);
2066         shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), pgsz);
2067         size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)), pgsz);
2068         npages = btopr(size);
2069 
2070         /*
2071          * Now we need to convert from addr in segshm to addr in segspt.
2072          */
2073         anon_index = seg_page(seg, shm_addr);
2074         sptseg_addr = sptseg->s_base + ptob(anon_index);
2075 
2076         /*
2077          * And now we may have to adjust npages downward if we have
2078          * exceeded the realsize of the segment or initial anon
2079          * allocations.
2080          */
2081         if ((sptseg_addr + ptob(npages)) >
2082             (sptseg->s_base + sptd->spt_realsize))
2083                 size = (sptseg->s_base + sptd->spt_realsize) - sptseg_addr;
2084 
2085         npages = btopr(size);
2086 
2087         ASSERT(sptseg_addr < (sptseg->s_base + sptseg->s_size));
2088         ASSERT((sptd->spt_flags & SHM_PAGEABLE) == 0);
2089 
2090         switch (type) {
2091 
2092         case F_SOFTLOCK:
2093 
2094                 /*
2095                  * availrmem is decremented once during anon_swap_adjust()
2096                  * and is incremented during the anon_unresv(), which is
2097                  * called from shm_rm_amp() when the segment is destroyed.
2098                  */
2099                 atomic_add_long((ulong_t *)(&(shmd->shm_softlockcnt)), npages);
2100                 /*
2101                  * Some platforms assume that ISM pages are SE_SHARED
2102                  * locked for the entire life of the segment.
2103                  */
2104                 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0))
2105                         return (0);
2106                 /*
2107                  * Fall through to the F_INVAL case to load up the hat layer
2108                  * entries with the HAT_LOAD_LOCK flag.
2109                  */
2110 
2111                 /* FALLTHRU */
2112         case F_INVAL:
2113 
2114                 if ((rw == S_EXEC) && !(sptd->spt_prot & PROT_EXEC))
2115                         return (FC_NOMAP);
2116 
2117                 /*
2118                  * Some platforms that do NOT support DYNAMIC_ISM_UNMAP
2119                  * may still rely on this call to hat_share(). That
2120                  * would imply that those hat's can fault on a
2121                  * HAT_LOAD_LOCK translation, which would seem
2122                  * contradictory.
2123                  */
2124                 if (!hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
2125                         if (hat_share(seg->s_as->a_hat, seg->s_base,
2126                             curspt->a_hat, sptseg->s_base,
2127                             sptseg->s_size, sptseg->s_szc) != 0) {
2128                                 panic("hat_share error in ISM fault");
2129                                 /*NOTREACHED*/
2130                         }
2131                         return (0);
2132                 }
2133                 ppa = kmem_zalloc(sizeof (page_t *) * npages, KM_SLEEP);
2134 
2135                 /*
2136                  * I see no need to lock the real seg,
2137                  * here, because all of our work will be on the underlying
2138                  * dummy seg.
2139                  *
2140                  * sptseg_addr and npages now account for large pages.
2141                  */
2142                 amp = sptd->spt_amp;
2143                 ASSERT(amp != NULL);
2144                 anon_index = seg_page(sptseg, sptseg_addr);
2145 
2146                 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2147                 for (i = 0; i < npages; i++) {
2148                         ap = anon_get_ptr(amp->ahp, anon_index++);
2149                         ASSERT(ap != NULL);
2150                         swap_xlate(ap, &vp, &offset);
2151                         pp = page_lookup(vp, offset, SE_SHARED);
2152                         ASSERT(pp != NULL);
2153                         ppa[i] = pp;
2154                 }
2155                 ANON_LOCK_EXIT(&amp->a_rwlock);
2156                 ASSERT(i == npages);
2157 
2158                 /*
2159                  * We are already holding the as->a_lock on the user's
2160                  * real segment, but we need to hold the a_lock on the
2161                  * underlying dummy as. This is mostly to satisfy the
2162                  * underlying HAT layer.
2163                  */
2164                 AS_LOCK_ENTER(sptseg->s_as, &sptseg->s_as->a_lock, RW_READER);
2165                 a = sptseg_addr;
2166                 pidx = 0;
2167                 if (type == F_SOFTLOCK) {
2168                         /*
2169                          * Load up the translation keeping it
2170                          * locked and don't unlock the page.
2171                          */
2172                         for (; pidx < npages; a += pgsz, pidx += pgcnt) {
2173                                 sz = MIN(pgsz, ptob(npages - pidx));
2174                                 hat_memload_array(sptseg->s_as->a_hat, a,
2175                                     sz, &ppa[pidx], sptd->spt_prot,
2176                                     HAT_LOAD_LOCK | HAT_LOAD_SHARE);
2177                         }
2178                 } else {
2179                         if (hat == seg->s_as->a_hat) {
2180 
2181                                 /*
2182                                  * Migrate pages marked for migration.
2183                                  */
2184                                 if (lgrp_optimizations())
2185                                         page_migrate(seg, shm_addr, ppa,
2186                                             npages);
2187 
2188                                 /* CPU HAT */
2189                                 for (; pidx < npages;
2190                                     a += pgsz, pidx += pgcnt) {
2191                                         sz = MIN(pgsz, ptob(npages - pidx));
2192                                         hat_memload_array(sptseg->s_as->a_hat,
2193                                             a, sz, &ppa[pidx],
2194                                             sptd->spt_prot, HAT_LOAD_SHARE);
2195                                 }
2196                         } else {
2197                                 /* XHAT. Pass real address */
2198                                 hat_memload_array(hat, shm_addr,
2199                                     ptob(npages), ppa, sptd->spt_prot,
2200                                     HAT_LOAD_SHARE);
2201                         }
2202 
2203                         /*
2204                          * And now drop the SE_SHARED lock(s).
2205                          */
2206                         for (i = 0; i < npages; i++)
2207                                 page_unlock(ppa[i]);
2208                 }
2209                 AS_LOCK_EXIT(sptseg->s_as, &sptseg->s_as->a_lock);
2210 
2211                 kmem_free(ppa, sizeof (page_t *) * npages);
2212                 return (0);
2213         case F_SOFTUNLOCK:
2214 
2215                 /*
2216                  * This is a bit ugly, we pass in the real seg pointer,
2217                  * but the sptseg_addr is the virtual address within the
2218                  * dummy seg.
2219                  */
2220                 segspt_softunlock(seg, sptseg_addr, ptob(npages), rw);
2221                 return (0);
2222 
2223         case F_PROT:
2224 
2225                 /*
2226                  * This takes care of the unusual case where a user
2227                  * allocates a stack in shared memory and a register
2228                  * window overflow is written to that stack page before
2229                  * it is otherwise modified.
2230                  *
2231                  * We can get away with this because ISM segments are
2232                  * always rw. Other than this unusual case, there
2233                  * should be no instances of protection violations.
2234                  */
2235                 return (0);
2236 
2237         default:
2238 #ifdef DEBUG
2239                 cmn_err(CE_WARN, "segspt_shmfault default type?");
2240 #endif
2241                 return (FC_NOMAP);
2242         }
2243 }
2244 
2245 /*ARGSUSED*/
2246 static faultcode_t
2247 segspt_shmfaulta(struct seg *seg, caddr_t addr)
2248 {
2249         return (0);
2250 }
2251 
2252 /*ARGSUSED*/
2253 static int
2254 segspt_shmkluster(struct seg *seg, caddr_t addr, ssize_t delta)
2255 {
2256         return (0);
2257 }
2258 
2259 /*ARGSUSED*/
2260 static size_t
2261 segspt_shmswapout(struct seg *seg)
2262 {
2263         return (0);
2264 }
2265 
2266 /*
2267  * duplicate the shared page tables
2268  */
2269 int
2270 segspt_shmdup(struct seg *seg, struct seg *newseg)
2271 {
2272         struct shm_data         *shmd = (struct shm_data *)seg->s_data;
2273         struct anon_map         *amp = shmd->shm_amp;
2274         struct shm_data         *shmd_new;
2275         struct seg              *spt_seg = shmd->shm_sptseg;
2276         struct spt_data         *sptd = spt_seg->s_data;
2277         int                     error = 0;
2278 
2279         ASSERT(seg->s_as && AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock));
2280 
2281         shmd_new = kmem_zalloc((sizeof (*shmd_new)), KM_SLEEP);
2282         newseg->s_data = (void *)shmd_new;
2283         shmd_new->shm_sptas = shmd->shm_sptas;
2284         shmd_new->shm_amp = amp;
2285         shmd_new->shm_sptseg = shmd->shm_sptseg;
2286         newseg->s_ops = &segspt_shmops;
2287         newseg->s_szc = seg->s_szc;
2288         ASSERT(seg->s_szc == shmd->shm_sptseg->s_szc);
2289 
2290         ANON_LOCK_ENTER(&amp->a_rwlock, RW_WRITER);
2291         amp->refcnt++;
2292         ANON_LOCK_EXIT(&amp->a_rwlock);
2293 
2294         if (sptd->spt_flags & SHM_PAGEABLE) {
2295                 shmd_new->shm_vpage = kmem_zalloc(btopr(amp->size), KM_SLEEP);
2296                 shmd_new->shm_lckpgs = 0;
2297                 if (hat_supported(HAT_DYNAMIC_ISM_UNMAP, (void *)0)) {
2298                         if ((error = hat_share(newseg->s_as->a_hat,
2299                             newseg->s_base, shmd->shm_sptas->a_hat, SEGSPTADDR,
2300                             seg->s_size, seg->s_szc)) != 0) {
2301                                 kmem_free(shmd_new->shm_vpage,
2302                                     btopr(amp->size));
2303                         }
2304                 }
2305                 return (error);
2306         } else {
2307                 return (hat_share(newseg->s_as->a_hat, newseg->s_base,
2308                     shmd->shm_sptas->a_hat, SEGSPTADDR, seg->s_size,
2309                     seg->s_szc));
2310 
2311         }
2312 }
2313 
2314 /*ARGSUSED*/
2315 int
2316 segspt_shmcheckprot(struct seg *seg, caddr_t addr, size_t size, uint_t prot)
2317 {
2318         struct shm_data *shmd = (struct shm_data *)seg->s_data;
2319         struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2320 
2321         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2322 
2323         /*
2324          * ISM segment is always rw.
2325          */
2326         return (((sptd->spt_prot & prot) != prot) ? EACCES : 0);
2327 }
2328 
2329 /*
2330  * Return an array of locked large pages, for empty slots allocate
2331  * private zero-filled anon pages.
2332  */
2333 static int
2334 spt_anon_getpages(
2335         struct seg *sptseg,
2336         caddr_t sptaddr,
2337         size_t len,
2338         page_t *ppa[])
2339 {
2340         struct  spt_data *sptd = sptseg->s_data;
2341         struct  anon_map *amp = sptd->spt_amp;
2342         enum    seg_rw rw = sptd->spt_prot;
2343         uint_t  szc = sptseg->s_szc;
2344         size_t  pg_sz, share_sz = page_get_pagesize(szc);
2345         pgcnt_t lp_npgs;
2346         caddr_t lp_addr, e_sptaddr;
2347         uint_t  vpprot, ppa_szc = 0;
2348         struct  vpage *vpage = NULL;
2349         ulong_t j, ppa_idx;
2350         int     err, ierr = 0;
2351         pgcnt_t an_idx;
2352         anon_sync_obj_t cookie;
2353         int anon_locked = 0;
2354         pgcnt_t amp_pgs;
2355 
2356 
2357         ASSERT(IS_P2ALIGNED(sptaddr, share_sz) && IS_P2ALIGNED(len, share_sz));
2358         ASSERT(len != 0);
2359 
2360         pg_sz = share_sz;
2361         lp_npgs = btop(pg_sz);
2362         lp_addr = sptaddr;
2363         e_sptaddr = sptaddr + len;
2364         an_idx = seg_page(sptseg, sptaddr);
2365         ppa_idx = 0;
2366 
2367         ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2368 
2369         amp_pgs = page_get_pagecnt(amp->a_szc);
2370 
2371         /*CONSTCOND*/
2372         while (1) {
2373                 for (; lp_addr < e_sptaddr;
2374                     an_idx += lp_npgs, lp_addr += pg_sz, ppa_idx += lp_npgs) {
2375 
2376                         /*
2377                          * If we're currently locked, and we get to a new
2378                          * page, unlock our current anon chunk.
2379                          */
2380                         if (anon_locked && P2PHASE(an_idx, amp_pgs) == 0) {
2381                                 anon_array_exit(&cookie);
2382                                 anon_locked = 0;
2383                         }
2384                         if (!anon_locked) {
2385                                 anon_array_enter(amp, an_idx, &cookie);
2386                                 anon_locked = 1;
2387                         }
2388                         ppa_szc = (uint_t)-1;
2389                         ierr = anon_map_getpages(amp, an_idx, szc, sptseg,
2390                             lp_addr, sptd->spt_prot, &vpprot, &ppa[ppa_idx],
2391                             &ppa_szc, vpage, rw, 0, segvn_anypgsz, 0, kcred);
2392 
2393                         if (ierr != 0) {
2394                                 if (ierr > 0) {
2395                                         err = FC_MAKE_ERR(ierr);
2396                                         goto lpgs_err;
2397                                 }
2398                                 break;
2399                         }
2400                 }
2401                 if (lp_addr == e_sptaddr) {
2402                         break;
2403                 }
2404                 ASSERT(lp_addr < e_sptaddr);
2405 
2406                 /*
2407                  * ierr == -1 means we failed to allocate a large page.
2408                  * so do a size down operation.
2409                  *
2410                  * ierr == -2 means some other process that privately shares
2411                  * pages with this process has allocated a larger page and we
2412                  * need to retry with larger pages. So do a size up
2413                  * operation. This relies on the fact that large pages are
2414                  * never partially shared i.e. if we share any constituent
2415                  * page of a large page with another process we must share the
2416                  * entire large page. Note this cannot happen for SOFTLOCK
2417                  * case, unless current address (lpaddr) is at the beginning
2418                  * of the next page size boundary because the other process
2419                  * couldn't have relocated locked pages.
2420                  */
2421                 ASSERT(ierr == -1 || ierr == -2);
2422                 if (segvn_anypgsz) {
2423                         ASSERT(ierr == -2 || szc != 0);
2424                         ASSERT(ierr == -1 || szc < sptseg->s_szc);
2425                         szc = (ierr == -1) ? szc - 1 : szc + 1;
2426                 } else {
2427                         /*
2428                          * For faults and segvn_anypgsz == 0
2429                          * we need to be careful not to loop forever
2430                          * if existing page is found with szc other
2431                          * than 0 or seg->s_szc. This could be due
2432                          * to page relocations on behalf of DR or
2433                          * more likely large page creation. For this
2434                          * case simply re-size to existing page's szc
2435                          * if returned by anon_map_getpages().
2436                          */
2437                         if (ppa_szc == (uint_t)-1) {
2438                                 szc = (ierr == -1) ? 0 : sptseg->s_szc;
2439                         } else {
2440                                 ASSERT(ppa_szc <= sptseg->s_szc);
2441                                 ASSERT(ierr == -2 || ppa_szc < szc);
2442                                 ASSERT(ierr == -1 || ppa_szc > szc);
2443                                 szc = ppa_szc;
2444                         }
2445                 }
2446                 pg_sz = page_get_pagesize(szc);
2447                 lp_npgs = btop(pg_sz);
2448                 ASSERT(IS_P2ALIGNED(lp_addr, pg_sz));
2449         }
2450         if (anon_locked) {
2451                 anon_array_exit(&cookie);
2452         }
2453         ANON_LOCK_EXIT(&amp->a_rwlock);
2454         return (0);
2455 
2456 lpgs_err:
2457         if (anon_locked) {
2458                 anon_array_exit(&cookie);
2459         }
2460         ANON_LOCK_EXIT(&amp->a_rwlock);
2461         for (j = 0; j < ppa_idx; j++)
2462                 page_unlock(ppa[j]);
2463         return (err);
2464 }
2465 
2466 /*
2467  * count the number of bytes in a set of spt pages that are currently not
2468  * locked
2469  */
2470 static rctl_qty_t
2471 spt_unlockedbytes(pgcnt_t npages, page_t **ppa)
2472 {
2473         ulong_t i;
2474         rctl_qty_t unlocked = 0;
2475 
2476         for (i = 0; i < npages; i++) {
2477                 if (ppa[i]->p_lckcnt == 0)
2478                         unlocked += PAGESIZE;
2479         }
2480         return (unlocked);
2481 }
2482 
2483 extern  u_longlong_t randtick(void);
2484 /* number of locks to reserve/skip by spt_lockpages() and spt_unlockpages() */
2485 #define NLCK    (NCPU_P2)
2486 /* Random number with a range [0, n-1], n must be power of two */
2487 #define RAND_P2(n)      \
2488         ((((long)curthread >> PTR24_LSB) ^ (long)randtick()) & ((n) - 1))
2489 
2490 int
2491 spt_lockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages,
2492     page_t **ppa, ulong_t *lockmap, size_t pos,
2493     rctl_qty_t *locked)
2494 {
2495         struct  shm_data *shmd = seg->s_data;
2496         struct  spt_data *sptd = shmd->shm_sptseg->s_data;
2497         ulong_t i;
2498         int     kernel;
2499         pgcnt_t nlck = 0;
2500         int     rv = 0;
2501         int     use_reserved = 1;
2502 
2503         /* return the number of bytes actually locked */
2504         *locked = 0;
2505 
2506         /*
2507          * To avoid contention on freemem_lock, availrmem and pages_locked
2508          * global counters are updated only every nlck locked pages instead of
2509          * every time.  Reserve nlck locks up front and deduct from this
2510          * reservation for each page that requires a lock.  When the reservation
2511          * is consumed, reserve again.  nlck is randomized, so the competing
2512          * threads do not fall into a cyclic lock contention pattern. When
2513          * memory is low, the lock ahead is disabled, and instead page_pp_lock()
2514          * is used to lock pages.
2515          */
2516         for (i = 0; i < npages; anon_index++, pos++, i++) {
2517                 if (nlck == 0 && use_reserved == 1) {
2518                         nlck = NLCK + RAND_P2(NLCK);
2519                         /* if fewer loops left, decrease nlck */
2520                         nlck = MIN(nlck, npages - i);
2521                         /*
2522                          * Reserve nlck locks up front and deduct from this
2523                          * reservation for each page that requires a lock.  When
2524                          * the reservation is consumed, reserve again.
2525                          */
2526                         mutex_enter(&freemem_lock);
2527                         if ((availrmem - nlck) < pages_pp_maximum) {
2528                                 /* Do not do advance memory reserves */
2529                                 use_reserved = 0;
2530                         } else {
2531                                 availrmem       -= nlck;
2532                                 pages_locked    += nlck;
2533                         }
2534                         mutex_exit(&freemem_lock);
2535                 }
2536                 if (!(shmd->shm_vpage[anon_index] & DISM_PG_LOCKED)) {
2537                         if (sptd->spt_ppa_lckcnt[anon_index] <
2538                             (ushort_t)DISM_LOCK_MAX) {
2539                                 if (++sptd->spt_ppa_lckcnt[anon_index] ==
2540                                     (ushort_t)DISM_LOCK_MAX) {
2541                                         cmn_err(CE_WARN,
2542                                             "DISM page lock limit "
2543                                             "reached on DISM offset 0x%lx\n",
2544                                             anon_index << PAGESHIFT);
2545                                 }
2546                                 kernel = (sptd->spt_ppa &&
2547                                     sptd->spt_ppa[anon_index]);
2548                                 if (!page_pp_lock(ppa[i], 0, kernel ||
2549                                     use_reserved)) {
2550                                         sptd->spt_ppa_lckcnt[anon_index]--;
2551                                         rv = EAGAIN;
2552                                         break;
2553                                 }
2554                                 /* if this is a newly locked page, count it */
2555                                 if (ppa[i]->p_lckcnt == 1) {
2556                                         if (kernel == 0 && use_reserved == 1)
2557                                                 nlck--;
2558                                         *locked += PAGESIZE;
2559                                 }
2560                                 shmd->shm_lckpgs++;
2561                                 shmd->shm_vpage[anon_index] |= DISM_PG_LOCKED;
2562                                 if (lockmap != NULL)
2563                                         BT_SET(lockmap, pos);
2564                         }
2565                 }
2566         }
2567         /* Return unused lock reservation */
2568         if (nlck != 0 && use_reserved == 1) {
2569                 mutex_enter(&freemem_lock);
2570                 availrmem       += nlck;
2571                 pages_locked    -= nlck;
2572                 mutex_exit(&freemem_lock);
2573         }
2574 
2575         return (rv);
2576 }
2577 
2578 int
2579 spt_unlockpages(struct seg *seg, pgcnt_t anon_index, pgcnt_t npages,
2580     rctl_qty_t *unlocked)
2581 {
2582         struct shm_data *shmd = seg->s_data;
2583         struct spt_data *sptd = shmd->shm_sptseg->s_data;
2584         struct anon_map *amp = sptd->spt_amp;
2585         struct anon     *ap;
2586         struct vnode    *vp;
2587         u_offset_t      off;
2588         struct page     *pp;
2589         int             kernel;
2590         anon_sync_obj_t cookie;
2591         ulong_t         i;
2592         pgcnt_t         nlck = 0;
2593         pgcnt_t         nlck_limit = NLCK;
2594 
2595         ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2596         for (i = 0; i < npages; i++, anon_index++) {
2597                 if (shmd->shm_vpage[anon_index] & DISM_PG_LOCKED) {
2598                         anon_array_enter(amp, anon_index, &cookie);
2599                         ap = anon_get_ptr(amp->ahp, anon_index);
2600                         ASSERT(ap);
2601 
2602                         swap_xlate(ap, &vp, &off);
2603                         anon_array_exit(&cookie);
2604                         pp = page_lookup(vp, off, SE_SHARED);
2605                         ASSERT(pp);
2606                         /*
2607                          * availrmem is decremented only for pages which are not
2608                          * in seg pcache, for pages in seg pcache availrmem was
2609                          * decremented in _dismpagelock()
2610                          */
2611                         kernel = (sptd->spt_ppa && sptd->spt_ppa[anon_index]);
2612                         ASSERT(pp->p_lckcnt > 0);
2613 
2614                         /*
2615                          * lock page but do not change availrmem, we do it
2616                          * ourselves every nlck loops.
2617                          */
2618                         page_pp_unlock(pp, 0, 1);
2619                         if (pp->p_lckcnt == 0) {
2620                                 if (kernel == 0)
2621                                         nlck++;
2622                                 *unlocked += PAGESIZE;
2623                         }
2624                         page_unlock(pp);
2625                         shmd->shm_vpage[anon_index] &= ~DISM_PG_LOCKED;
2626                         sptd->spt_ppa_lckcnt[anon_index]--;
2627                         shmd->shm_lckpgs--;
2628                 }
2629 
2630                 /*
2631                  * To reduce freemem_lock contention, do not update availrmem
2632                  * until at least NLCK pages have been unlocked.
2633                  * 1. No need to update if nlck is zero
2634                  * 2. Always update if the last iteration
2635                  */
2636                 if (nlck > 0 && (nlck == nlck_limit || i == npages - 1)) {
2637                         mutex_enter(&freemem_lock);
2638                         availrmem       += nlck;
2639                         pages_locked    -= nlck;
2640                         mutex_exit(&freemem_lock);
2641                         nlck = 0;
2642                         nlck_limit = NLCK + RAND_P2(NLCK);
2643                 }
2644         }
2645         ANON_LOCK_EXIT(&amp->a_rwlock);
2646 
2647         return (0);
2648 }
2649 
2650 /*ARGSUSED*/
2651 static int
2652 segspt_shmlockop(struct seg *seg, caddr_t addr, size_t len,
2653     int attr, int op, ulong_t *lockmap, size_t pos)
2654 {
2655         struct shm_data *shmd = seg->s_data;
2656         struct seg      *sptseg = shmd->shm_sptseg;
2657         struct spt_data *sptd = sptseg->s_data;
2658         struct kshmid   *sp = sptd->spt_amp->a_sp;
2659         pgcnt_t         npages, a_npages;
2660         page_t          **ppa;
2661         pgcnt_t         an_idx, a_an_idx, ppa_idx;
2662         caddr_t         spt_addr, a_addr;       /* spt and aligned address */
2663         size_t          a_len;                  /* aligned len */
2664         size_t          share_sz;
2665         ulong_t         i;
2666         int             sts = 0;
2667         rctl_qty_t      unlocked = 0;
2668         rctl_qty_t      locked = 0;
2669         struct proc     *p = curproc;
2670         kproject_t      *proj;
2671 
2672         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2673         ASSERT(sp != NULL);
2674 
2675         if ((sptd->spt_flags & SHM_PAGEABLE) == 0) {
2676                 return (0);
2677         }
2678 
2679         addr = (caddr_t)((uintptr_t)addr & (uintptr_t)PAGEMASK);
2680         an_idx = seg_page(seg, addr);
2681         npages = btopr(len);
2682 
2683         if (an_idx + npages > btopr(shmd->shm_amp->size)) {
2684                 return (ENOMEM);
2685         }
2686 
2687         /*
2688          * A shm's project never changes, so no lock needed.
2689          * The shm has a hold on the project, so it will not go away.
2690          * Since we have a mapping to shm within this zone, we know
2691          * that the zone will not go away.
2692          */
2693         proj = sp->shm_perm.ipc_proj;
2694 
2695         if (op == MC_LOCK) {
2696 
2697                 /*
2698                  * Need to align addr and size request if they are not
2699                  * aligned so we can always allocate large page(s) however
2700                  * we only lock what was requested in initial request.
2701                  */
2702                 share_sz = page_get_pagesize(sptseg->s_szc);
2703                 a_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_sz);
2704                 a_len = P2ROUNDUP((uintptr_t)(((addr + len) - a_addr)),
2705                     share_sz);
2706                 a_npages = btop(a_len);
2707                 a_an_idx = seg_page(seg, a_addr);
2708                 spt_addr = sptseg->s_base + ptob(a_an_idx);
2709                 ppa_idx = an_idx - a_an_idx;
2710 
2711                 if ((ppa = kmem_zalloc(((sizeof (page_t *)) * a_npages),
2712                     KM_NOSLEEP)) == NULL) {
2713                         return (ENOMEM);
2714                 }
2715 
2716                 /*
2717                  * Don't cache any new pages for IO and
2718                  * flush any cached pages.
2719                  */
2720                 mutex_enter(&sptd->spt_lock);
2721                 if (sptd->spt_ppa != NULL)
2722                         sptd->spt_flags |= DISM_PPA_CHANGED;
2723 
2724                 sts = spt_anon_getpages(sptseg, spt_addr, a_len, ppa);
2725                 if (sts != 0) {
2726                         mutex_exit(&sptd->spt_lock);
2727                         kmem_free(ppa, ((sizeof (page_t *)) * a_npages));
2728                         return (sts);
2729                 }
2730 
2731                 mutex_enter(&sp->shm_mlock);
2732                 /* enforce locked memory rctl */
2733                 unlocked = spt_unlockedbytes(npages, &ppa[ppa_idx]);
2734 
2735                 mutex_enter(&p->p_lock);
2736                 if (rctl_incr_locked_mem(p, proj, unlocked, 0)) {
2737                         mutex_exit(&p->p_lock);
2738                         sts = EAGAIN;
2739                 } else {
2740                         mutex_exit(&p->p_lock);
2741                         sts = spt_lockpages(seg, an_idx, npages,
2742                             &ppa[ppa_idx], lockmap, pos, &locked);
2743 
2744                         /*
2745                          * correct locked count if not all pages could be
2746                          * locked
2747                          */
2748                         if ((unlocked - locked) > 0) {
2749                                 rctl_decr_locked_mem(NULL, proj,
2750                                     (unlocked - locked), 0);
2751                         }
2752                 }
2753                 /*
2754                  * unlock pages
2755                  */
2756                 for (i = 0; i < a_npages; i++)
2757                         page_unlock(ppa[i]);
2758                 if (sptd->spt_ppa != NULL)
2759                         sptd->spt_flags |= DISM_PPA_CHANGED;
2760                 mutex_exit(&sp->shm_mlock);
2761                 mutex_exit(&sptd->spt_lock);
2762 
2763                 kmem_free(ppa, ((sizeof (page_t *)) * a_npages));
2764 
2765         } else if (op == MC_UNLOCK) { /* unlock */
2766                 page_t          **ppa;
2767 
2768                 mutex_enter(&sptd->spt_lock);
2769                 if (shmd->shm_lckpgs == 0) {
2770                         mutex_exit(&sptd->spt_lock);
2771                         return (0);
2772                 }
2773                 /*
2774                  * Don't cache new IO pages.
2775                  */
2776                 if (sptd->spt_ppa != NULL)
2777                         sptd->spt_flags |= DISM_PPA_CHANGED;
2778 
2779                 mutex_enter(&sp->shm_mlock);
2780                 sts = spt_unlockpages(seg, an_idx, npages, &unlocked);
2781                 if ((ppa = sptd->spt_ppa) != NULL)
2782                         sptd->spt_flags |= DISM_PPA_CHANGED;
2783                 mutex_exit(&sptd->spt_lock);
2784 
2785                 rctl_decr_locked_mem(NULL, proj, unlocked, 0);
2786                 mutex_exit(&sp->shm_mlock);
2787 
2788                 if (ppa != NULL)
2789                         seg_ppurge_wiredpp(ppa);
2790         }
2791         return (sts);
2792 }
2793 
2794 /*ARGSUSED*/
2795 int
2796 segspt_shmgetprot(struct seg *seg, caddr_t addr, size_t len, uint_t *protv)
2797 {
2798         struct shm_data *shmd = (struct shm_data *)seg->s_data;
2799         struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2800         spgcnt_t pgno = seg_page(seg, addr+len) - seg_page(seg, addr) + 1;
2801 
2802         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2803 
2804         /*
2805          * ISM segment is always rw.
2806          */
2807         while (--pgno >= 0)
2808                 *protv++ = sptd->spt_prot;
2809         return (0);
2810 }
2811 
2812 /*ARGSUSED*/
2813 u_offset_t
2814 segspt_shmgetoffset(struct seg *seg, caddr_t addr)
2815 {
2816         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2817 
2818         /* Offset does not matter in ISM memory */
2819 
2820         return ((u_offset_t)0);
2821 }
2822 
2823 /* ARGSUSED */
2824 int
2825 segspt_shmgettype(struct seg *seg, caddr_t addr)
2826 {
2827         struct shm_data *shmd = (struct shm_data *)seg->s_data;
2828         struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2829 
2830         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2831 
2832         /*
2833          * The shared memory mapping is always MAP_SHARED, SWAP is only
2834          * reserved for DISM
2835          */
2836         return (MAP_SHARED |
2837             ((sptd->spt_flags & SHM_PAGEABLE) ? 0 : MAP_NORESERVE));
2838 }
2839 
2840 /*ARGSUSED*/
2841 int
2842 segspt_shmgetvp(struct seg *seg, caddr_t addr, struct vnode **vpp)
2843 {
2844         struct shm_data *shmd = (struct shm_data *)seg->s_data;
2845         struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2846 
2847         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2848 
2849         *vpp = sptd->spt_vp;
2850         return (0);
2851 }
2852 
2853 /*
2854  * We need to wait for pending IO to complete to a DISM segment in order for
2855  * pages to get kicked out of the seg_pcache.  120 seconds should be more
2856  * than enough time to wait.
2857  */
2858 static clock_t spt_pcache_wait = 120;
2859 
2860 /*ARGSUSED*/
2861 static int
2862 segspt_shmadvise(struct seg *seg, caddr_t addr, size_t len, uint_t behav)
2863 {
2864         struct shm_data *shmd = (struct shm_data *)seg->s_data;
2865         struct spt_data *sptd = (struct spt_data *)shmd->shm_sptseg->s_data;
2866         struct anon_map *amp;
2867         pgcnt_t pg_idx;
2868         ushort_t gen;
2869         clock_t end_lbolt;
2870         int writer;
2871         page_t **ppa;
2872 
2873         ASSERT(seg->s_as && AS_LOCK_HELD(seg->s_as, &seg->s_as->a_lock));
2874 
2875         if (behav == MADV_FREE) {
2876                 if ((sptd->spt_flags & SHM_PAGEABLE) == 0)
2877                         return (0);
2878 
2879                 amp = sptd->spt_amp;
2880                 pg_idx = seg_page(seg, addr);
2881 
2882                 mutex_enter(&sptd->spt_lock);
2883                 if ((ppa = sptd->spt_ppa) == NULL) {
2884                         mutex_exit(&sptd->spt_lock);
2885                         ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2886                         anon_disclaim(amp, pg_idx, len);
2887                         ANON_LOCK_EXIT(&amp->a_rwlock);
2888                         return (0);
2889                 }
2890 
2891                 sptd->spt_flags |= DISM_PPA_CHANGED;
2892                 gen = sptd->spt_gen;
2893 
2894                 mutex_exit(&sptd->spt_lock);
2895 
2896                 /*
2897                  * Purge all DISM cached pages
2898                  */
2899                 seg_ppurge_wiredpp(ppa);
2900 
2901                 /*
2902                  * Drop the AS_LOCK so that other threads can grab it
2903                  * in the as_pageunlock path and hopefully get the segment
2904                  * kicked out of the seg_pcache.  We bump the shm_softlockcnt
2905                  * to keep this segment resident.
2906                  */
2907                 writer = AS_WRITE_HELD(seg->s_as, &seg->s_as->a_lock);
2908                 atomic_inc_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
2909                 AS_LOCK_EXIT(seg->s_as, &seg->s_as->a_lock);
2910 
2911                 mutex_enter(&sptd->spt_lock);
2912 
2913                 end_lbolt = ddi_get_lbolt() + (hz * spt_pcache_wait);
2914 
2915                 /*
2916                  * Try to wait for pages to get kicked out of the seg_pcache.
2917                  */
2918                 while (sptd->spt_gen == gen &&
2919                     (sptd->spt_flags & DISM_PPA_CHANGED) &&
2920                     ddi_get_lbolt() < end_lbolt) {
2921                         if (!cv_timedwait_sig(&sptd->spt_cv,
2922                             &sptd->spt_lock, end_lbolt)) {
2923                                 break;
2924                         }
2925                 }
2926 
2927                 mutex_exit(&sptd->spt_lock);
2928 
2929                 /* Regrab the AS_LOCK and release our hold on the segment */
2930                 AS_LOCK_ENTER(seg->s_as, &seg->s_as->a_lock,
2931                     writer ? RW_WRITER : RW_READER);
2932                 atomic_dec_ulong((ulong_t *)(&(shmd->shm_softlockcnt)));
2933                 if (shmd->shm_softlockcnt <= 0) {
2934                         if (AS_ISUNMAPWAIT(seg->s_as)) {
2935                                 mutex_enter(&seg->s_as->a_contents);
2936                                 if (AS_ISUNMAPWAIT(seg->s_as)) {
2937                                         AS_CLRUNMAPWAIT(seg->s_as);
2938                                         cv_broadcast(&seg->s_as->a_cv);
2939                                 }
2940                                 mutex_exit(&seg->s_as->a_contents);
2941                         }
2942                 }
2943 
2944                 ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
2945                 anon_disclaim(amp, pg_idx, len);
2946                 ANON_LOCK_EXIT(&amp->a_rwlock);
2947         } else if (lgrp_optimizations() && (behav == MADV_ACCESS_LWP ||
2948             behav == MADV_ACCESS_MANY || behav == MADV_ACCESS_DEFAULT)) {
2949                 int                     already_set;
2950                 ulong_t                 anon_index;
2951                 lgrp_mem_policy_t       policy;
2952                 caddr_t                 shm_addr;
2953                 size_t                  share_size;
2954                 size_t                  size;
2955                 struct seg              *sptseg = shmd->shm_sptseg;
2956                 caddr_t                 sptseg_addr;
2957 
2958                 /*
2959                  * Align address and length to page size of underlying segment
2960                  */
2961                 share_size = page_get_pagesize(shmd->shm_sptseg->s_szc);
2962                 shm_addr = (caddr_t)P2ALIGN((uintptr_t)(addr), share_size);
2963                 size = P2ROUNDUP((uintptr_t)(((addr + len) - shm_addr)),
2964                     share_size);
2965 
2966                 amp = shmd->shm_amp;
2967                 anon_index = seg_page(seg, shm_addr);
2968 
2969                 /*
2970                  * And now we may have to adjust size downward if we have
2971                  * exceeded the realsize of the segment or initial anon
2972                  * allocations.
2973                  */
2974                 sptseg_addr = sptseg->s_base + ptob(anon_index);
2975                 if ((sptseg_addr + size) >
2976                     (sptseg->s_base + sptd->spt_realsize))
2977                         size = (sptseg->s_base + sptd->spt_realsize) -
2978                             sptseg_addr;
2979 
2980                 /*
2981                  * Set memory allocation policy for this segment
2982                  */
2983                 policy = lgrp_madv_to_policy(behav, len, MAP_SHARED);
2984                 already_set = lgrp_shm_policy_set(policy, amp, anon_index,
2985                     NULL, 0, len);
2986 
2987                 /*
2988                  * If random memory allocation policy set already,
2989                  * don't bother reapplying it.
2990                  */
2991                 if (already_set && !LGRP_MEM_POLICY_REAPPLICABLE(policy))
2992                         return (0);
2993 
2994                 /*
2995                  * Mark any existing pages in the given range for
2996                  * migration, flushing the I/O page cache, and using
2997                  * underlying segment to calculate anon index and get
2998                  * anonmap and vnode pointer from
2999                  */
3000                 if (shmd->shm_softlockcnt > 0)
3001                         segspt_purge(seg);
3002 
3003                 page_mark_migrate(seg, shm_addr, size, amp, 0, NULL, 0, 0);
3004         }
3005 
3006         return (0);
3007 }
3008 
3009 /*
3010  * get a memory ID for an addr in a given segment
3011  */
3012 static int
3013 segspt_shmgetmemid(struct seg *seg, caddr_t addr, memid_t *memidp)
3014 {
3015         struct shm_data *shmd = (struct shm_data *)seg->s_data;
3016         struct anon     *ap;
3017         size_t          anon_index;
3018         struct anon_map *amp = shmd->shm_amp;
3019         struct spt_data *sptd = shmd->shm_sptseg->s_data;
3020         struct seg      *sptseg = shmd->shm_sptseg;
3021         anon_sync_obj_t cookie;
3022 
3023         anon_index = seg_page(seg, addr);
3024 
3025         if (addr > (seg->s_base + sptd->spt_realsize)) {
3026                 return (EFAULT);
3027         }
3028 
3029         ANON_LOCK_ENTER(&amp->a_rwlock, RW_READER);
3030         anon_array_enter(amp, anon_index, &cookie);
3031         ap = anon_get_ptr(amp->ahp, anon_index);
3032         if (ap == NULL) {
3033                 struct page *pp;
3034                 caddr_t spt_addr = sptseg->s_base + ptob(anon_index);
3035 
3036                 pp = anon_zero(sptseg, spt_addr, &ap, kcred);
3037                 if (pp == NULL) {
3038                         anon_array_exit(&cookie);
3039                         ANON_LOCK_EXIT(&amp->a_rwlock);
3040                         return (ENOMEM);
3041                 }
3042                 (void) anon_set_ptr(amp->ahp, anon_index, ap, ANON_SLEEP);
3043                 page_unlock(pp);
3044         }
3045         anon_array_exit(&cookie);
3046         ANON_LOCK_EXIT(&amp->a_rwlock);
3047         memidp->val[0] = (uintptr_t)ap;
3048         memidp->val[1] = (uintptr_t)addr & PAGEOFFSET;
3049         return (0);
3050 }
3051 
3052 /*
3053  * Get memory allocation policy info for specified address in given segment
3054  */
3055 static lgrp_mem_policy_info_t *
3056 segspt_shmgetpolicy(struct seg *seg, caddr_t addr)
3057 {
3058         struct anon_map         *amp;
3059         ulong_t                 anon_index;
3060         lgrp_mem_policy_info_t  *policy_info;
3061         struct shm_data         *shm_data;
3062 
3063         ASSERT(seg != NULL);
3064 
3065         /*
3066          * Get anon_map from segshm
3067          *
3068          * Assume that no lock needs to be held on anon_map, since
3069          * it should be protected by its reference count which must be
3070          * nonzero for an existing segment
3071          * Need to grab readers lock on policy tree though
3072          */
3073         shm_data = (struct shm_data *)seg->s_data;
3074         if (shm_data == NULL)
3075                 return (NULL);
3076         amp = shm_data->shm_amp;
3077         ASSERT(amp->refcnt != 0);
3078 
3079         /*
3080          * Get policy info
3081          *
3082          * Assume starting anon index of 0
3083          */
3084         anon_index = seg_page(seg, addr);
3085         policy_info = lgrp_shm_policy_get(amp, anon_index, NULL, 0);
3086 
3087         return (policy_info);
3088 }