1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 
  22 /*
  23  * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
  24  * Copyright 2013, Joyent, Inc. All rights reserved.
  25  */
  26 
  27 #include <sys/types.h>
  28 #include <sys/param.h>
  29 #include <sys/sysmacros.h>
  30 #include <sys/cred.h>
  31 #include <sys/proc.h>
  32 #include <sys/strsubr.h>
  33 #include <sys/priocntl.h>
  34 #include <sys/class.h>
  35 #include <sys/disp.h>
  36 #include <sys/procset.h>
  37 #include <sys/debug.h>
  38 #include <sys/kmem.h>
  39 #include <sys/errno.h>
  40 #include <sys/systm.h>
  41 #include <sys/schedctl.h>
  42 #include <sys/vmsystm.h>
  43 #include <sys/atomic.h>
  44 #include <sys/project.h>
  45 #include <sys/modctl.h>
  46 #include <sys/fss.h>
  47 #include <sys/fsspriocntl.h>
  48 #include <sys/cpupart.h>
  49 #include <sys/zone.h>
  50 #include <vm/rm.h>
  51 #include <vm/seg_kmem.h>
  52 #include <sys/tnf_probe.h>
  53 #include <sys/policy.h>
  54 #include <sys/sdt.h>
  55 #include <sys/cpucaps.h>
  56 
  57 /*
  58  * FSS Data Structures:
  59  *
  60  *                 fsszone
  61  *                  -----           -----
  62  *  -----          |     |         |     |
  63  * |     |-------->|     |<------->|     |<---->...
  64  * |     |          -----           -----
  65  * |     |          ^    ^            ^
  66  * |     |---       |     \            \
  67  *  -----    |      |      \            \
  68  * fsspset   |      |       \            \
  69  *           |      |        \            \
  70  *           |    -----       -----       -----
  71  *            -->|     |<--->|     |<--->|     |
  72  *               |     |     |     |     |     |
  73  *                -----       -----       -----
  74  *               fssproj
  75  *
  76  *
  77  * That is, fsspsets contain a list of fsszone's that are currently active in
  78  * the pset, and a list of fssproj's, corresponding to projects with runnable
  79  * threads on the pset.  fssproj's in turn point to the fsszone which they
  80  * are a member of.
  81  *
  82  * An fssproj_t is removed when there are no threads in it.
  83  *
  84  * An fsszone_t is removed when there are no projects with threads in it.
  85  *
  86  * Projects in a zone compete with each other for cpu time, receiving cpu
  87  * allocation within a zone proportional to fssproj->fssp_shares
  88  * (project.cpu-shares); at a higher level zones compete with each other,
  89  * receiving allocation in a pset proportional to fsszone->fssz_shares
  90  * (zone.cpu-shares).  See fss_decay_usage() for the precise formula.
  91  */
  92 
  93 static pri_t fss_init(id_t, int, classfuncs_t **);
  94 
  95 static struct sclass fss = {
  96         "FSS",
  97         fss_init,
  98         0
  99 };
 100 
 101 extern struct mod_ops mod_schedops;
 102 
 103 /*
 104  * Module linkage information for the kernel.
 105  */
 106 static struct modlsched modlsched = {
 107         &mod_schedops, "fair share scheduling class", &fss
 108 };
 109 
 110 static struct modlinkage modlinkage = {
 111         MODREV_1, (void *)&modlsched, NULL
 112 };
 113 
 114 #define FSS_MAXUPRI     60
 115 
 116 /*
 117  * The fssproc_t structures are kept in an array of circular doubly linked
 118  * lists.  A hash on the thread pointer is used to determine which list each
 119  * thread should be placed in.  Each list has a dummy "head" which is never
 120  * removed, so the list is never empty.  fss_update traverses these lists to
 121  * update the priorities of threads that have been waiting on the run queue.
 122  */
 123 #define FSS_LISTS               16 /* number of lists, must be power of 2 */
 124 #define FSS_LIST_HASH(t)        (((uintptr_t)(t) >> 9) & (FSS_LISTS - 1))
 125 #define FSS_LIST_NEXT(i)        (((i) + 1) & (FSS_LISTS - 1))
 126 
 127 #define FSS_LIST_INSERT(fssproc)                                \
 128 {                                                               \
 129         int index = FSS_LIST_HASH(fssproc->fss_tp);          \
 130         kmutex_t *lockp = &fss_listlock[index];                     \
 131         fssproc_t *headp = &fss_listhead[index];            \
 132         mutex_enter(lockp);                                     \
 133         fssproc->fss_next = headp->fss_next;                      \
 134         fssproc->fss_prev = headp;                           \
 135         headp->fss_next->fss_prev = fssproc;                      \
 136         headp->fss_next = fssproc;                           \
 137         mutex_exit(lockp);                                      \
 138 }
 139 
 140 #define FSS_LIST_DELETE(fssproc)                                \
 141 {                                                               \
 142         int index = FSS_LIST_HASH(fssproc->fss_tp);          \
 143         kmutex_t *lockp = &fss_listlock[index];                     \
 144         mutex_enter(lockp);                                     \
 145         fssproc->fss_prev->fss_next = fssproc->fss_next;       \
 146         fssproc->fss_next->fss_prev = fssproc->fss_prev;       \
 147         mutex_exit(lockp);                                      \
 148 }
 149 
 150 #define FSS_TICK_COST   1000    /* tick cost for threads with nice level = 0 */
 151 
 152 /*
 153  * Decay rate percentages are based on n/128 rather than n/100 so  that
 154  * calculations can avoid having to do an integer divide by 100 (divide
 155  * by FSS_DECAY_BASE == 128 optimizes to an arithmetic shift).
 156  *
 157  * FSS_DECAY_MIN        =  83/128 ~= 65%
 158  * FSS_DECAY_MAX        = 108/128 ~= 85%
 159  * FSS_DECAY_USG        =  96/128 ~= 75%
 160  */
 161 #define FSS_DECAY_MIN   83      /* fsspri decay pct for threads w/ nice -20 */
 162 #define FSS_DECAY_MAX   108     /* fsspri decay pct for threads w/ nice +19 */
 163 #define FSS_DECAY_USG   96      /* fssusage decay pct for projects */
 164 #define FSS_DECAY_BASE  128     /* base for decay percentages above */
 165 
 166 #define FSS_NICE_MIN    0
 167 #define FSS_NICE_MAX    (2 * NZERO - 1)
 168 #define FSS_NICE_RANGE  (FSS_NICE_MAX - FSS_NICE_MIN + 1)
 169 
 170 static int      fss_nice_tick[FSS_NICE_RANGE];
 171 static int      fss_nice_decay[FSS_NICE_RANGE];
 172 
 173 static pri_t    fss_maxupri = FSS_MAXUPRI; /* maximum FSS user priority */
 174 static pri_t    fss_maxumdpri; /* maximum user mode fss priority */
 175 static pri_t    fss_maxglobpri; /* maximum global priority used by fss class */
 176 static pri_t    fss_minglobpri; /* minimum global priority */
 177 
 178 static fssproc_t fss_listhead[FSS_LISTS];
 179 static kmutex_t fss_listlock[FSS_LISTS];
 180 
 181 static fsspset_t *fsspsets;
 182 static kmutex_t fsspsets_lock;  /* protects fsspsets */
 183 
 184 static id_t     fss_cid;
 185 
 186 static time_t   fss_minrun = 2; /* t_pri becomes 59 within 2 secs */
 187 static time_t   fss_minslp = 2; /* min time on sleep queue for hardswap */
 188 static int      fss_quantum = 11;
 189 
 190 static void     fss_newpri(fssproc_t *);
 191 static void     fss_update(void *);
 192 static int      fss_update_list(int);
 193 static void     fss_change_priority(kthread_t *, fssproc_t *);
 194 
 195 static int      fss_admin(caddr_t, cred_t *);
 196 static int      fss_getclinfo(void *);
 197 static int      fss_parmsin(void *);
 198 static int      fss_parmsout(void *, pc_vaparms_t *);
 199 static int      fss_vaparmsin(void *, pc_vaparms_t *);
 200 static int      fss_vaparmsout(void *, pc_vaparms_t *);
 201 static int      fss_getclpri(pcpri_t *);
 202 static int      fss_alloc(void **, int);
 203 static void     fss_free(void *);
 204 
 205 static int      fss_enterclass(kthread_t *, id_t, void *, cred_t *, void *);
 206 static void     fss_exitclass(void *);
 207 static int      fss_canexit(kthread_t *, cred_t *);
 208 static int      fss_fork(kthread_t *, kthread_t *, void *);
 209 static void     fss_forkret(kthread_t *, kthread_t *);
 210 static void     fss_parmsget(kthread_t *, void *);
 211 static int      fss_parmsset(kthread_t *, void *, id_t, cred_t *);
 212 static void     fss_stop(kthread_t *, int, int);
 213 static void     fss_exit(kthread_t *);
 214 static void     fss_active(kthread_t *);
 215 static void     fss_inactive(kthread_t *);
 216 static pri_t    fss_swapin(kthread_t *, int);
 217 static pri_t    fss_swapout(kthread_t *, int);
 218 static void     fss_trapret(kthread_t *);
 219 static void     fss_preempt(kthread_t *);
 220 static void     fss_setrun(kthread_t *);
 221 static void     fss_sleep(kthread_t *);
 222 static void     fss_tick(kthread_t *);
 223 static void     fss_wakeup(kthread_t *);
 224 static int      fss_donice(kthread_t *, cred_t *, int, int *);
 225 static int      fss_doprio(kthread_t *, cred_t *, int, int *);
 226 static pri_t    fss_globpri(kthread_t *);
 227 static void     fss_yield(kthread_t *);
 228 static void     fss_nullsys();
 229 
 230 static struct classfuncs fss_classfuncs = {
 231         /* class functions */
 232         fss_admin,
 233         fss_getclinfo,
 234         fss_parmsin,
 235         fss_parmsout,
 236         fss_vaparmsin,
 237         fss_vaparmsout,
 238         fss_getclpri,
 239         fss_alloc,
 240         fss_free,
 241 
 242         /* thread functions */
 243         fss_enterclass,
 244         fss_exitclass,
 245         fss_canexit,
 246         fss_fork,
 247         fss_forkret,
 248         fss_parmsget,
 249         fss_parmsset,
 250         fss_stop,
 251         fss_exit,
 252         fss_active,
 253         fss_inactive,
 254         fss_swapin,
 255         fss_swapout,
 256         fss_trapret,
 257         fss_preempt,
 258         fss_setrun,
 259         fss_sleep,
 260         fss_tick,
 261         fss_wakeup,
 262         fss_donice,
 263         fss_globpri,
 264         fss_nullsys,    /* set_process_group */
 265         fss_yield,
 266         fss_doprio,
 267 };
 268 
 269 int
 270 _init()
 271 {
 272         return (mod_install(&modlinkage));
 273 }
 274 
 275 int
 276 _fini()
 277 {
 278         return (EBUSY);
 279 }
 280 
 281 int
 282 _info(struct modinfo *modinfop)
 283 {
 284         return (mod_info(&modlinkage, modinfop));
 285 }
 286 
 287 /*ARGSUSED*/
 288 static int
 289 fss_project_walker(kproject_t *kpj, void *buf)
 290 {
 291         return (0);
 292 }
 293 
 294 void *
 295 fss_allocbuf(int op, int type)
 296 {
 297         fssbuf_t *fssbuf;
 298         void **fsslist;
 299         int cnt;
 300         int i;
 301         size_t size;
 302 
 303         ASSERT(op == FSS_NPSET_BUF || op == FSS_NPROJ_BUF || op == FSS_ONE_BUF);
 304         ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
 305         ASSERT(MUTEX_HELD(&cpu_lock));
 306 
 307         fssbuf = kmem_zalloc(sizeof (fssbuf_t), KM_SLEEP);
 308         switch (op) {
 309         case FSS_NPSET_BUF:
 310                 cnt = cpupart_list(NULL, 0, CP_NONEMPTY);
 311                 break;
 312         case FSS_NPROJ_BUF:
 313                 cnt = project_walk_all(ALL_ZONES, fss_project_walker, NULL);
 314                 break;
 315         case FSS_ONE_BUF:
 316                 cnt = 1;
 317                 break;
 318         }
 319 
 320         switch (type) {
 321         case FSS_ALLOC_PROJ:
 322                 size = sizeof (fssproj_t);
 323                 break;
 324         case FSS_ALLOC_ZONE:
 325                 size = sizeof (fsszone_t);
 326                 break;
 327         }
 328         fsslist = kmem_zalloc(cnt * sizeof (void *), KM_SLEEP);
 329         fssbuf->fssb_size = cnt;
 330         fssbuf->fssb_list = fsslist;
 331         for (i = 0; i < cnt; i++)
 332                 fsslist[i] = kmem_zalloc(size, KM_SLEEP);
 333         return (fssbuf);
 334 }
 335 
 336 void
 337 fss_freebuf(fssbuf_t *fssbuf, int type)
 338 {
 339         void **fsslist;
 340         int i;
 341         size_t size;
 342 
 343         ASSERT(fssbuf != NULL);
 344         ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
 345         fsslist = fssbuf->fssb_list;
 346 
 347         switch (type) {
 348         case FSS_ALLOC_PROJ:
 349                 size = sizeof (fssproj_t);
 350                 break;
 351         case FSS_ALLOC_ZONE:
 352                 size = sizeof (fsszone_t);
 353                 break;
 354         }
 355 
 356         for (i = 0; i < fssbuf->fssb_size; i++) {
 357                 if (fsslist[i] != NULL)
 358                         kmem_free(fsslist[i], size);
 359         }
 360         kmem_free(fsslist, sizeof (void *) * fssbuf->fssb_size);
 361         kmem_free(fssbuf, sizeof (fssbuf_t));
 362 }
 363 
 364 static fsspset_t *
 365 fss_find_fsspset(cpupart_t *cpupart)
 366 {
 367         int i;
 368         fsspset_t *fsspset = NULL;
 369         int found = 0;
 370 
 371         ASSERT(cpupart != NULL);
 372         ASSERT(MUTEX_HELD(&fsspsets_lock));
 373 
 374         /*
 375          * Search for the cpupart pointer in the array of fsspsets.
 376          */
 377         for (i = 0; i < max_ncpus; i++) {
 378                 fsspset = &fsspsets[i];
 379                 if (fsspset->fssps_cpupart == cpupart) {
 380                         ASSERT(fsspset->fssps_nproj > 0);
 381                         found = 1;
 382                         break;
 383                 }
 384         }
 385         if (found == 0) {
 386                 /*
 387                  * If we didn't find anything, then use the first
 388                  * available slot in the fsspsets array.
 389                  */
 390                 for (i = 0; i < max_ncpus; i++) {
 391                         fsspset = &fsspsets[i];
 392                         if (fsspset->fssps_cpupart == NULL) {
 393                                 ASSERT(fsspset->fssps_nproj == 0);
 394                                 found = 1;
 395                                 break;
 396                         }
 397                 }
 398                 fsspset->fssps_cpupart = cpupart;
 399         }
 400         ASSERT(found == 1);
 401         return (fsspset);
 402 }
 403 
 404 static void
 405 fss_del_fsspset(fsspset_t *fsspset)
 406 {
 407         ASSERT(MUTEX_HELD(&fsspsets_lock));
 408         ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
 409         ASSERT(fsspset->fssps_nproj == 0);
 410         ASSERT(fsspset->fssps_list == NULL);
 411         ASSERT(fsspset->fssps_zones == NULL);
 412         fsspset->fssps_cpupart = NULL;
 413         fsspset->fssps_maxfsspri = 0;
 414         fsspset->fssps_shares = 0;
 415 }
 416 
 417 /*
 418  * The following routine returns a pointer to the fsszone structure which
 419  * belongs to zone "zone" and cpu partition fsspset, if such structure exists.
 420  */
 421 static fsszone_t *
 422 fss_find_fsszone(fsspset_t *fsspset, zone_t *zone)
 423 {
 424         fsszone_t *fsszone;
 425 
 426         ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
 427 
 428         if (fsspset->fssps_list != NULL) {
 429                 /*
 430                  * There are projects/zones active on this cpu partition
 431                  * already.  Try to find our zone among them.
 432                  */
 433                 fsszone = fsspset->fssps_zones;
 434                 do {
 435                         if (fsszone->fssz_zone == zone) {
 436                                 return (fsszone);
 437                         }
 438                         fsszone = fsszone->fssz_next;
 439                 } while (fsszone != fsspset->fssps_zones);
 440         }
 441         return (NULL);
 442 }
 443 
 444 /*
 445  * The following routine links new fsszone structure into doubly linked list of
 446  * zones active on the specified cpu partition.
 447  */
 448 static void
 449 fss_insert_fsszone(fsspset_t *fsspset, zone_t *zone, fsszone_t *fsszone)
 450 {
 451         ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
 452 
 453         fsszone->fssz_zone = zone;
 454         fsszone->fssz_rshares = zone->zone_shares;
 455 
 456         if (fsspset->fssps_zones == NULL) {
 457                 /*
 458                  * This will be the first fsszone for this fsspset
 459                  */
 460                 fsszone->fssz_next = fsszone->fssz_prev = fsszone;
 461                 fsspset->fssps_zones = fsszone;
 462         } else {
 463                 /*
 464                  * Insert this fsszone to the doubly linked list.
 465                  */
 466                 fsszone_t *fssz_head = fsspset->fssps_zones;
 467 
 468                 fsszone->fssz_next = fssz_head;
 469                 fsszone->fssz_prev = fssz_head->fssz_prev;
 470                 fssz_head->fssz_prev->fssz_next = fsszone;
 471                 fssz_head->fssz_prev = fsszone;
 472                 fsspset->fssps_zones = fsszone;
 473         }
 474 }
 475 
 476 /*
 477  * The following routine removes a single fsszone structure from the doubly
 478  * linked list of zones active on the specified cpu partition.  Note that
 479  * global fsspsets_lock must be held in case this fsszone structure is the last
 480  * on the above mentioned list.  Also note that the fsszone structure is not
 481  * freed here, it is the responsibility of the caller to call kmem_free for it.
 482  */
 483 static void
 484 fss_remove_fsszone(fsspset_t *fsspset, fsszone_t *fsszone)
 485 {
 486         ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
 487         ASSERT(fsszone->fssz_nproj == 0);
 488         ASSERT(fsszone->fssz_shares == 0);
 489         ASSERT(fsszone->fssz_runnable == 0);
 490 
 491         if (fsszone->fssz_next != fsszone) {
 492                 /*
 493                  * This is not the last zone in the list.
 494                  */
 495                 fsszone->fssz_prev->fssz_next = fsszone->fssz_next;
 496                 fsszone->fssz_next->fssz_prev = fsszone->fssz_prev;
 497                 if (fsspset->fssps_zones == fsszone)
 498                         fsspset->fssps_zones = fsszone->fssz_next;
 499         } else {
 500                 /*
 501                  * This was the last zone active in this cpu partition.
 502                  */
 503                 fsspset->fssps_zones = NULL;
 504         }
 505 }
 506 
 507 /*
 508  * The following routine returns a pointer to the fssproj structure
 509  * which belongs to project kpj and cpu partition fsspset, if such structure
 510  * exists.
 511  */
 512 static fssproj_t *
 513 fss_find_fssproj(fsspset_t *fsspset, kproject_t *kpj)
 514 {
 515         fssproj_t *fssproj;
 516 
 517         ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
 518 
 519         if (fsspset->fssps_list != NULL) {
 520                 /*
 521                  * There are projects running on this cpu partition already.
 522                  * Try to find our project among them.
 523                  */
 524                 fssproj = fsspset->fssps_list;
 525                 do {
 526                         if (fssproj->fssp_proj == kpj) {
 527                                 ASSERT(fssproj->fssp_pset == fsspset);
 528                                 return (fssproj);
 529                         }
 530                         fssproj = fssproj->fssp_next;
 531                 } while (fssproj != fsspset->fssps_list);
 532         }
 533         return (NULL);
 534 }
 535 
 536 /*
 537  * The following routine links new fssproj structure into doubly linked list
 538  * of projects running on the specified cpu partition.
 539  */
 540 static void
 541 fss_insert_fssproj(fsspset_t *fsspset, kproject_t *kpj, fsszone_t *fsszone,
 542     fssproj_t *fssproj)
 543 {
 544         ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
 545 
 546         fssproj->fssp_pset = fsspset;
 547         fssproj->fssp_proj = kpj;
 548         fssproj->fssp_shares = kpj->kpj_shares;
 549 
 550         fsspset->fssps_nproj++;
 551 
 552         if (fsspset->fssps_list == NULL) {
 553                 /*
 554                  * This will be the first fssproj for this fsspset
 555                  */
 556                 fssproj->fssp_next = fssproj->fssp_prev = fssproj;
 557                 fsspset->fssps_list = fssproj;
 558         } else {
 559                 /*
 560                  * Insert this fssproj to the doubly linked list.
 561                  */
 562                 fssproj_t *fssp_head = fsspset->fssps_list;
 563 
 564                 fssproj->fssp_next = fssp_head;
 565                 fssproj->fssp_prev = fssp_head->fssp_prev;
 566                 fssp_head->fssp_prev->fssp_next = fssproj;
 567                 fssp_head->fssp_prev = fssproj;
 568                 fsspset->fssps_list = fssproj;
 569         }
 570         fssproj->fssp_fsszone = fsszone;
 571         fsszone->fssz_nproj++;
 572         ASSERT(fsszone->fssz_nproj != 0);
 573 }
 574 
 575 /*
 576  * The following routine removes a single fssproj structure from the doubly
 577  * linked list of projects running on the specified cpu partition.  Note that
 578  * global fsspsets_lock must be held in case if this fssproj structure is the
 579  * last on the above mentioned list.  Also note that the fssproj structure is
 580  * not freed here, it is the responsibility of the caller to call kmem_free
 581  * for it.
 582  */
 583 static void
 584 fss_remove_fssproj(fsspset_t *fsspset, fssproj_t *fssproj)
 585 {
 586         fsszone_t *fsszone;
 587 
 588         ASSERT(MUTEX_HELD(&fsspsets_lock));
 589         ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
 590         ASSERT(fssproj->fssp_runnable == 0);
 591 
 592         fsspset->fssps_nproj--;
 593 
 594         fsszone = fssproj->fssp_fsszone;
 595         fsszone->fssz_nproj--;
 596 
 597         if (fssproj->fssp_next != fssproj) {
 598                 /*
 599                  * This is not the last part in the list.
 600                  */
 601                 fssproj->fssp_prev->fssp_next = fssproj->fssp_next;
 602                 fssproj->fssp_next->fssp_prev = fssproj->fssp_prev;
 603                 if (fsspset->fssps_list == fssproj)
 604                         fsspset->fssps_list = fssproj->fssp_next;
 605                 if (fsszone->fssz_nproj == 0)
 606                         fss_remove_fsszone(fsspset, fsszone);
 607         } else {
 608                 /*
 609                  * This was the last project part running
 610                  * at this cpu partition.
 611                  */
 612                 fsspset->fssps_list = NULL;
 613                 ASSERT(fsspset->fssps_nproj == 0);
 614                 ASSERT(fsszone->fssz_nproj == 0);
 615                 fss_remove_fsszone(fsspset, fsszone);
 616                 fss_del_fsspset(fsspset);
 617         }
 618 }
 619 
 620 static void
 621 fss_inactive(kthread_t *t)
 622 {
 623         fssproc_t *fssproc;
 624         fssproj_t *fssproj;
 625         fsspset_t *fsspset;
 626         fsszone_t *fsszone;
 627 
 628         ASSERT(THREAD_LOCK_HELD(t));
 629         fssproc = FSSPROC(t);
 630         fssproj = FSSPROC2FSSPROJ(fssproc);
 631         if (fssproj == NULL)    /* if this thread already exited */
 632                 return;
 633         fsspset = FSSPROJ2FSSPSET(fssproj);
 634         fsszone = fssproj->fssp_fsszone;
 635         disp_lock_enter_high(&fsspset->fssps_displock);
 636         ASSERT(fssproj->fssp_runnable > 0);
 637         if (--fssproj->fssp_runnable == 0) {
 638                 fsszone->fssz_shares -= fssproj->fssp_shares;
 639                 if (--fsszone->fssz_runnable == 0)
 640                         fsspset->fssps_shares -= fsszone->fssz_rshares;
 641         }
 642         ASSERT(fssproc->fss_runnable == 1);
 643         fssproc->fss_runnable = 0;
 644         disp_lock_exit_high(&fsspset->fssps_displock);
 645 }
 646 
 647 static void
 648 fss_active(kthread_t *t)
 649 {
 650         fssproc_t *fssproc;
 651         fssproj_t *fssproj;
 652         fsspset_t *fsspset;
 653         fsszone_t *fsszone;
 654 
 655         ASSERT(THREAD_LOCK_HELD(t));
 656         fssproc = FSSPROC(t);
 657         fssproj = FSSPROC2FSSPROJ(fssproc);
 658         if (fssproj == NULL)    /* if this thread already exited */
 659                 return;
 660         fsspset = FSSPROJ2FSSPSET(fssproj);
 661         fsszone = fssproj->fssp_fsszone;
 662         disp_lock_enter_high(&fsspset->fssps_displock);
 663         if (++fssproj->fssp_runnable == 1) {
 664                 fsszone->fssz_shares += fssproj->fssp_shares;
 665                 if (++fsszone->fssz_runnable == 1)
 666                         fsspset->fssps_shares += fsszone->fssz_rshares;
 667         }
 668         ASSERT(fssproc->fss_runnable == 0);
 669         fssproc->fss_runnable = 1;
 670         disp_lock_exit_high(&fsspset->fssps_displock);
 671 }
 672 
 673 /*
 674  * Fair share scheduler initialization. Called by dispinit() at boot time.
 675  * We can ignore clparmsz argument since we know that the smallest possible
 676  * parameter buffer is big enough for us.
 677  */
 678 /*ARGSUSED*/
 679 static pri_t
 680 fss_init(id_t cid, int clparmsz, classfuncs_t **clfuncspp)
 681 {
 682         int i;
 683 
 684         ASSERT(MUTEX_HELD(&cpu_lock));
 685 
 686         fss_cid = cid;
 687         fss_maxumdpri = minclsyspri - 1;
 688         fss_maxglobpri = minclsyspri;
 689         fss_minglobpri = 0;
 690         fsspsets = kmem_zalloc(sizeof (fsspset_t) * max_ncpus, KM_SLEEP);
 691 
 692         /*
 693          * Initialize the fssproc hash table.
 694          */
 695         for (i = 0; i < FSS_LISTS; i++)
 696                 fss_listhead[i].fss_next = fss_listhead[i].fss_prev =
 697                     &fss_listhead[i];
 698 
 699         *clfuncspp = &fss_classfuncs;
 700 
 701         /*
 702          * Fill in fss_nice_tick and fss_nice_decay arrays:
 703          * The cost of a tick is lower at positive nice values (so that it
 704          * will not increase its project's usage as much as normal) with 50%
 705          * drop at the maximum level and 50% increase at the minimum level.
 706          * The fsspri decay is slower at positive nice values.  fsspri values
 707          * of processes with negative nice levels must decay faster to receive
 708          * time slices more frequently than normal.
 709          */
 710         for (i = 0; i < FSS_NICE_RANGE; i++) {
 711                 fss_nice_tick[i] = (FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2)
 712                     - i)) / FSS_NICE_RANGE;
 713                 fss_nice_decay[i] = FSS_DECAY_MIN +
 714                     ((FSS_DECAY_MAX - FSS_DECAY_MIN) * i) /
 715                     (FSS_NICE_RANGE - 1);
 716         }
 717 
 718         return (fss_maxglobpri);
 719 }
 720 
 721 /*
 722  * Calculate the new cpupri based on the usage, the number of shares and
 723  * the number of active threads.  Reset the tick counter for this thread.
 724  */
 725 static void
 726 fss_newpri(fssproc_t *fssproc)
 727 {
 728         kthread_t *tp;
 729         fssproj_t *fssproj;
 730         fsspset_t *fsspset;
 731         fsszone_t *fsszone;
 732         fsspri_t fsspri, maxfsspri;
 733         pri_t invpri;
 734         uint32_t ticks;
 735 
 736         tp = fssproc->fss_tp;
 737         ASSERT(tp != NULL);
 738 
 739         if (tp->t_cid != fss_cid)
 740                 return;
 741 
 742         ASSERT(THREAD_LOCK_HELD(tp));
 743 
 744         fssproj = FSSPROC2FSSPROJ(fssproc);
 745         fsszone = FSSPROJ2FSSZONE(fssproj);
 746         if (fssproj == NULL)
 747                 /*
 748                  * No need to change priority of exited threads.
 749                  */
 750                 return;
 751 
 752         fsspset = FSSPROJ2FSSPSET(fssproj);
 753         disp_lock_enter_high(&fsspset->fssps_displock);
 754 
 755         if (fssproj->fssp_shares == 0 || fsszone->fssz_rshares == 0) {
 756                 /*
 757                  * Special case: threads with no shares.
 758                  */
 759                 fssproc->fss_umdpri = fss_minglobpri;
 760                 fssproc->fss_ticks = 0;
 761                 disp_lock_exit_high(&fsspset->fssps_displock);
 762                 return;
 763         }
 764 
 765         /*
 766          * fsspri += shusage * nrunnable * ticks
 767          */
 768         ticks = fssproc->fss_ticks;
 769         fssproc->fss_ticks = 0;
 770         fsspri = fssproc->fss_fsspri;
 771         fsspri += fssproj->fssp_shusage * fssproj->fssp_runnable * ticks;
 772         fssproc->fss_fsspri = fsspri;
 773 
 774         if (fsspri < fss_maxumdpri)
 775                 fsspri = fss_maxumdpri; /* so that maxfsspri is != 0 */
 776 
 777         /*
 778          * The general priority formula:
 779          *
 780          *                      (fsspri * umdprirange)
 781          *   pri = maxumdpri - ------------------------
 782          *                              maxfsspri
 783          *
 784          * If this thread's fsspri is greater than the previous largest
 785          * fsspri, then record it as the new high and priority for this
 786          * thread will be one (the lowest priority assigned to a thread
 787          * that has non-zero shares).
 788          * Note that this formula cannot produce out of bounds priority
 789          * values; if it is changed, additional checks may need  to  be
 790          * added.
 791          */
 792         maxfsspri = fsspset->fssps_maxfsspri;
 793         if (fsspri >= maxfsspri) {
 794                 fsspset->fssps_maxfsspri = fsspri;
 795                 disp_lock_exit_high(&fsspset->fssps_displock);
 796                 fssproc->fss_umdpri = 1;
 797         } else {
 798                 disp_lock_exit_high(&fsspset->fssps_displock);
 799                 invpri = (fsspri * (fss_maxumdpri - 1)) / maxfsspri;
 800                 fssproc->fss_umdpri = fss_maxumdpri - invpri;
 801         }
 802 }
 803 
 804 /*
 805  * Decays usages of all running projects and resets their tick counters.
 806  * Called once per second from fss_update() after updating priorities.
 807  */
 808 static void
 809 fss_decay_usage()
 810 {
 811         uint32_t zone_ext_shares, zone_int_shares;
 812         uint32_t kpj_shares, pset_shares;
 813         fsspset_t *fsspset;
 814         fssproj_t *fssproj;
 815         fsszone_t *fsszone;
 816         fsspri_t maxfsspri;
 817         int psetid;
 818 
 819         mutex_enter(&fsspsets_lock);
 820         /*
 821          * Go through all active processor sets and decay usages of projects
 822          * running on them.
 823          */
 824         for (psetid = 0; psetid < max_ncpus; psetid++) {
 825                 fsspset = &fsspsets[psetid];
 826                 mutex_enter(&fsspset->fssps_lock);
 827 
 828                 if (fsspset->fssps_cpupart == NULL ||
 829                     (fssproj = fsspset->fssps_list) == NULL) {
 830                         mutex_exit(&fsspset->fssps_lock);
 831                         continue;
 832                 }
 833 
 834                 /*
 835                  * Decay maxfsspri for this cpu partition with the
 836                  * fastest possible decay rate.
 837                  */
 838                 disp_lock_enter(&fsspset->fssps_displock);
 839 
 840                 maxfsspri = (fsspset->fssps_maxfsspri *
 841                     fss_nice_decay[NZERO]) / FSS_DECAY_BASE;
 842                 if (maxfsspri < fss_maxumdpri)
 843                         maxfsspri = fss_maxumdpri;
 844                 fsspset->fssps_maxfsspri = maxfsspri;
 845 
 846                 do {
 847                         /*
 848                          * Decay usage for each project running on
 849                          * this cpu partition.
 850                          */
 851                         fssproj->fssp_usage =
 852                             (fssproj->fssp_usage * FSS_DECAY_USG) /
 853                             FSS_DECAY_BASE + fssproj->fssp_ticks;
 854                         fssproj->fssp_ticks = 0;
 855 
 856                         fsszone = fssproj->fssp_fsszone;
 857                         /*
 858                          * Readjust the project's number of shares if it has
 859                          * changed since we checked it last time.
 860                          */
 861                         kpj_shares = fssproj->fssp_proj->kpj_shares;
 862                         if (fssproj->fssp_shares != kpj_shares) {
 863                                 if (fssproj->fssp_runnable != 0) {
 864                                         fsszone->fssz_shares -=
 865                                             fssproj->fssp_shares;
 866                                         fsszone->fssz_shares += kpj_shares;
 867                                 }
 868                                 fssproj->fssp_shares = kpj_shares;
 869                         }
 870 
 871                         /*
 872                          * Readjust the zone's number of shares if it
 873                          * has changed since we checked it last time.
 874                          */
 875                         zone_ext_shares = fsszone->fssz_zone->zone_shares;
 876                         if (fsszone->fssz_rshares != zone_ext_shares) {
 877                                 if (fsszone->fssz_runnable != 0) {
 878                                         fsspset->fssps_shares -=
 879                                             fsszone->fssz_rshares;
 880                                         fsspset->fssps_shares +=
 881                                             zone_ext_shares;
 882                                 }
 883                                 fsszone->fssz_rshares = zone_ext_shares;
 884                         }
 885                         zone_int_shares = fsszone->fssz_shares;
 886                         pset_shares = fsspset->fssps_shares;
 887                         /*
 888                          * Calculate fssp_shusage value to be used
 889                          * for fsspri increments for the next second.
 890                          */
 891                         if (kpj_shares == 0 || zone_ext_shares == 0) {
 892                                 fssproj->fssp_shusage = 0;
 893                         } else if (FSSPROJ2KPROJ(fssproj) == proj0p) {
 894                                 /*
 895                                  * Project 0 in the global zone has 50%
 896                                  * of its zone.
 897                                  */
 898                                 fssproj->fssp_shusage = (fssproj->fssp_usage *
 899                                     zone_int_shares * zone_int_shares) /
 900                                     (zone_ext_shares * zone_ext_shares);
 901                         } else {
 902                                 /*
 903                                  * Thread's priority is based on its project's
 904                                  * normalized usage (shusage) value which gets
 905                                  * calculated this way:
 906                                  *
 907                                  *         pset_shares^2    zone_int_shares^2
 908                                  * usage * ------------- * ------------------
 909                                  *         kpj_shares^2     zone_ext_shares^2
 910                                  *
 911                                  * Where zone_int_shares is the sum of shares
 912                                  * of all active projects within the zone (and
 913                                  * the pset), and zone_ext_shares is the number
 914                                  * of zone shares (ie, zone.cpu-shares).
 915                                  *
 916                                  * If there is only one zone active on the pset
 917                                  * the above reduces to:
 918                                  *
 919                                  *                      zone_int_shares^2
 920                                  * shusage = usage * ---------------------
 921                                  *                      kpj_shares^2
 922                                  *
 923                                  * If there's only one project active in the
 924                                  * zone this formula reduces to:
 925                                  *
 926                                  *                      pset_shares^2
 927                                  * shusage = usage * ----------------------
 928                                  *                      zone_ext_shares^2
 929                                  */
 930                                 fssproj->fssp_shusage = fssproj->fssp_usage *
 931                                     pset_shares * zone_int_shares;
 932                                 fssproj->fssp_shusage /=
 933                                     kpj_shares * zone_ext_shares;
 934                                 fssproj->fssp_shusage *=
 935                                     pset_shares * zone_int_shares;
 936                                 fssproj->fssp_shusage /=
 937                                     kpj_shares * zone_ext_shares;
 938                         }
 939                         fssproj = fssproj->fssp_next;
 940                 } while (fssproj != fsspset->fssps_list);
 941 
 942                 disp_lock_exit(&fsspset->fssps_displock);
 943                 mutex_exit(&fsspset->fssps_lock);
 944         }
 945         mutex_exit(&fsspsets_lock);
 946 }
 947 
 948 static void
 949 fss_change_priority(kthread_t *t, fssproc_t *fssproc)
 950 {
 951         pri_t new_pri;
 952 
 953         ASSERT(THREAD_LOCK_HELD(t));
 954         new_pri = fssproc->fss_umdpri;
 955         ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
 956 
 957         t->t_cpri = fssproc->fss_upri;
 958         fssproc->fss_flags &= ~FSSRESTORE;
 959         if (t == curthread || t->t_state == TS_ONPROC) {
 960                 /*
 961                  * curthread is always onproc
 962                  */
 963                 cpu_t *cp = t->t_disp_queue->disp_cpu;
 964                 THREAD_CHANGE_PRI(t, new_pri);
 965                 if (t == cp->cpu_dispthread)
 966                         cp->cpu_dispatch_pri = DISP_PRIO(t);
 967                 if (DISP_MUST_SURRENDER(t)) {
 968                         fssproc->fss_flags |= FSSBACKQ;
 969                         cpu_surrender(t);
 970                 } else {
 971                         fssproc->fss_timeleft = fss_quantum;
 972                 }
 973         } else {
 974                 /*
 975                  * When the priority of a thread is changed, it may be
 976                  * necessary to adjust its position on a sleep queue or
 977                  * dispatch queue.  The function thread_change_pri accomplishes
 978                  * this.
 979                  */
 980                 if (thread_change_pri(t, new_pri, 0)) {
 981                         /*
 982                          * The thread was on a run queue.
 983                          */
 984                         fssproc->fss_timeleft = fss_quantum;
 985                 } else {
 986                         fssproc->fss_flags |= FSSBACKQ;
 987                 }
 988         }
 989 }
 990 
 991 /*
 992  * Update priorities of all fair-sharing threads that are currently runnable
 993  * at a user mode priority based on the number of shares and current usage.
 994  * Called once per second via timeout which we reset here.
 995  *
 996  * There are several lists of fair-sharing threads broken up by a hash on the
 997  * thread pointer.  Each list has its own lock.  This avoids blocking all
 998  * fss_enterclass, fss_fork, and fss_exitclass operations while fss_update runs.
 999  * fss_update traverses each list in turn.
1000  */
1001 static void
1002 fss_update(void *arg)
1003 {
1004         int i;
1005         int new_marker = -1;
1006         static int fss_update_marker;
1007 
1008         /*
1009          * Decay and update usages for all projects.
1010          */
1011         fss_decay_usage();
1012 
1013         /*
1014          * Start with the fss_update_marker list, then do the rest.
1015          */
1016         i = fss_update_marker;
1017 
1018         /*
1019          * Go around all threads, set new priorities and decay
1020          * per-thread CPU usages.
1021          */
1022         do {
1023                 /*
1024                  * If this is the first list after the current marker to have
1025                  * threads with priorities updates, advance the marker to this
1026                  * list for the next time fss_update runs.
1027                  */
1028                 if (fss_update_list(i) &&
1029                     new_marker == -1 && i != fss_update_marker)
1030                         new_marker = i;
1031         } while ((i = FSS_LIST_NEXT(i)) != fss_update_marker);
1032 
1033         /*
1034          * Advance marker for the next fss_update call
1035          */
1036         if (new_marker != -1)
1037                 fss_update_marker = new_marker;
1038 
1039         (void) timeout(fss_update, arg, hz);
1040 }
1041 
1042 /*
1043  * Updates priority for a list of threads.  Returns 1 if the priority of one
1044  * of the threads was actually updated, 0 if none were for various reasons
1045  * (thread is no longer in the FSS class, is not runnable, has the preemption
1046  * control no-preempt bit set, etc.)
1047  */
1048 static int
1049 fss_update_list(int i)
1050 {
1051         fssproc_t *fssproc;
1052         fssproj_t *fssproj;
1053         fsspri_t fsspri;
1054         kthread_t *t;
1055         int updated = 0;
1056 
1057         mutex_enter(&fss_listlock[i]);
1058         for (fssproc = fss_listhead[i].fss_next; fssproc != &fss_listhead[i];
1059             fssproc = fssproc->fss_next) {
1060                 t = fssproc->fss_tp;
1061                 /*
1062                  * Lock the thread and verify the state.
1063                  */
1064                 thread_lock(t);
1065                 /*
1066                  * Skip the thread if it is no longer in the FSS class or
1067                  * is running with kernel mode priority.
1068                  */
1069                 if (t->t_cid != fss_cid)
1070                         goto next;
1071                 if ((fssproc->fss_flags & FSSKPRI) != 0)
1072                         goto next;
1073 
1074                 fssproj = FSSPROC2FSSPROJ(fssproc);
1075                 if (fssproj == NULL)
1076                         goto next;
1077                 if (fssproj->fssp_shares != 0) {
1078                         /*
1079                          * Decay fsspri value.
1080                          */
1081                         fsspri = fssproc->fss_fsspri;
1082                         fsspri = (fsspri * fss_nice_decay[fssproc->fss_nice]) /
1083                             FSS_DECAY_BASE;
1084                         fssproc->fss_fsspri = fsspri;
1085                 }
1086 
1087                 if (t->t_schedctl && schedctl_get_nopreempt(t))
1088                         goto next;
1089                 if (t->t_state != TS_RUN && t->t_state != TS_WAIT) {
1090                         /*
1091                          * Make next syscall/trap call fss_trapret
1092                          */
1093                         t->t_trapret = 1;
1094                         aston(t);
1095                         goto next;
1096                 }
1097                 fss_newpri(fssproc);
1098                 updated = 1;
1099 
1100                 /*
1101                  * Only dequeue the thread if it needs to be moved; otherwise
1102                  * it should just round-robin here.
1103                  */
1104                 if (t->t_pri != fssproc->fss_umdpri)
1105                         fss_change_priority(t, fssproc);
1106 next:
1107                 thread_unlock(t);
1108         }
1109         mutex_exit(&fss_listlock[i]);
1110         return (updated);
1111 }
1112 
1113 /*ARGSUSED*/
1114 static int
1115 fss_admin(caddr_t uaddr, cred_t *reqpcredp)
1116 {
1117         fssadmin_t fssadmin;
1118 
1119         if (copyin(uaddr, &fssadmin, sizeof (fssadmin_t)))
1120                 return (EFAULT);
1121 
1122         switch (fssadmin.fss_cmd) {
1123         case FSS_SETADMIN:
1124                 if (secpolicy_dispadm(reqpcredp) != 0)
1125                         return (EPERM);
1126                 if (fssadmin.fss_quantum <= 0 || fssadmin.fss_quantum >= hz)
1127                         return (EINVAL);
1128                 fss_quantum = fssadmin.fss_quantum;
1129                 break;
1130         case FSS_GETADMIN:
1131                 fssadmin.fss_quantum = fss_quantum;
1132                 if (copyout(&fssadmin, uaddr, sizeof (fssadmin_t)))
1133                         return (EFAULT);
1134                 break;
1135         default:
1136                 return (EINVAL);
1137         }
1138         return (0);
1139 }
1140 
1141 static int
1142 fss_getclinfo(void *infop)
1143 {
1144         fssinfo_t *fssinfo = (fssinfo_t *)infop;
1145         fssinfo->fss_maxupri = fss_maxupri;
1146         return (0);
1147 }
1148 
1149 static int
1150 fss_parmsin(void *parmsp)
1151 {
1152         fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1153 
1154         /*
1155          * Check validity of parameters.
1156          */
1157         if ((fssparmsp->fss_uprilim > fss_maxupri ||
1158             fssparmsp->fss_uprilim < -fss_maxupri) &&
1159             fssparmsp->fss_uprilim != FSS_NOCHANGE)
1160                 return (EINVAL);
1161 
1162         if ((fssparmsp->fss_upri > fss_maxupri ||
1163             fssparmsp->fss_upri < -fss_maxupri) &&
1164             fssparmsp->fss_upri != FSS_NOCHANGE)
1165                 return (EINVAL);
1166 
1167         return (0);
1168 }
1169 
1170 /*ARGSUSED*/
1171 static int
1172 fss_parmsout(void *parmsp, pc_vaparms_t *vaparmsp)
1173 {
1174         return (0);
1175 }
1176 
1177 static int
1178 fss_vaparmsin(void *parmsp, pc_vaparms_t *vaparmsp)
1179 {
1180         fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1181         int priflag = 0;
1182         int limflag = 0;
1183         uint_t cnt;
1184         pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
1185 
1186         /*
1187          * FSS_NOCHANGE (-32768) is outside of the range of values for
1188          * fss_uprilim and fss_upri.  If the structure fssparms_t is changed,
1189          * FSS_NOCHANGE should be replaced by a flag word.
1190          */
1191         fssparmsp->fss_uprilim = FSS_NOCHANGE;
1192         fssparmsp->fss_upri = FSS_NOCHANGE;
1193 
1194         /*
1195          * Get the varargs parameter and check validity of parameters.
1196          */
1197         if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
1198                 return (EINVAL);
1199 
1200         for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
1201                 switch (vpp->pc_key) {
1202                 case FSS_KY_UPRILIM:
1203                         if (limflag++)
1204                                 return (EINVAL);
1205                         fssparmsp->fss_uprilim = (pri_t)vpp->pc_parm;
1206                         if (fssparmsp->fss_uprilim > fss_maxupri ||
1207                             fssparmsp->fss_uprilim < -fss_maxupri)
1208                                 return (EINVAL);
1209                         break;
1210                 case FSS_KY_UPRI:
1211                         if (priflag++)
1212                                 return (EINVAL);
1213                         fssparmsp->fss_upri = (pri_t)vpp->pc_parm;
1214                         if (fssparmsp->fss_upri > fss_maxupri ||
1215                             fssparmsp->fss_upri < -fss_maxupri)
1216                                 return (EINVAL);
1217                         break;
1218                 default:
1219                         return (EINVAL);
1220                 }
1221         }
1222 
1223         if (vaparmsp->pc_vaparmscnt == 0) {
1224                 /*
1225                  * Use default parameters.
1226                  */
1227                 fssparmsp->fss_upri = fssparmsp->fss_uprilim = 0;
1228         }
1229 
1230         return (0);
1231 }
1232 
1233 /*
1234  * Copy all selected fair-sharing class parameters to the user.  The parameters
1235  * are specified by a key.
1236  */
1237 static int
1238 fss_vaparmsout(void *parmsp, pc_vaparms_t *vaparmsp)
1239 {
1240         fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1241         int priflag = 0;
1242         int limflag = 0;
1243         uint_t cnt;
1244         pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
1245 
1246         ASSERT(MUTEX_NOT_HELD(&curproc->p_lock));
1247 
1248         if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
1249                 return (EINVAL);
1250 
1251         for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
1252                 switch (vpp->pc_key) {
1253                 case FSS_KY_UPRILIM:
1254                         if (limflag++)
1255                                 return (EINVAL);
1256                         if (copyout(&fssparmsp->fss_uprilim,
1257                             (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
1258                                 return (EFAULT);
1259                         break;
1260                 case FSS_KY_UPRI:
1261                         if (priflag++)
1262                                 return (EINVAL);
1263                         if (copyout(&fssparmsp->fss_upri,
1264                             (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
1265                                 return (EFAULT);
1266                         break;
1267                 default:
1268                         return (EINVAL);
1269                 }
1270         }
1271 
1272         return (0);
1273 }
1274 
1275 /*
1276  * Return the user mode scheduling priority range.
1277  */
1278 static int
1279 fss_getclpri(pcpri_t *pcprip)
1280 {
1281         pcprip->pc_clpmax = fss_maxupri;
1282         pcprip->pc_clpmin = -fss_maxupri;
1283         return (0);
1284 }
1285 
1286 static int
1287 fss_alloc(void **p, int flag)
1288 {
1289         void *bufp;
1290 
1291         if ((bufp = kmem_zalloc(sizeof (fssproc_t), flag)) == NULL) {
1292                 return (ENOMEM);
1293         } else {
1294                 *p = bufp;
1295                 return (0);
1296         }
1297 }
1298 
1299 static void
1300 fss_free(void *bufp)
1301 {
1302         if (bufp)
1303                 kmem_free(bufp, sizeof (fssproc_t));
1304 }
1305 
1306 /*
1307  * Thread functions
1308  */
1309 static int
1310 fss_enterclass(kthread_t *t, id_t cid, void *parmsp, cred_t *reqpcredp,
1311     void *bufp)
1312 {
1313         fssparms_t      *fssparmsp = (fssparms_t *)parmsp;
1314         fssproc_t       *fssproc;
1315         pri_t           reqfssuprilim;
1316         pri_t           reqfssupri;
1317         static uint32_t fssexists = 0;
1318         fsspset_t       *fsspset;
1319         fssproj_t       *fssproj;
1320         fsszone_t       *fsszone;
1321         kproject_t      *kpj;
1322         zone_t          *zone;
1323         int             fsszone_allocated = 0;
1324 
1325         fssproc = (fssproc_t *)bufp;
1326         ASSERT(fssproc != NULL);
1327 
1328         ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1329 
1330         /*
1331          * Only root can move threads to FSS class.
1332          */
1333         if (reqpcredp != NULL && secpolicy_setpriority(reqpcredp) != 0)
1334                 return (EPERM);
1335         /*
1336          * Initialize the fssproc structure.
1337          */
1338         fssproc->fss_umdpri = fss_maxumdpri / 2;
1339 
1340         if (fssparmsp == NULL) {
1341                 /*
1342                  * Use default values.
1343                  */
1344                 fssproc->fss_nice = NZERO;
1345                 fssproc->fss_uprilim = fssproc->fss_upri = 0;
1346         } else {
1347                 /*
1348                  * Use supplied values.
1349                  */
1350                 if (fssparmsp->fss_uprilim == FSS_NOCHANGE) {
1351                         reqfssuprilim = 0;
1352                 } else {
1353                         if (fssparmsp->fss_uprilim > 0 &&
1354                             secpolicy_setpriority(reqpcredp) != 0)
1355                                 return (EPERM);
1356                         reqfssuprilim = fssparmsp->fss_uprilim;
1357                 }
1358                 if (fssparmsp->fss_upri == FSS_NOCHANGE) {
1359                         reqfssupri = reqfssuprilim;
1360                 } else {
1361                         if (fssparmsp->fss_upri > 0 &&
1362                             secpolicy_setpriority(reqpcredp) != 0)
1363                                 return (EPERM);
1364                         /*
1365                          * Set the user priority to the requested value or
1366                          * the upri limit, whichever is lower.
1367                          */
1368                         reqfssupri = fssparmsp->fss_upri;
1369                         if (reqfssupri > reqfssuprilim)
1370                                 reqfssupri = reqfssuprilim;
1371                 }
1372                 fssproc->fss_uprilim = reqfssuprilim;
1373                 fssproc->fss_upri = reqfssupri;
1374                 fssproc->fss_nice = NZERO - (NZERO * reqfssupri) / fss_maxupri;
1375                 if (fssproc->fss_nice > FSS_NICE_MAX)
1376                         fssproc->fss_nice = FSS_NICE_MAX;
1377         }
1378 
1379         fssproc->fss_timeleft = fss_quantum;
1380         fssproc->fss_tp = t;
1381         cpucaps_sc_init(&fssproc->fss_caps);
1382 
1383         /*
1384          * Put a lock on our fsspset structure.
1385          */
1386         mutex_enter(&fsspsets_lock);
1387         fsspset = fss_find_fsspset(t->t_cpupart);
1388         mutex_enter(&fsspset->fssps_lock);
1389         mutex_exit(&fsspsets_lock);
1390 
1391         zone = ttoproc(t)->p_zone;
1392         if ((fsszone = fss_find_fsszone(fsspset, zone)) == NULL) {
1393                 if ((fsszone = kmem_zalloc(sizeof (fsszone_t), KM_NOSLEEP))
1394                     == NULL) {
1395                         mutex_exit(&fsspset->fssps_lock);
1396                         return (ENOMEM);
1397                 } else {
1398                         fsszone_allocated = 1;
1399                         fss_insert_fsszone(fsspset, zone, fsszone);
1400                 }
1401         }
1402         kpj = ttoproj(t);
1403         if ((fssproj = fss_find_fssproj(fsspset, kpj)) == NULL) {
1404                 if ((fssproj = kmem_zalloc(sizeof (fssproj_t), KM_NOSLEEP))
1405                     == NULL) {
1406                         if (fsszone_allocated) {
1407                                 fss_remove_fsszone(fsspset, fsszone);
1408                                 kmem_free(fsszone, sizeof (fsszone_t));
1409                         }
1410                         mutex_exit(&fsspset->fssps_lock);
1411                         return (ENOMEM);
1412                 } else {
1413                         fss_insert_fssproj(fsspset, kpj, fsszone, fssproj);
1414                 }
1415         }
1416         fssproj->fssp_threads++;
1417         fssproc->fss_proj = fssproj;
1418 
1419         /*
1420          * Reset priority. Process goes to a "user mode" priority here
1421          * regardless of whether or not it has slept since entering the kernel.
1422          */
1423         thread_lock(t);
1424         t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
1425         t->t_cid = cid;
1426         t->t_cldata = (void *)fssproc;
1427         t->t_schedflag |= TS_RUNQMATCH;
1428         fss_change_priority(t, fssproc);
1429         if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
1430             t->t_state == TS_WAIT)
1431                 fss_active(t);
1432         thread_unlock(t);
1433 
1434         mutex_exit(&fsspset->fssps_lock);
1435 
1436         /*
1437          * Link new structure into fssproc list.
1438          */
1439         FSS_LIST_INSERT(fssproc);
1440 
1441         /*
1442          * If this is the first fair-sharing thread to occur since boot,
1443          * we set up the initial call to fss_update() here. Use an atomic
1444          * compare-and-swap since that's easier and faster than a mutex
1445          * (but check with an ordinary load first since most of the time
1446          * this will already be done).
1447          */
1448         if (fssexists == 0 && cas32(&fssexists, 0, 1) == 0)
1449                 (void) timeout(fss_update, NULL, hz);
1450 
1451         return (0);
1452 }
1453 
1454 /*
1455  * Remove fssproc_t from the list.
1456  */
1457 static void
1458 fss_exitclass(void *procp)
1459 {
1460         fssproc_t *fssproc = (fssproc_t *)procp;
1461         fssproj_t *fssproj;
1462         fsspset_t *fsspset;
1463         fsszone_t *fsszone;
1464         kthread_t *t = fssproc->fss_tp;
1465 
1466         /*
1467          * We should be either getting this thread off the deathrow or
1468          * this thread has already moved to another scheduling class and
1469          * we're being called with its old cldata buffer pointer.  In both
1470          * cases, the content of this buffer can not be changed while we're
1471          * here.
1472          */
1473         mutex_enter(&fsspsets_lock);
1474         thread_lock(t);
1475         if (t->t_cid != fss_cid) {
1476                 /*
1477                  * We're being called as a result of the priocntl() system
1478                  * call -- someone is trying to move our thread to another
1479                  * scheduling class. We can't call fss_inactive() here
1480                  * because our thread's t_cldata pointer already points
1481                  * to another scheduling class specific data.
1482                  */
1483                 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1484 
1485                 fssproj = FSSPROC2FSSPROJ(fssproc);
1486                 fsspset = FSSPROJ2FSSPSET(fssproj);
1487                 fsszone = fssproj->fssp_fsszone;
1488 
1489                 if (fssproc->fss_runnable) {
1490                         disp_lock_enter_high(&fsspset->fssps_displock);
1491                         if (--fssproj->fssp_runnable == 0) {
1492                                 fsszone->fssz_shares -= fssproj->fssp_shares;
1493                                 if (--fsszone->fssz_runnable == 0)
1494                                         fsspset->fssps_shares -=
1495                                             fsszone->fssz_rshares;
1496                         }
1497                         disp_lock_exit_high(&fsspset->fssps_displock);
1498                 }
1499                 thread_unlock(t);
1500 
1501                 mutex_enter(&fsspset->fssps_lock);
1502                 if (--fssproj->fssp_threads == 0) {
1503                         fss_remove_fssproj(fsspset, fssproj);
1504                         if (fsszone->fssz_nproj == 0)
1505                                 kmem_free(fsszone, sizeof (fsszone_t));
1506                         kmem_free(fssproj, sizeof (fssproj_t));
1507                 }
1508                 mutex_exit(&fsspset->fssps_lock);
1509 
1510         } else {
1511                 ASSERT(t->t_state == TS_FREE);
1512                 /*
1513                  * We're being called from thread_free() when our thread
1514                  * is removed from the deathrow. There is nothing we need
1515                  * do here since everything should've been done earlier
1516                  * in fss_exit().
1517                  */
1518                 thread_unlock(t);
1519         }
1520         mutex_exit(&fsspsets_lock);
1521 
1522         FSS_LIST_DELETE(fssproc);
1523         fss_free(fssproc);
1524 }
1525 
1526 /*ARGSUSED*/
1527 static int
1528 fss_canexit(kthread_t *t, cred_t *credp)
1529 {
1530         /*
1531          * A thread is allowed to exit FSS only if we have sufficient
1532          * privileges.
1533          */
1534         if (credp != NULL && secpolicy_setpriority(credp) != 0)
1535                 return (EPERM);
1536         else
1537                 return (0);
1538 }
1539 
1540 /*
1541  * Initialize fair-share class specific proc structure for a child.
1542  */
1543 static int
1544 fss_fork(kthread_t *pt, kthread_t *ct, void *bufp)
1545 {
1546         fssproc_t *pfssproc;    /* ptr to parent's fssproc structure    */
1547         fssproc_t *cfssproc;    /* ptr to child's fssproc structure     */
1548         fssproj_t *fssproj;
1549         fsspset_t *fsspset;
1550 
1551         ASSERT(MUTEX_HELD(&ttoproc(pt)->p_lock));
1552         ASSERT(ct->t_state == TS_STOPPED);
1553 
1554         cfssproc = (fssproc_t *)bufp;
1555         ASSERT(cfssproc != NULL);
1556         bzero(cfssproc, sizeof (fssproc_t));
1557 
1558         thread_lock(pt);
1559         pfssproc = FSSPROC(pt);
1560         fssproj = FSSPROC2FSSPROJ(pfssproc);
1561         fsspset = FSSPROJ2FSSPSET(fssproj);
1562         thread_unlock(pt);
1563 
1564         mutex_enter(&fsspset->fssps_lock);
1565         /*
1566          * Initialize child's fssproc structure.
1567          */
1568         thread_lock(pt);
1569         ASSERT(FSSPROJ(pt) == fssproj);
1570         cfssproc->fss_proj = fssproj;
1571         cfssproc->fss_timeleft = fss_quantum;
1572         cfssproc->fss_umdpri = pfssproc->fss_umdpri;
1573         cfssproc->fss_fsspri = 0;
1574         cfssproc->fss_uprilim = pfssproc->fss_uprilim;
1575         cfssproc->fss_upri = pfssproc->fss_upri;
1576         cfssproc->fss_tp = ct;
1577         cfssproc->fss_nice = pfssproc->fss_nice;
1578         cpucaps_sc_init(&cfssproc->fss_caps);
1579 
1580         cfssproc->fss_flags =
1581             pfssproc->fss_flags & ~(FSSKPRI | FSSBACKQ | FSSRESTORE);
1582         ct->t_cldata = (void *)cfssproc;
1583         ct->t_schedflag |= TS_RUNQMATCH;
1584         thread_unlock(pt);
1585 
1586         fssproj->fssp_threads++;
1587         mutex_exit(&fsspset->fssps_lock);
1588 
1589         /*
1590          * Link new structure into fssproc hash table.
1591          */
1592         FSS_LIST_INSERT(cfssproc);
1593         return (0);
1594 }
1595 
1596 /*
1597  * Child is placed at back of dispatcher queue and parent gives up processor
1598  * so that the child runs first after the fork. This allows the child
1599  * immediately execing to break the multiple use of copy on write pages with no
1600  * disk home. The parent will get to steal them back rather than uselessly
1601  * copying them.
1602  */
1603 static void
1604 fss_forkret(kthread_t *t, kthread_t *ct)
1605 {
1606         proc_t *pp = ttoproc(t);
1607         proc_t *cp = ttoproc(ct);
1608         fssproc_t *fssproc;
1609 
1610         ASSERT(t == curthread);
1611         ASSERT(MUTEX_HELD(&pidlock));
1612 
1613         /*
1614          * Grab the child's p_lock before dropping pidlock to ensure the
1615          * process does not disappear before we set it running.
1616          */
1617         mutex_enter(&cp->p_lock);
1618         continuelwps(cp);
1619         mutex_exit(&cp->p_lock);
1620 
1621         mutex_enter(&pp->p_lock);
1622         mutex_exit(&pidlock);
1623         continuelwps(pp);
1624 
1625         thread_lock(t);
1626 
1627         fssproc = FSSPROC(t);
1628         fss_newpri(fssproc);
1629         fssproc->fss_timeleft = fss_quantum;
1630         t->t_pri = fssproc->fss_umdpri;
1631         ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
1632         fssproc->fss_flags &= ~FSSKPRI;
1633         THREAD_TRANSITION(t);
1634 
1635         /*
1636          * We don't want to call fss_setrun(t) here because it may call
1637          * fss_active, which we don't need.
1638          */
1639         fssproc->fss_flags &= ~FSSBACKQ;
1640 
1641         if (t->t_disp_time != ddi_get_lbolt())
1642                 setbackdq(t);
1643         else
1644                 setfrontdq(t);
1645 
1646         thread_unlock(t);
1647         /*
1648          * Safe to drop p_lock now since it is safe to change
1649          * the scheduling class after this point.
1650          */
1651         mutex_exit(&pp->p_lock);
1652 
1653         swtch();
1654 }
1655 
1656 /*
1657  * Get the fair-sharing parameters of the thread pointed to by fssprocp into
1658  * the buffer pointed by fssparmsp.
1659  */
1660 static void
1661 fss_parmsget(kthread_t *t, void *parmsp)
1662 {
1663         fssproc_t *fssproc = FSSPROC(t);
1664         fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1665 
1666         fssparmsp->fss_uprilim = fssproc->fss_uprilim;
1667         fssparmsp->fss_upri = fssproc->fss_upri;
1668 }
1669 
1670 /*ARGSUSED*/
1671 static int
1672 fss_parmsset(kthread_t *t, void *parmsp, id_t reqpcid, cred_t *reqpcredp)
1673 {
1674         char            nice;
1675         pri_t           reqfssuprilim;
1676         pri_t           reqfssupri;
1677         fssproc_t       *fssproc = FSSPROC(t);
1678         fssparms_t      *fssparmsp = (fssparms_t *)parmsp;
1679 
1680         ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
1681 
1682         if (fssparmsp->fss_uprilim == FSS_NOCHANGE)
1683                 reqfssuprilim = fssproc->fss_uprilim;
1684         else
1685                 reqfssuprilim = fssparmsp->fss_uprilim;
1686 
1687         if (fssparmsp->fss_upri == FSS_NOCHANGE)
1688                 reqfssupri = fssproc->fss_upri;
1689         else
1690                 reqfssupri = fssparmsp->fss_upri;
1691 
1692         /*
1693          * Make sure the user priority doesn't exceed the upri limit.
1694          */
1695         if (reqfssupri > reqfssuprilim)
1696                 reqfssupri = reqfssuprilim;
1697 
1698         /*
1699          * Basic permissions enforced by generic kernel code for all classes
1700          * require that a thread attempting to change the scheduling parameters
1701          * of a target thread be privileged or have a real or effective UID
1702          * matching that of the target thread. We are not called unless these
1703          * basic permission checks have already passed. The fair-sharing class
1704          * requires in addition that the calling thread be privileged if it
1705          * is attempting to raise the upri limit above its current value.
1706          * This may have been checked previously but if our caller passed us
1707          * a non-NULL credential pointer we assume it hasn't and we check it
1708          * here.
1709          */
1710         if ((reqpcredp != NULL) &&
1711             (reqfssuprilim > fssproc->fss_uprilim) &&
1712             secpolicy_raisepriority(reqpcredp) != 0)
1713                 return (EPERM);
1714 
1715         /*
1716          * Set fss_nice to the nice value corresponding to the user priority we
1717          * are setting.  Note that setting the nice field of the parameter
1718          * struct won't affect upri or nice.
1719          */
1720         nice = NZERO - (reqfssupri * NZERO) / fss_maxupri;
1721         if (nice > FSS_NICE_MAX)
1722                 nice = FSS_NICE_MAX;
1723 
1724         thread_lock(t);
1725 
1726         fssproc->fss_uprilim = reqfssuprilim;
1727         fssproc->fss_upri = reqfssupri;
1728         fssproc->fss_nice = nice;
1729         fss_newpri(fssproc);
1730 
1731         if ((fssproc->fss_flags & FSSKPRI) != 0) {
1732                 thread_unlock(t);
1733                 return (0);
1734         }
1735 
1736         fss_change_priority(t, fssproc);
1737         thread_unlock(t);
1738         return (0);
1739 
1740 }
1741 
1742 /*
1743  * The thread is being stopped.
1744  */
1745 /*ARGSUSED*/
1746 static void
1747 fss_stop(kthread_t *t, int why, int what)
1748 {
1749         ASSERT(THREAD_LOCK_HELD(t));
1750         ASSERT(t == curthread);
1751 
1752         fss_inactive(t);
1753 }
1754 
1755 /*
1756  * The current thread is exiting, do necessary adjustments to its project
1757  */
1758 static void
1759 fss_exit(kthread_t *t)
1760 {
1761         fsspset_t *fsspset;
1762         fssproj_t *fssproj;
1763         fssproc_t *fssproc;
1764         fsszone_t *fsszone;
1765         int free = 0;
1766 
1767         /*
1768          * Thread t here is either a current thread (in which case we hold
1769          * its process' p_lock), or a thread being destroyed by forklwp_fail(),
1770          * in which case we hold pidlock and thread is no longer on the
1771          * thread list.
1772          */
1773         ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock) || MUTEX_HELD(&pidlock));
1774 
1775         fssproc = FSSPROC(t);
1776         fssproj = FSSPROC2FSSPROJ(fssproc);
1777         fsspset = FSSPROJ2FSSPSET(fssproj);
1778         fsszone = fssproj->fssp_fsszone;
1779 
1780         mutex_enter(&fsspsets_lock);
1781         mutex_enter(&fsspset->fssps_lock);
1782 
1783         thread_lock(t);
1784         disp_lock_enter_high(&fsspset->fssps_displock);
1785         if (t->t_state == TS_ONPROC || t->t_state == TS_RUN) {
1786                 if (--fssproj->fssp_runnable == 0) {
1787                         fsszone->fssz_shares -= fssproj->fssp_shares;
1788                         if (--fsszone->fssz_runnable == 0)
1789                                 fsspset->fssps_shares -= fsszone->fssz_rshares;
1790                 }
1791                 ASSERT(fssproc->fss_runnable == 1);
1792                 fssproc->fss_runnable = 0;
1793         }
1794         if (--fssproj->fssp_threads == 0) {
1795                 fss_remove_fssproj(fsspset, fssproj);
1796                 free = 1;
1797         }
1798         disp_lock_exit_high(&fsspset->fssps_displock);
1799         fssproc->fss_proj = NULL;    /* mark this thread as already exited */
1800         thread_unlock(t);
1801 
1802         if (free) {
1803                 if (fsszone->fssz_nproj == 0)
1804                         kmem_free(fsszone, sizeof (fsszone_t));
1805                 kmem_free(fssproj, sizeof (fssproj_t));
1806         }
1807         mutex_exit(&fsspset->fssps_lock);
1808         mutex_exit(&fsspsets_lock);
1809 
1810         /*
1811          * A thread could be exiting in between clock ticks, so we need to
1812          * calculate how much CPU time it used since it was charged last time.
1813          *
1814          * CPU caps are not enforced on exiting processes - it is usually
1815          * desirable to exit as soon as possible to free resources.
1816          */
1817         if (CPUCAPS_ON()) {
1818                 thread_lock(t);
1819                 fssproc = FSSPROC(t);
1820                 (void) cpucaps_charge(t, &fssproc->fss_caps,
1821                     CPUCAPS_CHARGE_ONLY);
1822                 thread_unlock(t);
1823         }
1824 }
1825 
1826 static void
1827 fss_nullsys()
1828 {
1829 }
1830 
1831 /*
1832  * fss_swapin() returns -1 if the thread is loaded or is not eligible to be
1833  * swapped in. Otherwise, it returns the thread's effective priority based
1834  * on swapout time and size of process (0 <= epri <= 0 SHRT_MAX).
1835  */
1836 /*ARGSUSED*/
1837 static pri_t
1838 fss_swapin(kthread_t *t, int flags)
1839 {
1840         fssproc_t *fssproc = FSSPROC(t);
1841         long epri = -1;
1842         proc_t *pp = ttoproc(t);
1843 
1844         ASSERT(THREAD_LOCK_HELD(t));
1845 
1846         if (t->t_state == TS_RUN && (t->t_schedflag & TS_LOAD) == 0) {
1847                 time_t swapout_time;
1848 
1849                 swapout_time = (ddi_get_lbolt() - t->t_stime) / hz;
1850                 if (INHERITED(t) || (fssproc->fss_flags & FSSKPRI)) {
1851                         epri = (long)DISP_PRIO(t) + swapout_time;
1852                 } else {
1853                         /*
1854                          * Threads which have been out for a long time,
1855                          * have high user mode priority and are associated
1856                          * with a small address space are more deserving.
1857                          */
1858                         epri = fssproc->fss_umdpri;
1859                         ASSERT(epri >= 0 && epri <= fss_maxumdpri);
1860                         epri += swapout_time - pp->p_swrss / nz(maxpgio)/2;
1861                 }
1862                 /*
1863                  * Scale epri so that SHRT_MAX / 2 represents zero priority.
1864                  */
1865                 epri += SHRT_MAX / 2;
1866                 if (epri < 0)
1867                         epri = 0;
1868                 else if (epri > SHRT_MAX)
1869                         epri = SHRT_MAX;
1870         }
1871         return ((pri_t)epri);
1872 }
1873 
1874 /*
1875  * fss_swapout() returns -1 if the thread isn't loaded or is not eligible to
1876  * be swapped out. Otherwise, it returns the thread's effective priority
1877  * based on if the swapper is in softswap or hardswap mode.
1878  */
1879 static pri_t
1880 fss_swapout(kthread_t *t, int flags)
1881 {
1882         fssproc_t *fssproc = FSSPROC(t);
1883         long epri = -1;
1884         proc_t *pp = ttoproc(t);
1885         time_t swapin_time;
1886 
1887         ASSERT(THREAD_LOCK_HELD(t));
1888 
1889         if (INHERITED(t) ||
1890             (fssproc->fss_flags & FSSKPRI) ||
1891             (t->t_proc_flag & TP_LWPEXIT) ||
1892             (t->t_state & (TS_ZOMB|TS_FREE|TS_STOPPED|TS_ONPROC|TS_WAIT)) ||
1893             !(t->t_schedflag & TS_LOAD) ||
1894             !(SWAP_OK(t)))
1895                 return (-1);
1896 
1897         ASSERT(t->t_state & (TS_SLEEP | TS_RUN));
1898 
1899         swapin_time = (ddi_get_lbolt() - t->t_stime) / hz;
1900 
1901         if (flags == SOFTSWAP) {
1902                 if (t->t_state == TS_SLEEP && swapin_time > maxslp) {
1903                         epri = 0;
1904                 } else {
1905                         return ((pri_t)epri);
1906                 }
1907         } else {
1908                 pri_t pri;
1909 
1910                 if ((t->t_state == TS_SLEEP && swapin_time > fss_minslp) ||
1911                     (t->t_state == TS_RUN && swapin_time > fss_minrun)) {
1912                         pri = fss_maxumdpri;
1913                         epri = swapin_time -
1914                             (rm_asrss(pp->p_as) / nz(maxpgio)/2) - (long)pri;
1915                 } else {
1916                         return ((pri_t)epri);
1917                 }
1918         }
1919 
1920         /*
1921          * Scale epri so that SHRT_MAX / 2 represents zero priority.
1922          */
1923         epri += SHRT_MAX / 2;
1924         if (epri < 0)
1925                 epri = 0;
1926         else if (epri > SHRT_MAX)
1927                 epri = SHRT_MAX;
1928 
1929         return ((pri_t)epri);
1930 }
1931 
1932 /*
1933  * If thread is currently at a kernel mode priority (has slept) and is
1934  * returning to the userland we assign it the appropriate user mode priority
1935  * and time quantum here.  If we're lowering the thread's priority below that
1936  * of other runnable threads then we will set runrun via cpu_surrender() to
1937  * cause preemption.
1938  */
1939 static void
1940 fss_trapret(kthread_t *t)
1941 {
1942         fssproc_t *fssproc = FSSPROC(t);
1943         cpu_t *cp = CPU;
1944 
1945         ASSERT(THREAD_LOCK_HELD(t));
1946         ASSERT(t == curthread);
1947         ASSERT(cp->cpu_dispthread == t);
1948         ASSERT(t->t_state == TS_ONPROC);
1949 
1950         t->t_kpri_req = 0;
1951         if (fssproc->fss_flags & FSSKPRI) {
1952                 /*
1953                  * If thread has blocked in the kernel
1954                  */
1955                 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
1956                 cp->cpu_dispatch_pri = DISP_PRIO(t);
1957                 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
1958                 fssproc->fss_flags &= ~FSSKPRI;
1959 
1960                 if (DISP_MUST_SURRENDER(t))
1961                         cpu_surrender(t);
1962         }
1963 
1964         /*
1965          * Swapout lwp if the swapper is waiting for this thread to reach
1966          * a safe point.
1967          */
1968         if (t->t_schedflag & TS_SWAPENQ) {
1969                 thread_unlock(t);
1970                 swapout_lwp(ttolwp(t));
1971                 thread_lock(t);
1972         }
1973 }
1974 
1975 /*
1976  * Arrange for thread to be placed in appropriate location on dispatcher queue.
1977  * This is called with the current thread in TS_ONPROC and locked.
1978  */
1979 static void
1980 fss_preempt(kthread_t *t)
1981 {
1982         fssproc_t *fssproc = FSSPROC(t);
1983         klwp_t *lwp;
1984         uint_t flags;
1985 
1986         ASSERT(t == curthread);
1987         ASSERT(THREAD_LOCK_HELD(curthread));
1988         ASSERT(t->t_state == TS_ONPROC);
1989 
1990         /*
1991          * If preempted in the kernel, make sure the thread has a kernel
1992          * priority if needed.
1993          */
1994         lwp = curthread->t_lwp;
1995         if (!(fssproc->fss_flags & FSSKPRI) && lwp != NULL && t->t_kpri_req) {
1996                 fssproc->fss_flags |= FSSKPRI;
1997                 THREAD_CHANGE_PRI(t, minclsyspri);
1998                 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
1999                 t->t_trapret = 1;    /* so that fss_trapret will run */
2000                 aston(t);
2001         }
2002 
2003         /*
2004          * This thread may be placed on wait queue by CPU Caps. In this case we
2005          * do not need to do anything until it is removed from the wait queue.
2006          * Do not enforce CPU caps on threads running at a kernel priority
2007          */
2008         if (CPUCAPS_ON()) {
2009                 (void) cpucaps_charge(t, &fssproc->fss_caps,
2010                     CPUCAPS_CHARGE_ENFORCE);
2011 
2012                 if (!(fssproc->fss_flags & FSSKPRI) && CPUCAPS_ENFORCE(t))
2013                         return;
2014         }
2015 
2016         /*
2017          * If preempted in user-land mark the thread as swappable because it
2018          * cannot be holding any kernel locks.
2019          */
2020         ASSERT(t->t_schedflag & TS_DONT_SWAP);
2021         if (lwp != NULL && lwp->lwp_state == LWP_USER)
2022                 t->t_schedflag &= ~TS_DONT_SWAP;
2023 
2024         /*
2025          * Check to see if we're doing "preemption control" here.  If
2026          * we are, and if the user has requested that this thread not
2027          * be preempted, and if preemptions haven't been put off for
2028          * too long, let the preemption happen here but try to make
2029          * sure the thread is rescheduled as soon as possible.  We do
2030          * this by putting it on the front of the highest priority run
2031          * queue in the FSS class.  If the preemption has been put off
2032          * for too long, clear the "nopreempt" bit and let the thread
2033          * be preempted.
2034          */
2035         if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2036                 if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2037                         DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);
2038                         if (!(fssproc->fss_flags & FSSKPRI)) {
2039                                 /*
2040                                  * If not already remembered, remember current
2041                                  * priority for restoration in fss_yield().
2042                                  */
2043                                 if (!(fssproc->fss_flags & FSSRESTORE)) {
2044                                         fssproc->fss_scpri = t->t_pri;
2045                                         fssproc->fss_flags |= FSSRESTORE;
2046                                 }
2047                                 THREAD_CHANGE_PRI(t, fss_maxumdpri);
2048                                 t->t_schedflag |= TS_DONT_SWAP;
2049                         }
2050                         schedctl_set_yield(t, 1);
2051                         setfrontdq(t);
2052                         return;
2053                 } else {
2054                         if (fssproc->fss_flags & FSSRESTORE) {
2055                                 THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
2056                                 fssproc->fss_flags &= ~FSSRESTORE;
2057                         }
2058                         schedctl_set_nopreempt(t, 0);
2059                         DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
2060                         /*
2061                          * Fall through and be preempted below.
2062                          */
2063                 }
2064         }
2065 
2066         flags = fssproc->fss_flags & (FSSBACKQ | FSSKPRI);
2067 
2068         if (flags == FSSBACKQ) {
2069                 fssproc->fss_timeleft = fss_quantum;
2070                 fssproc->fss_flags &= ~FSSBACKQ;
2071                 setbackdq(t);
2072         } else if (flags == (FSSBACKQ | FSSKPRI)) {
2073                 fssproc->fss_flags &= ~FSSBACKQ;
2074                 setbackdq(t);
2075         } else {
2076                 setfrontdq(t);
2077         }
2078 }
2079 
2080 /*
2081  * Called when a thread is waking up and is to be placed on the run queue.
2082  */
2083 static void
2084 fss_setrun(kthread_t *t)
2085 {
2086         fssproc_t *fssproc = FSSPROC(t);
2087 
2088         ASSERT(THREAD_LOCK_HELD(t));    /* t should be in transition */
2089 
2090         if (t->t_state == TS_SLEEP || t->t_state == TS_STOPPED)
2091                 fss_active(t);
2092 
2093         fssproc->fss_timeleft = fss_quantum;
2094 
2095         fssproc->fss_flags &= ~FSSBACKQ;
2096         /*
2097          * If previously were running at the kernel priority then keep that
2098          * priority and the fss_timeleft doesn't matter.
2099          */
2100         if ((fssproc->fss_flags & FSSKPRI) == 0)
2101                 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2102 
2103         if (t->t_disp_time != ddi_get_lbolt())
2104                 setbackdq(t);
2105         else
2106                 setfrontdq(t);
2107 }
2108 
2109 /*
2110  * Prepare thread for sleep. We reset the thread priority so it will run at the
2111  * kernel priority level when it wakes up.
2112  */
2113 static void
2114 fss_sleep(kthread_t *t)
2115 {
2116         fssproc_t *fssproc = FSSPROC(t);
2117 
2118         ASSERT(t == curthread);
2119         ASSERT(THREAD_LOCK_HELD(t));
2120 
2121         ASSERT(t->t_state == TS_ONPROC);
2122 
2123         /*
2124          * Account for time spent on CPU before going to sleep.
2125          */
2126         (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2127 
2128         fss_inactive(t);
2129 
2130         /*
2131          * Assign a system priority to the thread and arrange for it to be
2132          * retained when the thread is next placed on the run queue (i.e.,
2133          * when it wakes up) instead of being given a new pri.  Also arrange
2134          * for trapret processing as the thread leaves the system call so it
2135          * will drop back to normal priority range.
2136          */
2137         if (t->t_kpri_req) {
2138                 THREAD_CHANGE_PRI(t, minclsyspri);
2139                 fssproc->fss_flags |= FSSKPRI;
2140                 t->t_trapret = 1;    /* so that fss_trapret will run */
2141                 aston(t);
2142         } else if (fssproc->fss_flags & FSSKPRI) {
2143                 /*
2144                  * The thread has done a THREAD_KPRI_REQUEST(), slept, then
2145                  * done THREAD_KPRI_RELEASE() (so no t_kpri_req is 0 again),
2146                  * then slept again all without finishing the current system
2147                  * call so trapret won't have cleared FSSKPRI
2148                  */
2149                 fssproc->fss_flags &= ~FSSKPRI;
2150                 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2151                 if (DISP_MUST_SURRENDER(curthread))
2152                         cpu_surrender(t);
2153         }
2154         t->t_stime = ddi_get_lbolt();        /* time stamp for the swapper */
2155 }
2156 
2157 /*
2158  * A tick interrupt has ocurrend on a running thread. Check to see if our
2159  * time slice has expired.  We must also clear the TS_DONT_SWAP flag in
2160  * t_schedflag if the thread is eligible to be swapped out.
2161  */
2162 static void
2163 fss_tick(kthread_t *t)
2164 {
2165         fssproc_t *fssproc;
2166         fssproj_t *fssproj;
2167         klwp_t *lwp;
2168         boolean_t call_cpu_surrender = B_FALSE;
2169         boolean_t cpucaps_enforce = B_FALSE;
2170 
2171         ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
2172 
2173         /*
2174          * It's safe to access fsspset and fssproj structures because we're
2175          * holding our p_lock here.
2176          */
2177         thread_lock(t);
2178         fssproc = FSSPROC(t);
2179         fssproj = FSSPROC2FSSPROJ(fssproc);
2180         if (fssproj != NULL) {
2181                 fsspset_t *fsspset = FSSPROJ2FSSPSET(fssproj);
2182                 disp_lock_enter_high(&fsspset->fssps_displock);
2183                 fssproj->fssp_ticks += fss_nice_tick[fssproc->fss_nice];
2184                 fssproc->fss_ticks++;
2185                 disp_lock_exit_high(&fsspset->fssps_displock);
2186         }
2187 
2188         /*
2189          * Keep track of thread's project CPU usage.  Note that projects
2190          * get charged even when threads are running in the kernel.
2191          * Do not surrender CPU if running in the SYS class.
2192          */
2193         if (CPUCAPS_ON()) {
2194                 cpucaps_enforce = cpucaps_charge(t,
2195                     &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE) &&
2196                     !(fssproc->fss_flags & FSSKPRI);
2197         }
2198 
2199         /*
2200          * A thread's execution time for threads running in the SYS class
2201          * is not tracked.
2202          */
2203         if ((fssproc->fss_flags & FSSKPRI) == 0) {
2204                 /*
2205                  * If thread is not in kernel mode, decrement its fss_timeleft
2206                  */
2207                 if (--fssproc->fss_timeleft <= 0) {
2208                         pri_t new_pri;
2209 
2210                         /*
2211                          * If we're doing preemption control and trying to
2212                          * avoid preempting this thread, just note that the
2213                          * thread should yield soon and let it keep running
2214                          * (unless it's been a while).
2215                          */
2216                         if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2217                                 if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2218                                         DTRACE_SCHED1(schedctl__nopreempt,
2219                                             kthread_t *, t);
2220                                         schedctl_set_yield(t, 1);
2221                                         thread_unlock_nopreempt(t);
2222                                         return;
2223                                 }
2224                         }
2225                         fssproc->fss_flags &= ~FSSRESTORE;
2226 
2227                         fss_newpri(fssproc);
2228                         new_pri = fssproc->fss_umdpri;
2229                         ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
2230 
2231                         /*
2232                          * When the priority of a thread is changed, it may
2233                          * be necessary to adjust its position on a sleep queue
2234                          * or dispatch queue. The function thread_change_pri
2235                          * accomplishes this.
2236                          */
2237                         if (thread_change_pri(t, new_pri, 0)) {
2238                                 if ((t->t_schedflag & TS_LOAD) &&
2239                                     (lwp = t->t_lwp) &&
2240                                     lwp->lwp_state == LWP_USER)
2241                                         t->t_schedflag &= ~TS_DONT_SWAP;
2242                                 fssproc->fss_timeleft = fss_quantum;
2243                         } else {
2244                                 call_cpu_surrender = B_TRUE;
2245                         }
2246                 } else if (t->t_state == TS_ONPROC &&
2247                     t->t_pri < t->t_disp_queue->disp_maxrunpri) {
2248                         /*
2249                          * If there is a higher-priority thread which is
2250                          * waiting for a processor, then thread surrenders
2251                          * the processor.
2252                          */
2253                         call_cpu_surrender = B_TRUE;
2254                 }
2255         }
2256 
2257         if (cpucaps_enforce && 2 * fssproc->fss_timeleft > fss_quantum) {
2258                 /*
2259                  * The thread used more than half of its quantum, so assume that
2260                  * it used the whole quantum.
2261                  *
2262                  * Update thread's priority just before putting it on the wait
2263                  * queue so that it gets charged for the CPU time from its
2264                  * quantum even before that quantum expires.
2265                  */
2266                 fss_newpri(fssproc);
2267                 if (t->t_pri != fssproc->fss_umdpri)
2268                         fss_change_priority(t, fssproc);
2269 
2270                 /*
2271                  * We need to call cpu_surrender for this thread due to cpucaps
2272                  * enforcement, but fss_change_priority may have already done
2273                  * so. In this case FSSBACKQ is set and there is no need to call
2274                  * cpu-surrender again.
2275                  */
2276                 if (!(fssproc->fss_flags & FSSBACKQ))
2277                         call_cpu_surrender = B_TRUE;
2278         }
2279 
2280         if (call_cpu_surrender) {
2281                 fssproc->fss_flags |= FSSBACKQ;
2282                 cpu_surrender(t);
2283         }
2284 
2285         thread_unlock_nopreempt(t);     /* clock thread can't be preempted */
2286 }
2287 
2288 /*
2289  * Processes waking up go to the back of their queue.  We don't need to assign
2290  * a time quantum here because thread is still at a kernel mode priority and
2291  * the time slicing is not done for threads running in the kernel after
2292  * sleeping.  The proper time quantum will be assigned by fss_trapret before the
2293  * thread returns to user mode.
2294  */
2295 static void
2296 fss_wakeup(kthread_t *t)
2297 {
2298         fssproc_t *fssproc;
2299 
2300         ASSERT(THREAD_LOCK_HELD(t));
2301         ASSERT(t->t_state == TS_SLEEP);
2302 
2303         fss_active(t);
2304 
2305         t->t_stime = ddi_get_lbolt();                /* time stamp for the swapper */
2306         fssproc = FSSPROC(t);
2307         fssproc->fss_flags &= ~FSSBACKQ;
2308 
2309         if (fssproc->fss_flags & FSSKPRI) {
2310                 /*
2311                  * If we already have a kernel priority assigned, then we
2312                  * just use it.
2313                  */
2314                 setbackdq(t);
2315         } else if (t->t_kpri_req) {
2316                 /*
2317                  * Give thread a priority boost if we were asked.
2318                  */
2319                 fssproc->fss_flags |= FSSKPRI;
2320                 THREAD_CHANGE_PRI(t, minclsyspri);
2321                 setbackdq(t);
2322                 t->t_trapret = 1;    /* so that fss_trapret will run */
2323                 aston(t);
2324         } else {
2325                 /*
2326                  * Otherwise, we recalculate the priority.
2327                  */
2328                 if (t->t_disp_time == ddi_get_lbolt()) {
2329                         setfrontdq(t);
2330                 } else {
2331                         fssproc->fss_timeleft = fss_quantum;
2332                         THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2333                         setbackdq(t);
2334                 }
2335         }
2336 }
2337 
2338 /*
2339  * fss_donice() is called when a nice(1) command is issued on the thread to
2340  * alter the priority. The nice(1) command exists in Solaris for compatibility.
2341  * Thread priority adjustments should be done via priocntl(1).
2342  */
2343 static int
2344 fss_donice(kthread_t *t, cred_t *cr, int incr, int *retvalp)
2345 {
2346         int newnice;
2347         fssproc_t *fssproc = FSSPROC(t);
2348         fssparms_t fssparms;
2349 
2350         /*
2351          * If there is no change to priority, just return current setting.
2352          */
2353         if (incr == 0) {
2354                 if (retvalp)
2355                         *retvalp = fssproc->fss_nice - NZERO;
2356                 return (0);
2357         }
2358 
2359         if ((incr < 0 || incr > 2 * NZERO) && secpolicy_raisepriority(cr) != 0)
2360                 return (EPERM);
2361 
2362         /*
2363          * Specifying a nice increment greater than the upper limit of
2364          * FSS_NICE_MAX (== 2 * NZERO - 1) will result in the thread's nice
2365          * value being set to the upper limit.  We check for this before
2366          * computing the new value because otherwise we could get overflow
2367          * if a privileged user specified some ridiculous increment.
2368          */
2369         if (incr > FSS_NICE_MAX)
2370                 incr = FSS_NICE_MAX;
2371 
2372         newnice = fssproc->fss_nice + incr;
2373         if (newnice > FSS_NICE_MAX)
2374                 newnice = FSS_NICE_MAX;
2375         else if (newnice < FSS_NICE_MIN)
2376                 newnice = FSS_NICE_MIN;
2377 
2378         fssparms.fss_uprilim = fssparms.fss_upri =
2379             -((newnice - NZERO) * fss_maxupri) / NZERO;
2380 
2381         /*
2382          * Reset the uprilim and upri values of the thread.
2383          */
2384         (void) fss_parmsset(t, (void *)&fssparms, (id_t)0, (cred_t *)NULL);
2385 
2386         /*
2387          * Although fss_parmsset already reset fss_nice it may not have been
2388          * set to precisely the value calculated above because fss_parmsset
2389          * determines the nice value from the user priority and we may have
2390          * truncated during the integer conversion from nice value to user
2391          * priority and back. We reset fss_nice to the value we calculated
2392          * above.
2393          */
2394         fssproc->fss_nice = (char)newnice;
2395 
2396         if (retvalp)
2397                 *retvalp = newnice - NZERO;
2398         return (0);
2399 }
2400 
2401 /*
2402  * Increment the priority of the specified thread by incr and
2403  * return the new value in *retvalp.
2404  */
2405 static int
2406 fss_doprio(kthread_t *t, cred_t *cr, int incr, int *retvalp)
2407 {
2408         int newpri;
2409         fssproc_t *fssproc = FSSPROC(t);
2410         fssparms_t fssparms;
2411 
2412         /*
2413          * If there is no change to priority, just return current setting.
2414          */
2415         if (incr == 0) {
2416                 *retvalp = fssproc->fss_upri;
2417                 return (0);
2418         }
2419 
2420         newpri = fssproc->fss_upri + incr;
2421         if (newpri > fss_maxupri || newpri < -fss_maxupri)
2422                 return (EINVAL);
2423 
2424         *retvalp = newpri;
2425         fssparms.fss_uprilim = fssparms.fss_upri = newpri;
2426 
2427         /*
2428          * Reset the uprilim and upri values of the thread.
2429          */
2430         return (fss_parmsset(t, &fssparms, (id_t)0, cr));
2431 }
2432 
2433 /*
2434  * Return the global scheduling priority that would be assigned to a thread
2435  * entering the fair-sharing class with the fss_upri.
2436  */
2437 /*ARGSUSED*/
2438 static pri_t
2439 fss_globpri(kthread_t *t)
2440 {
2441         ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2442 
2443         return (fss_maxumdpri / 2);
2444 }
2445 
2446 /*
2447  * Called from the yield(2) system call when a thread is yielding (surrendering)
2448  * the processor. The kernel thread is placed at the back of a dispatch queue.
2449  */
2450 static void
2451 fss_yield(kthread_t *t)
2452 {
2453         fssproc_t *fssproc = FSSPROC(t);
2454 
2455         ASSERT(t == curthread);
2456         ASSERT(THREAD_LOCK_HELD(t));
2457 
2458         /*
2459          * Collect CPU usage spent before yielding
2460          */
2461         (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2462 
2463         /*
2464          * Clear the preemption control "yield" bit since the user is
2465          * doing a yield.
2466          */
2467         if (t->t_schedctl)
2468                 schedctl_set_yield(t, 0);
2469         /*
2470          * If fss_preempt() artifically increased the thread's priority
2471          * to avoid preemption, restore the original priority now.
2472          */
2473         if (fssproc->fss_flags & FSSRESTORE) {
2474                 THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
2475                 fssproc->fss_flags &= ~FSSRESTORE;
2476         }
2477         if (fssproc->fss_timeleft < 0) {
2478                 /*
2479                  * Time slice was artificially extended to avoid preemption,
2480                  * so pretend we're preempting it now.
2481                  */
2482                 DTRACE_SCHED1(schedctl__yield, int, -fssproc->fss_timeleft);
2483                 fssproc->fss_timeleft = fss_quantum;
2484         }
2485         fssproc->fss_flags &= ~FSSBACKQ;
2486         setbackdq(t);
2487 }
2488 
2489 void
2490 fss_changeproj(kthread_t *t, void *kp, void *zp, fssbuf_t *projbuf,
2491     fssbuf_t *zonebuf)
2492 {
2493         kproject_t *kpj_new = kp;
2494         zone_t *zone = zp;
2495         fssproj_t *fssproj_old, *fssproj_new;
2496         fsspset_t *fsspset;
2497         kproject_t *kpj_old;
2498         fssproc_t *fssproc;
2499         fsszone_t *fsszone_old, *fsszone_new;
2500         int free = 0;
2501         int id;
2502 
2503         ASSERT(MUTEX_HELD(&cpu_lock));
2504         ASSERT(MUTEX_HELD(&pidlock));
2505         ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2506 
2507         if (t->t_cid != fss_cid)
2508                 return;
2509 
2510         fssproc = FSSPROC(t);
2511         mutex_enter(&fsspsets_lock);
2512         fssproj_old = FSSPROC2FSSPROJ(fssproc);
2513         if (fssproj_old == NULL) {
2514                 mutex_exit(&fsspsets_lock);
2515                 return;
2516         }
2517 
2518         fsspset = FSSPROJ2FSSPSET(fssproj_old);
2519         mutex_enter(&fsspset->fssps_lock);
2520         kpj_old = FSSPROJ2KPROJ(fssproj_old);
2521         fsszone_old = fssproj_old->fssp_fsszone;
2522 
2523         ASSERT(t->t_cpupart == fsspset->fssps_cpupart);
2524 
2525         if (kpj_old == kpj_new) {
2526                 mutex_exit(&fsspset->fssps_lock);
2527                 mutex_exit(&fsspsets_lock);
2528                 return;
2529         }
2530 
2531         if ((fsszone_new = fss_find_fsszone(fsspset, zone)) == NULL) {
2532                 /*
2533                  * If the zone for the new project is not currently active on
2534                  * the cpu partition we're on, get one of the pre-allocated
2535                  * buffers and link it in our per-pset zone list.  Such buffers
2536                  * should already exist.
2537                  */
2538                 for (id = 0; id < zonebuf->fssb_size; id++) {
2539                         if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
2540                                 fss_insert_fsszone(fsspset, zone, fsszone_new);
2541                                 zonebuf->fssb_list[id] = NULL;
2542                                 break;
2543                         }
2544                 }
2545         }
2546         ASSERT(fsszone_new != NULL);
2547         if ((fssproj_new = fss_find_fssproj(fsspset, kpj_new)) == NULL) {
2548                 /*
2549                  * If our new project is not currently running
2550                  * on the cpu partition we're on, get one of the
2551                  * pre-allocated buffers and link it in our new cpu
2552                  * partition doubly linked list. Such buffers should already
2553                  * exist.
2554                  */
2555                 for (id = 0; id < projbuf->fssb_size; id++) {
2556                         if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
2557                                 fss_insert_fssproj(fsspset, kpj_new,
2558                                     fsszone_new, fssproj_new);
2559                                 projbuf->fssb_list[id] = NULL;
2560                                 break;
2561                         }
2562                 }
2563         }
2564         ASSERT(fssproj_new != NULL);
2565 
2566         thread_lock(t);
2567         if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2568             t->t_state == TS_WAIT)
2569                 fss_inactive(t);
2570         ASSERT(fssproj_old->fssp_threads > 0);
2571         if (--fssproj_old->fssp_threads == 0) {
2572                 fss_remove_fssproj(fsspset, fssproj_old);
2573                 free = 1;
2574         }
2575         fssproc->fss_proj = fssproj_new;
2576         fssproc->fss_fsspri = 0;
2577         fssproj_new->fssp_threads++;
2578         if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2579             t->t_state == TS_WAIT)
2580                 fss_active(t);
2581         thread_unlock(t);
2582         if (free) {
2583                 if (fsszone_old->fssz_nproj == 0)
2584                         kmem_free(fsszone_old, sizeof (fsszone_t));
2585                 kmem_free(fssproj_old, sizeof (fssproj_t));
2586         }
2587 
2588         mutex_exit(&fsspset->fssps_lock);
2589         mutex_exit(&fsspsets_lock);
2590 }
2591 
2592 void
2593 fss_changepset(kthread_t *t, void *newcp, fssbuf_t *projbuf,
2594     fssbuf_t *zonebuf)
2595 {
2596         fsspset_t *fsspset_old, *fsspset_new;
2597         fssproj_t *fssproj_old, *fssproj_new;
2598         fsszone_t *fsszone_old, *fsszone_new;
2599         fssproc_t *fssproc;
2600         kproject_t *kpj;
2601         zone_t *zone;
2602         int id;
2603 
2604         ASSERT(MUTEX_HELD(&cpu_lock));
2605         ASSERT(MUTEX_HELD(&pidlock));
2606         ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2607 
2608         if (t->t_cid != fss_cid)
2609                 return;
2610 
2611         fssproc = FSSPROC(t);
2612         zone = ttoproc(t)->p_zone;
2613         mutex_enter(&fsspsets_lock);
2614         fssproj_old = FSSPROC2FSSPROJ(fssproc);
2615         if (fssproj_old == NULL) {
2616                 mutex_exit(&fsspsets_lock);
2617                 return;
2618         }
2619         fsszone_old = fssproj_old->fssp_fsszone;
2620         fsspset_old = FSSPROJ2FSSPSET(fssproj_old);
2621         kpj = FSSPROJ2KPROJ(fssproj_old);
2622 
2623         if (fsspset_old->fssps_cpupart == newcp) {
2624                 mutex_exit(&fsspsets_lock);
2625                 return;
2626         }
2627 
2628         ASSERT(ttoproj(t) == kpj);
2629 
2630         fsspset_new = fss_find_fsspset(newcp);
2631 
2632         mutex_enter(&fsspset_new->fssps_lock);
2633         if ((fsszone_new = fss_find_fsszone(fsspset_new, zone)) == NULL) {
2634                 for (id = 0; id < zonebuf->fssb_size; id++) {
2635                         if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
2636                                 fss_insert_fsszone(fsspset_new, zone,
2637                                     fsszone_new);
2638                                 zonebuf->fssb_list[id] = NULL;
2639                                 break;
2640                         }
2641                 }
2642         }
2643         ASSERT(fsszone_new != NULL);
2644         if ((fssproj_new = fss_find_fssproj(fsspset_new, kpj)) == NULL) {
2645                 for (id = 0; id < projbuf->fssb_size; id++) {
2646                         if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
2647                                 fss_insert_fssproj(fsspset_new, kpj,
2648                                     fsszone_new, fssproj_new);
2649                                 projbuf->fssb_list[id] = NULL;
2650                                 break;
2651                         }
2652                 }
2653         }
2654         ASSERT(fssproj_new != NULL);
2655 
2656         fssproj_new->fssp_threads++;
2657         thread_lock(t);
2658         if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2659             t->t_state == TS_WAIT)
2660                 fss_inactive(t);
2661         fssproc->fss_proj = fssproj_new;
2662         fssproc->fss_fsspri = 0;
2663         if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2664             t->t_state == TS_WAIT)
2665                 fss_active(t);
2666         thread_unlock(t);
2667         mutex_exit(&fsspset_new->fssps_lock);
2668 
2669         mutex_enter(&fsspset_old->fssps_lock);
2670         if (--fssproj_old->fssp_threads == 0) {
2671                 fss_remove_fssproj(fsspset_old, fssproj_old);
2672                 if (fsszone_old->fssz_nproj == 0)
2673                         kmem_free(fsszone_old, sizeof (fsszone_t));
2674                 kmem_free(fssproj_old, sizeof (fssproj_t));
2675         }
2676         mutex_exit(&fsspset_old->fssps_lock);
2677 
2678         mutex_exit(&fsspsets_lock);
2679 }