332 #define FSS_NICE_MIN 0
333 #define FSS_NICE_MAX (2 * NZERO - 1)
334 #define FSS_NICE_RANGE (FSS_NICE_MAX - FSS_NICE_MIN + 1)
335
336 static int fss_nice_tick[FSS_NICE_RANGE];
337 static int fss_nice_decay[FSS_NICE_RANGE];
338
339 static pri_t fss_maxupri = FSS_MAXUPRI; /* maximum FSS user priority */
340 static pri_t fss_maxumdpri; /* maximum user mode fss priority */
341 static pri_t fss_maxglobpri; /* maximum global priority used by fss class */
342 static pri_t fss_minglobpri; /* minimum global priority */
343
344 static fssproc_t fss_listhead[FSS_LISTS];
345 static kmutex_t fss_listlock[FSS_LISTS];
346
347 static fsspset_t *fsspsets;
348 static kmutex_t fsspsets_lock; /* protects fsspsets */
349
350 static id_t fss_cid;
351
352 static time_t fss_minrun = 2; /* t_pri becomes 59 within 2 secs */
353 static time_t fss_minslp = 2; /* min time on sleep queue for hardswap */
354 static int fss_quantum = 11;
355
356 static void fss_newpri(fssproc_t *, boolean_t);
357 static void fss_update(void *);
358 static int fss_update_list(int);
359 static void fss_change_priority(kthread_t *, fssproc_t *);
360
361 static int fss_admin(caddr_t, cred_t *);
362 static int fss_getclinfo(void *);
363 static int fss_parmsin(void *);
364 static int fss_parmsout(void *, pc_vaparms_t *);
365 static int fss_vaparmsin(void *, pc_vaparms_t *);
366 static int fss_vaparmsout(void *, pc_vaparms_t *);
367 static int fss_getclpri(pcpri_t *);
368 static int fss_alloc(void **, int);
369 static void fss_free(void *);
370
371 static int fss_enterclass(kthread_t *, id_t, void *, cred_t *, void *);
372 static void fss_exitclass(void *);
373 static int fss_canexit(kthread_t *, cred_t *);
374 static int fss_fork(kthread_t *, kthread_t *, void *);
375 static void fss_forkret(kthread_t *, kthread_t *);
376 static void fss_parmsget(kthread_t *, void *);
377 static int fss_parmsset(kthread_t *, void *, id_t, cred_t *);
378 static void fss_stop(kthread_t *, int, int);
379 static void fss_exit(kthread_t *);
380 static void fss_active(kthread_t *);
381 static void fss_inactive(kthread_t *);
382 static pri_t fss_swapin(kthread_t *, int);
383 static pri_t fss_swapout(kthread_t *, int);
384 static void fss_trapret(kthread_t *);
385 static void fss_preempt(kthread_t *);
386 static void fss_setrun(kthread_t *);
387 static void fss_sleep(kthread_t *);
388 static void fss_tick(kthread_t *);
389 static void fss_wakeup(kthread_t *);
390 static int fss_donice(kthread_t *, cred_t *, int, int *);
391 static int fss_doprio(kthread_t *, cred_t *, int, int *);
392 static pri_t fss_globpri(kthread_t *);
393 static void fss_yield(kthread_t *);
394 static void fss_nullsys();
395
396 static struct classfuncs fss_classfuncs = {
397 /* class functions */
398 fss_admin,
399 fss_getclinfo,
400 fss_parmsin,
401 fss_parmsout,
402 fss_vaparmsin,
403 fss_vaparmsout,
404 fss_getclpri,
405 fss_alloc,
406 fss_free,
407
408 /* thread functions */
409 fss_enterclass,
410 fss_exitclass,
411 fss_canexit,
412 fss_fork,
413 fss_forkret,
414 fss_parmsget,
415 fss_parmsset,
416 fss_stop,
417 fss_exit,
418 fss_active,
419 fss_inactive,
420 fss_swapin,
421 fss_swapout,
422 fss_trapret,
423 fss_preempt,
424 fss_setrun,
425 fss_sleep,
426 fss_tick,
427 fss_wakeup,
428 fss_donice,
429 fss_globpri,
430 fss_nullsys, /* set_process_group */
431 fss_yield,
432 fss_doprio,
433 };
434
435 int
436 _init()
437 {
438 return (mod_install(&modlinkage));
439 }
440
441 int
2123 * calculate how much CPU time it used since it was charged last time.
2124 *
2125 * CPU caps are not enforced on exiting processes - it is usually
2126 * desirable to exit as soon as possible to free resources.
2127 */
2128 if (CPUCAPS_ON()) {
2129 thread_lock(t);
2130 fssproc = FSSPROC(t);
2131 (void) cpucaps_charge(t, &fssproc->fss_caps,
2132 CPUCAPS_CHARGE_ONLY);
2133 thread_unlock(t);
2134 }
2135 }
2136
2137 static void
2138 fss_nullsys()
2139 {
2140 }
2141
2142 /*
2143 * fss_swapin() returns -1 if the thread is loaded or is not eligible to be
2144 * swapped in. Otherwise, it returns the thread's effective priority based
2145 * on swapout time and size of process (0 <= epri <= 0 SHRT_MAX).
2146 */
2147 /*ARGSUSED*/
2148 static pri_t
2149 fss_swapin(kthread_t *t, int flags)
2150 {
2151 fssproc_t *fssproc = FSSPROC(t);
2152 long epri = -1;
2153 proc_t *pp = ttoproc(t);
2154
2155 ASSERT(THREAD_LOCK_HELD(t));
2156
2157 if (t->t_state == TS_RUN && (t->t_schedflag & TS_LOAD) == 0) {
2158 time_t swapout_time;
2159
2160 swapout_time = (ddi_get_lbolt() - t->t_stime) / hz;
2161 if (INHERITED(t) || (fssproc->fss_flags & FSSKPRI)) {
2162 epri = (long)DISP_PRIO(t) + swapout_time;
2163 } else {
2164 /*
2165 * Threads which have been out for a long time,
2166 * have high user mode priority and are associated
2167 * with a small address space are more deserving.
2168 */
2169 epri = fssproc->fss_umdpri;
2170 ASSERT(epri >= 0 && epri <= fss_maxumdpri);
2171 epri += swapout_time - pp->p_swrss / nz(maxpgio)/2;
2172 }
2173 /*
2174 * Scale epri so that SHRT_MAX / 2 represents zero priority.
2175 */
2176 epri += SHRT_MAX / 2;
2177 if (epri < 0)
2178 epri = 0;
2179 else if (epri > SHRT_MAX)
2180 epri = SHRT_MAX;
2181 }
2182 return ((pri_t)epri);
2183 }
2184
2185 /*
2186 * fss_swapout() returns -1 if the thread isn't loaded or is not eligible to
2187 * be swapped out. Otherwise, it returns the thread's effective priority
2188 * based on if the swapper is in softswap or hardswap mode.
2189 */
2190 static pri_t
2191 fss_swapout(kthread_t *t, int flags)
2192 {
2193 fssproc_t *fssproc = FSSPROC(t);
2194 long epri = -1;
2195 proc_t *pp = ttoproc(t);
2196 time_t swapin_time;
2197
2198 ASSERT(THREAD_LOCK_HELD(t));
2199
2200 if (INHERITED(t) ||
2201 (fssproc->fss_flags & FSSKPRI) ||
2202 (t->t_proc_flag & TP_LWPEXIT) ||
2203 (t->t_state & (TS_ZOMB|TS_FREE|TS_STOPPED|TS_ONPROC|TS_WAIT)) ||
2204 !(t->t_schedflag & TS_LOAD) ||
2205 !(SWAP_OK(t)))
2206 return (-1);
2207
2208 ASSERT(t->t_state & (TS_SLEEP | TS_RUN));
2209
2210 swapin_time = (ddi_get_lbolt() - t->t_stime) / hz;
2211
2212 if (flags == SOFTSWAP) {
2213 if (t->t_state == TS_SLEEP && swapin_time > maxslp) {
2214 epri = 0;
2215 } else {
2216 return ((pri_t)epri);
2217 }
2218 } else {
2219 pri_t pri;
2220
2221 if ((t->t_state == TS_SLEEP && swapin_time > fss_minslp) ||
2222 (t->t_state == TS_RUN && swapin_time > fss_minrun)) {
2223 pri = fss_maxumdpri;
2224 epri = swapin_time -
2225 (rm_asrss(pp->p_as) / nz(maxpgio)/2) - (long)pri;
2226 } else {
2227 return ((pri_t)epri);
2228 }
2229 }
2230
2231 /*
2232 * Scale epri so that SHRT_MAX / 2 represents zero priority.
2233 */
2234 epri += SHRT_MAX / 2;
2235 if (epri < 0)
2236 epri = 0;
2237 else if (epri > SHRT_MAX)
2238 epri = SHRT_MAX;
2239
2240 return ((pri_t)epri);
2241 }
2242
2243 /*
2244 * If thread is currently at a kernel mode priority (has slept) and is
2245 * returning to the userland we assign it the appropriate user mode priority
2246 * and time quantum here. If we're lowering the thread's priority below that
2247 * of other runnable threads then we will set runrun via cpu_surrender() to
2248 * cause preemption.
2249 */
2250 static void
2251 fss_trapret(kthread_t *t)
2252 {
2253 fssproc_t *fssproc = FSSPROC(t);
2254 cpu_t *cp = CPU;
2255
2256 ASSERT(THREAD_LOCK_HELD(t));
2257 ASSERT(t == curthread);
2258 ASSERT(cp->cpu_dispthread == t);
2259 ASSERT(t->t_state == TS_ONPROC);
2260
2261 t->t_kpri_req = 0;
2262 if (fssproc->fss_flags & FSSKPRI) {
2263 /*
2264 * If thread has blocked in the kernel
2265 */
2266 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2267 cp->cpu_dispatch_pri = DISP_PRIO(t);
2268 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
2269 fssproc->fss_flags &= ~FSSKPRI;
2270
2271 if (DISP_MUST_SURRENDER(t))
2272 cpu_surrender(t);
2273 }
2274
2275 /*
2276 * Swapout lwp if the swapper is waiting for this thread to reach
2277 * a safe point.
2278 */
2279 if (t->t_schedflag & TS_SWAPENQ) {
2280 thread_unlock(t);
2281 swapout_lwp(ttolwp(t));
2282 thread_lock(t);
2283 }
2284 }
2285
2286 /*
2287 * Arrange for thread to be placed in appropriate location on dispatcher queue.
2288 * This is called with the current thread in TS_ONPROC and locked.
2289 */
2290 static void
2291 fss_preempt(kthread_t *t)
2292 {
2293 fssproc_t *fssproc = FSSPROC(t);
2294 klwp_t *lwp;
2295 uint_t flags;
2296
2297 ASSERT(t == curthread);
2298 ASSERT(THREAD_LOCK_HELD(curthread));
2299 ASSERT(t->t_state == TS_ONPROC);
2300
2301 /*
2302 * If preempted in the kernel, make sure the thread has a kernel
2303 * priority if needed.
2308 THREAD_CHANGE_PRI(t, minclsyspri);
2309 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
2310 t->t_trapret = 1; /* so that fss_trapret will run */
2311 aston(t);
2312 }
2313
2314 /*
2315 * This thread may be placed on wait queue by CPU Caps. In this case we
2316 * do not need to do anything until it is removed from the wait queue.
2317 * Do not enforce CPU caps on threads running at a kernel priority
2318 */
2319 if (CPUCAPS_ON()) {
2320 (void) cpucaps_charge(t, &fssproc->fss_caps,
2321 CPUCAPS_CHARGE_ENFORCE);
2322
2323 if (!(fssproc->fss_flags & FSSKPRI) && CPUCAPS_ENFORCE(t))
2324 return;
2325 }
2326
2327 /*
2328 * If preempted in user-land mark the thread as swappable because it
2329 * cannot be holding any kernel locks.
2330 */
2331 ASSERT(t->t_schedflag & TS_DONT_SWAP);
2332 if (lwp != NULL && lwp->lwp_state == LWP_USER)
2333 t->t_schedflag &= ~TS_DONT_SWAP;
2334
2335 /*
2336 * Check to see if we're doing "preemption control" here. If
2337 * we are, and if the user has requested that this thread not
2338 * be preempted, and if preemptions haven't been put off for
2339 * too long, let the preemption happen here but try to make
2340 * sure the thread is rescheduled as soon as possible. We do
2341 * this by putting it on the front of the highest priority run
2342 * queue in the FSS class. If the preemption has been put off
2343 * for too long, clear the "nopreempt" bit and let the thread
2344 * be preempted.
2345 */
2346 if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2347 if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2348 DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);
2349 if (!(fssproc->fss_flags & FSSKPRI)) {
2350 /*
2351 * If not already remembered, remember current
2352 * priority for restoration in fss_yield().
2353 */
2354 if (!(fssproc->fss_flags & FSSRESTORE)) {
2355 fssproc->fss_scpri = t->t_pri;
2356 fssproc->fss_flags |= FSSRESTORE;
2357 }
2358 THREAD_CHANGE_PRI(t, fss_maxumdpri);
2359 t->t_schedflag |= TS_DONT_SWAP;
2360 }
2361 schedctl_set_yield(t, 1);
2362 setfrontdq(t);
2363 return;
2364 } else {
2365 if (fssproc->fss_flags & FSSRESTORE) {
2366 THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
2367 fssproc->fss_flags &= ~FSSRESTORE;
2368 }
2369 schedctl_set_nopreempt(t, 0);
2370 DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
2371 /*
2372 * Fall through and be preempted below.
2373 */
2374 }
2375 }
2376
2377 flags = fssproc->fss_flags & (FSSBACKQ | FSSKPRI);
2378
2379 if (flags == FSSBACKQ) {
2445 * for trapret processing as the thread leaves the system call so it
2446 * will drop back to normal priority range.
2447 */
2448 if (t->t_kpri_req) {
2449 THREAD_CHANGE_PRI(t, minclsyspri);
2450 fssproc->fss_flags |= FSSKPRI;
2451 t->t_trapret = 1; /* so that fss_trapret will run */
2452 aston(t);
2453 } else if (fssproc->fss_flags & FSSKPRI) {
2454 /*
2455 * The thread has done a THREAD_KPRI_REQUEST(), slept, then
2456 * done THREAD_KPRI_RELEASE() (so no t_kpri_req is 0 again),
2457 * then slept again all without finishing the current system
2458 * call so trapret won't have cleared FSSKPRI
2459 */
2460 fssproc->fss_flags &= ~FSSKPRI;
2461 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2462 if (DISP_MUST_SURRENDER(curthread))
2463 cpu_surrender(t);
2464 }
2465 t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */
2466 }
2467
2468 /*
2469 * A tick interrupt has ocurrend on a running thread. Check to see if our
2470 * time slice has expired. We must also clear the TS_DONT_SWAP flag in
2471 * t_schedflag if the thread is eligible to be swapped out.
2472 */
2473 static void
2474 fss_tick(kthread_t *t)
2475 {
2476 fssproc_t *fssproc;
2477 fssproj_t *fssproj;
2478 klwp_t *lwp;
2479 boolean_t call_cpu_surrender = B_FALSE;
2480 boolean_t cpucaps_enforce = B_FALSE;
2481
2482 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
2483
2484 /*
2485 * It's safe to access fsspset and fssproj structures because we're
2486 * holding our p_lock here.
2487 */
2488 thread_lock(t);
2489 fssproc = FSSPROC(t);
2490 fssproj = FSSPROC2FSSPROJ(fssproc);
2491 if (fssproj != NULL) {
2492 fsspset_t *fsspset = FSSPROJ2FSSPSET(fssproj);
2493 disp_lock_enter_high(&fsspset->fssps_displock);
2494 fssproj->fssp_ticks += fss_nice_tick[fssproc->fss_nice];
2495 fssproj->fssp_tick_cnt++;
2496 fssproc->fss_ticks++;
2497 disp_lock_exit_high(&fsspset->fssps_displock);
2498 }
2530 DTRACE_SCHED1(schedctl__nopreempt,
2531 kthread_t *, t);
2532 schedctl_set_yield(t, 1);
2533 thread_unlock_nopreempt(t);
2534 return;
2535 }
2536 }
2537 fssproc->fss_flags &= ~FSSRESTORE;
2538
2539 fss_newpri(fssproc, B_TRUE);
2540 new_pri = fssproc->fss_umdpri;
2541 ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
2542
2543 /*
2544 * When the priority of a thread is changed, it may
2545 * be necessary to adjust its position on a sleep queue
2546 * or dispatch queue. The function thread_change_pri
2547 * accomplishes this.
2548 */
2549 if (thread_change_pri(t, new_pri, 0)) {
2550 if ((t->t_schedflag & TS_LOAD) &&
2551 (lwp = t->t_lwp) &&
2552 lwp->lwp_state == LWP_USER)
2553 t->t_schedflag &= ~TS_DONT_SWAP;
2554 fssproc->fss_timeleft = fss_quantum;
2555 } else {
2556 call_cpu_surrender = B_TRUE;
2557 }
2558 } else if (t->t_state == TS_ONPROC &&
2559 t->t_pri < t->t_disp_queue->disp_maxrunpri) {
2560 /*
2561 * If there is a higher-priority thread which is
2562 * waiting for a processor, then thread surrenders
2563 * the processor.
2564 */
2565 call_cpu_surrender = B_TRUE;
2566 }
2567 }
2568
2569 if (cpucaps_enforce && 2 * fssproc->fss_timeleft > fss_quantum) {
2570 /*
2571 * The thread used more than half of its quantum, so assume that
2572 * it used the whole quantum.
2573 *
2597 thread_unlock_nopreempt(t); /* clock thread can't be preempted */
2598 }
2599
2600 /*
2601 * Processes waking up go to the back of their queue. We don't need to assign
2602 * a time quantum here because thread is still at a kernel mode priority and
2603 * the time slicing is not done for threads running in the kernel after
2604 * sleeping. The proper time quantum will be assigned by fss_trapret before the
2605 * thread returns to user mode.
2606 */
2607 static void
2608 fss_wakeup(kthread_t *t)
2609 {
2610 fssproc_t *fssproc;
2611
2612 ASSERT(THREAD_LOCK_HELD(t));
2613 ASSERT(t->t_state == TS_SLEEP);
2614
2615 fss_active(t);
2616
2617 t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */
2618 fssproc = FSSPROC(t);
2619 fssproc->fss_flags &= ~FSSBACKQ;
2620
2621 if (fssproc->fss_flags & FSSKPRI) {
2622 /*
2623 * If we already have a kernel priority assigned, then we
2624 * just use it.
2625 */
2626 setbackdq(t);
2627 } else if (t->t_kpri_req) {
2628 /*
2629 * Give thread a priority boost if we were asked.
2630 */
2631 fssproc->fss_flags |= FSSKPRI;
2632 THREAD_CHANGE_PRI(t, minclsyspri);
2633 setbackdq(t);
2634 t->t_trapret = 1; /* so that fss_trapret will run */
2635 aston(t);
2636 } else {
2637 /*
|
332 #define FSS_NICE_MIN 0
333 #define FSS_NICE_MAX (2 * NZERO - 1)
334 #define FSS_NICE_RANGE (FSS_NICE_MAX - FSS_NICE_MIN + 1)
335
336 static int fss_nice_tick[FSS_NICE_RANGE];
337 static int fss_nice_decay[FSS_NICE_RANGE];
338
339 static pri_t fss_maxupri = FSS_MAXUPRI; /* maximum FSS user priority */
340 static pri_t fss_maxumdpri; /* maximum user mode fss priority */
341 static pri_t fss_maxglobpri; /* maximum global priority used by fss class */
342 static pri_t fss_minglobpri; /* minimum global priority */
343
344 static fssproc_t fss_listhead[FSS_LISTS];
345 static kmutex_t fss_listlock[FSS_LISTS];
346
347 static fsspset_t *fsspsets;
348 static kmutex_t fsspsets_lock; /* protects fsspsets */
349
350 static id_t fss_cid;
351
352 static int fss_quantum = 11;
353
354 static void fss_newpri(fssproc_t *, boolean_t);
355 static void fss_update(void *);
356 static int fss_update_list(int);
357 static void fss_change_priority(kthread_t *, fssproc_t *);
358
359 static int fss_admin(caddr_t, cred_t *);
360 static int fss_getclinfo(void *);
361 static int fss_parmsin(void *);
362 static int fss_parmsout(void *, pc_vaparms_t *);
363 static int fss_vaparmsin(void *, pc_vaparms_t *);
364 static int fss_vaparmsout(void *, pc_vaparms_t *);
365 static int fss_getclpri(pcpri_t *);
366 static int fss_alloc(void **, int);
367 static void fss_free(void *);
368
369 static int fss_enterclass(kthread_t *, id_t, void *, cred_t *, void *);
370 static void fss_exitclass(void *);
371 static int fss_canexit(kthread_t *, cred_t *);
372 static int fss_fork(kthread_t *, kthread_t *, void *);
373 static void fss_forkret(kthread_t *, kthread_t *);
374 static void fss_parmsget(kthread_t *, void *);
375 static int fss_parmsset(kthread_t *, void *, id_t, cred_t *);
376 static void fss_stop(kthread_t *, int, int);
377 static void fss_exit(kthread_t *);
378 static void fss_active(kthread_t *);
379 static void fss_inactive(kthread_t *);
380 static void fss_trapret(kthread_t *);
381 static void fss_preempt(kthread_t *);
382 static void fss_setrun(kthread_t *);
383 static void fss_sleep(kthread_t *);
384 static void fss_tick(kthread_t *);
385 static void fss_wakeup(kthread_t *);
386 static int fss_donice(kthread_t *, cred_t *, int, int *);
387 static int fss_doprio(kthread_t *, cred_t *, int, int *);
388 static pri_t fss_globpri(kthread_t *);
389 static void fss_yield(kthread_t *);
390 static void fss_nullsys();
391
392 static struct classfuncs fss_classfuncs = {
393 /* class functions */
394 fss_admin,
395 fss_getclinfo,
396 fss_parmsin,
397 fss_parmsout,
398 fss_vaparmsin,
399 fss_vaparmsout,
400 fss_getclpri,
401 fss_alloc,
402 fss_free,
403
404 /* thread functions */
405 fss_enterclass,
406 fss_exitclass,
407 fss_canexit,
408 fss_fork,
409 fss_forkret,
410 fss_parmsget,
411 fss_parmsset,
412 fss_stop,
413 fss_exit,
414 fss_active,
415 fss_inactive,
416 fss_trapret,
417 fss_preempt,
418 fss_setrun,
419 fss_sleep,
420 fss_tick,
421 fss_wakeup,
422 fss_donice,
423 fss_globpri,
424 fss_nullsys, /* set_process_group */
425 fss_yield,
426 fss_doprio,
427 };
428
429 int
430 _init()
431 {
432 return (mod_install(&modlinkage));
433 }
434
435 int
2117 * calculate how much CPU time it used since it was charged last time.
2118 *
2119 * CPU caps are not enforced on exiting processes - it is usually
2120 * desirable to exit as soon as possible to free resources.
2121 */
2122 if (CPUCAPS_ON()) {
2123 thread_lock(t);
2124 fssproc = FSSPROC(t);
2125 (void) cpucaps_charge(t, &fssproc->fss_caps,
2126 CPUCAPS_CHARGE_ONLY);
2127 thread_unlock(t);
2128 }
2129 }
2130
2131 static void
2132 fss_nullsys()
2133 {
2134 }
2135
2136 /*
2137 * If thread is currently at a kernel mode priority (has slept) and is
2138 * returning to the userland we assign it the appropriate user mode priority
2139 * and time quantum here. If we're lowering the thread's priority below that
2140 * of other runnable threads then we will set runrun via cpu_surrender() to
2141 * cause preemption.
2142 */
2143 static void
2144 fss_trapret(kthread_t *t)
2145 {
2146 fssproc_t *fssproc = FSSPROC(t);
2147 cpu_t *cp = CPU;
2148
2149 ASSERT(THREAD_LOCK_HELD(t));
2150 ASSERT(t == curthread);
2151 ASSERT(cp->cpu_dispthread == t);
2152 ASSERT(t->t_state == TS_ONPROC);
2153
2154 t->t_kpri_req = 0;
2155 if (fssproc->fss_flags & FSSKPRI) {
2156 /*
2157 * If thread has blocked in the kernel
2158 */
2159 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2160 cp->cpu_dispatch_pri = DISP_PRIO(t);
2161 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
2162 fssproc->fss_flags &= ~FSSKPRI;
2163
2164 if (DISP_MUST_SURRENDER(t))
2165 cpu_surrender(t);
2166 }
2167 }
2168
2169 /*
2170 * Arrange for thread to be placed in appropriate location on dispatcher queue.
2171 * This is called with the current thread in TS_ONPROC and locked.
2172 */
2173 static void
2174 fss_preempt(kthread_t *t)
2175 {
2176 fssproc_t *fssproc = FSSPROC(t);
2177 klwp_t *lwp;
2178 uint_t flags;
2179
2180 ASSERT(t == curthread);
2181 ASSERT(THREAD_LOCK_HELD(curthread));
2182 ASSERT(t->t_state == TS_ONPROC);
2183
2184 /*
2185 * If preempted in the kernel, make sure the thread has a kernel
2186 * priority if needed.
2191 THREAD_CHANGE_PRI(t, minclsyspri);
2192 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
2193 t->t_trapret = 1; /* so that fss_trapret will run */
2194 aston(t);
2195 }
2196
2197 /*
2198 * This thread may be placed on wait queue by CPU Caps. In this case we
2199 * do not need to do anything until it is removed from the wait queue.
2200 * Do not enforce CPU caps on threads running at a kernel priority
2201 */
2202 if (CPUCAPS_ON()) {
2203 (void) cpucaps_charge(t, &fssproc->fss_caps,
2204 CPUCAPS_CHARGE_ENFORCE);
2205
2206 if (!(fssproc->fss_flags & FSSKPRI) && CPUCAPS_ENFORCE(t))
2207 return;
2208 }
2209
2210 /*
2211 * Check to see if we're doing "preemption control" here. If
2212 * we are, and if the user has requested that this thread not
2213 * be preempted, and if preemptions haven't been put off for
2214 * too long, let the preemption happen here but try to make
2215 * sure the thread is rescheduled as soon as possible. We do
2216 * this by putting it on the front of the highest priority run
2217 * queue in the FSS class. If the preemption has been put off
2218 * for too long, clear the "nopreempt" bit and let the thread
2219 * be preempted.
2220 */
2221 if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2222 if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2223 DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);
2224 if (!(fssproc->fss_flags & FSSKPRI)) {
2225 /*
2226 * If not already remembered, remember current
2227 * priority for restoration in fss_yield().
2228 */
2229 if (!(fssproc->fss_flags & FSSRESTORE)) {
2230 fssproc->fss_scpri = t->t_pri;
2231 fssproc->fss_flags |= FSSRESTORE;
2232 }
2233 THREAD_CHANGE_PRI(t, fss_maxumdpri);
2234 }
2235 schedctl_set_yield(t, 1);
2236 setfrontdq(t);
2237 return;
2238 } else {
2239 if (fssproc->fss_flags & FSSRESTORE) {
2240 THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
2241 fssproc->fss_flags &= ~FSSRESTORE;
2242 }
2243 schedctl_set_nopreempt(t, 0);
2244 DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
2245 /*
2246 * Fall through and be preempted below.
2247 */
2248 }
2249 }
2250
2251 flags = fssproc->fss_flags & (FSSBACKQ | FSSKPRI);
2252
2253 if (flags == FSSBACKQ) {
2319 * for trapret processing as the thread leaves the system call so it
2320 * will drop back to normal priority range.
2321 */
2322 if (t->t_kpri_req) {
2323 THREAD_CHANGE_PRI(t, minclsyspri);
2324 fssproc->fss_flags |= FSSKPRI;
2325 t->t_trapret = 1; /* so that fss_trapret will run */
2326 aston(t);
2327 } else if (fssproc->fss_flags & FSSKPRI) {
2328 /*
2329 * The thread has done a THREAD_KPRI_REQUEST(), slept, then
2330 * done THREAD_KPRI_RELEASE() (so no t_kpri_req is 0 again),
2331 * then slept again all without finishing the current system
2332 * call so trapret won't have cleared FSSKPRI
2333 */
2334 fssproc->fss_flags &= ~FSSKPRI;
2335 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2336 if (DISP_MUST_SURRENDER(curthread))
2337 cpu_surrender(t);
2338 }
2339 }
2340
2341 /*
2342 * A tick interrupt has ocurrend on a running thread. Check to see if our
2343 * time slice has expired.
2344 */
2345 static void
2346 fss_tick(kthread_t *t)
2347 {
2348 fssproc_t *fssproc;
2349 fssproj_t *fssproj;
2350 boolean_t call_cpu_surrender = B_FALSE;
2351 boolean_t cpucaps_enforce = B_FALSE;
2352
2353 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
2354
2355 /*
2356 * It's safe to access fsspset and fssproj structures because we're
2357 * holding our p_lock here.
2358 */
2359 thread_lock(t);
2360 fssproc = FSSPROC(t);
2361 fssproj = FSSPROC2FSSPROJ(fssproc);
2362 if (fssproj != NULL) {
2363 fsspset_t *fsspset = FSSPROJ2FSSPSET(fssproj);
2364 disp_lock_enter_high(&fsspset->fssps_displock);
2365 fssproj->fssp_ticks += fss_nice_tick[fssproc->fss_nice];
2366 fssproj->fssp_tick_cnt++;
2367 fssproc->fss_ticks++;
2368 disp_lock_exit_high(&fsspset->fssps_displock);
2369 }
2401 DTRACE_SCHED1(schedctl__nopreempt,
2402 kthread_t *, t);
2403 schedctl_set_yield(t, 1);
2404 thread_unlock_nopreempt(t);
2405 return;
2406 }
2407 }
2408 fssproc->fss_flags &= ~FSSRESTORE;
2409
2410 fss_newpri(fssproc, B_TRUE);
2411 new_pri = fssproc->fss_umdpri;
2412 ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
2413
2414 /*
2415 * When the priority of a thread is changed, it may
2416 * be necessary to adjust its position on a sleep queue
2417 * or dispatch queue. The function thread_change_pri
2418 * accomplishes this.
2419 */
2420 if (thread_change_pri(t, new_pri, 0)) {
2421 fssproc->fss_timeleft = fss_quantum;
2422 } else {
2423 call_cpu_surrender = B_TRUE;
2424 }
2425 } else if (t->t_state == TS_ONPROC &&
2426 t->t_pri < t->t_disp_queue->disp_maxrunpri) {
2427 /*
2428 * If there is a higher-priority thread which is
2429 * waiting for a processor, then thread surrenders
2430 * the processor.
2431 */
2432 call_cpu_surrender = B_TRUE;
2433 }
2434 }
2435
2436 if (cpucaps_enforce && 2 * fssproc->fss_timeleft > fss_quantum) {
2437 /*
2438 * The thread used more than half of its quantum, so assume that
2439 * it used the whole quantum.
2440 *
2464 thread_unlock_nopreempt(t); /* clock thread can't be preempted */
2465 }
2466
2467 /*
2468 * Processes waking up go to the back of their queue. We don't need to assign
2469 * a time quantum here because thread is still at a kernel mode priority and
2470 * the time slicing is not done for threads running in the kernel after
2471 * sleeping. The proper time quantum will be assigned by fss_trapret before the
2472 * thread returns to user mode.
2473 */
2474 static void
2475 fss_wakeup(kthread_t *t)
2476 {
2477 fssproc_t *fssproc;
2478
2479 ASSERT(THREAD_LOCK_HELD(t));
2480 ASSERT(t->t_state == TS_SLEEP);
2481
2482 fss_active(t);
2483
2484 fssproc = FSSPROC(t);
2485 fssproc->fss_flags &= ~FSSBACKQ;
2486
2487 if (fssproc->fss_flags & FSSKPRI) {
2488 /*
2489 * If we already have a kernel priority assigned, then we
2490 * just use it.
2491 */
2492 setbackdq(t);
2493 } else if (t->t_kpri_req) {
2494 /*
2495 * Give thread a priority boost if we were asked.
2496 */
2497 fssproc->fss_flags |= FSSKPRI;
2498 THREAD_CHANGE_PRI(t, minclsyspri);
2499 setbackdq(t);
2500 t->t_trapret = 1; /* so that fss_trapret will run */
2501 aston(t);
2502 } else {
2503 /*
|