Print this page
6583 remove whole-process swapping
| Split |
Close |
| Expand all |
| Collapse all |
--- old/usr/src/uts/common/disp/fss.c
+++ new/usr/src/uts/common/disp/fss.c
1 1 /*
2 2 * CDDL HEADER START
3 3 *
4 4 * The contents of this file are subject to the terms of the
5 5 * Common Development and Distribution License (the "License").
6 6 * You may not use this file except in compliance with the License.
7 7 *
8 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 9 * or http://www.opensolaris.org/os/licensing.
10 10 * See the License for the specific language governing permissions
11 11 * and limitations under the License.
12 12 *
13 13 * When distributing Covered Code, include this CDDL HEADER in each
14 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 15 * If applicable, add the following below this CDDL HEADER, with the
16 16 * fields enclosed by brackets "[]" replaced with your own identifying
17 17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 18 *
19 19 * CDDL HEADER END
20 20 */
21 21
22 22 /*
23 23 * Copyright (c) 1994, 2010, Oracle and/or its affiliates. All rights reserved.
24 24 * Copyright 2013, Joyent, Inc. All rights reserved.
25 25 */
26 26
27 27 #include <sys/types.h>
28 28 #include <sys/param.h>
29 29 #include <sys/sysmacros.h>
30 30 #include <sys/cred.h>
31 31 #include <sys/proc.h>
32 32 #include <sys/strsubr.h>
33 33 #include <sys/priocntl.h>
34 34 #include <sys/class.h>
35 35 #include <sys/disp.h>
36 36 #include <sys/procset.h>
37 37 #include <sys/debug.h>
38 38 #include <sys/kmem.h>
39 39 #include <sys/errno.h>
40 40 #include <sys/systm.h>
41 41 #include <sys/schedctl.h>
42 42 #include <sys/vmsystm.h>
43 43 #include <sys/atomic.h>
44 44 #include <sys/project.h>
45 45 #include <sys/modctl.h>
46 46 #include <sys/fss.h>
47 47 #include <sys/fsspriocntl.h>
48 48 #include <sys/cpupart.h>
49 49 #include <sys/zone.h>
50 50 #include <vm/rm.h>
51 51 #include <vm/seg_kmem.h>
52 52 #include <sys/tnf_probe.h>
53 53 #include <sys/policy.h>
54 54 #include <sys/sdt.h>
55 55 #include <sys/cpucaps.h>
56 56
57 57 /*
58 58 * The fair share scheduling class ensures that collections of processes
59 59 * (zones and projects) each get their configured share of CPU. This is in
60 60 * contrast to the TS class which considers individual processes.
61 61 *
62 62 * The FSS cpu-share is set on zones using the zone.cpu-shares rctl and on
63 63 * projects using the project.cpu-shares rctl. By default the value is 1
64 64 * and it can range from 0 - 64k. A value of 0 means that processes in the
65 65 * collection will only get CPU resources when there are no other processes
66 66 * that need CPU. The cpu-share is used as one of the inputs to calculate a
67 67 * thread's "user-mode" priority (umdpri) for the scheduler. The umdpri falls
68 68 * in the range 0-59. FSS calculates other, internal, priorities which are not
69 69 * visible outside of the FSS class.
70 70 *
71 71 * The FSS class should approximate TS behavior when there are excess CPU
72 72 * resources. When there is a backlog of runnable processes, then the share
73 73 * is used as input into the runnable process's priority calculation, where
74 74 * the final umdpri is used by the scheduler to determine when the process runs.
75 75 *
76 76 * Projects in a zone compete with each other for CPU time, receiving CPU
77 77 * allocation within a zone proportional to the project's share; at a higher
78 78 * level zones compete with each other, receiving allocation in a pset
79 79 * proportional to the zone's share.
80 80 *
81 81 * The FSS priority calculation consists of several parts.
82 82 *
83 83 * 1) Once per second the fss_update function runs. The first thing it does is
84 84 * call fss_decay_usage. This function does three things.
85 85 *
86 86 * a) fss_decay_usage first decays the maxfsspri value for the pset. This
87 87 * value is used in the per-process priority calculation described in step
88 88 * (2b). The maxfsspri is decayed using the following formula:
89 89 *
90 90 * maxfsspri * fss_nice_decay[NZERO])
91 91 * maxfsspri = ------------------------------------
92 92 * FSS_DECAY_BASE
93 93 *
94 94 *
95 95 * - NZERO is the default process priority (i.e. 20)
96 96 *
97 97 * The fss_nice_decay array is a fixed set of values used to adjust the
98 98 * decay rate of processes based on their nice value. Entries in this
99 99 * array are initialized in fss_init using the following formula:
100 100 *
101 101 * (FSS_DECAY_MAX - FSS_DECAY_MIN) * i
102 102 * FSS_DECAY_MIN + -------------------------------------
103 103 * FSS_NICE_RANGE - 1
104 104 *
105 105 * - FSS_DECAY_MIN is 82 = approximates 65% (82/128)
106 106 * - FSS_DECAY_MAX is 108 = approximates 85% (108/128)
107 107 * - FSS_NICE_RANGE is 40 (range is 0 - 39)
108 108 *
109 109 * b) The second thing fss_decay_usage does is update each project's "usage"
110 110 * for the last second and then recalculates the project's "share usage".
111 111 *
112 112 * The usage value is the recent CPU usage for all of the threads in the
113 113 * project. It is decayed and updated this way:
114 114 *
115 115 * (usage * FSS_DECAY_USG)
116 116 * usage = ------------------------- + ticks;
117 117 * FSS_DECAY_BASE
118 118 *
119 119 * - FSS_DECAY_BASE is 128 - used instead of 100 so we can shift vs divide
120 120 * - FSS_DECAY_USG is 96 - approximates 75% (96/128)
121 121 * - ticks is updated whenever a process in this project is running
122 122 * when the scheduler's tick processing fires. This is not a simple
123 123 * counter, the values are based on the entries in the fss_nice_tick
124 124 * array (see section 3 below). ticks is then reset to 0 so it can track
125 125 * the next seconds worth of nice-adjusted time for the project.
126 126 *
127 127 * c) The third thing fss_decay_usage does is update each project's "share
128 128 * usage" (shusage). This is the normalized usage value for the project and
129 129 * is calculated this way:
130 130 *
131 131 * pset_shares^2 zone_int_shares^2
132 132 * usage * ------------- * ------------------
133 133 * kpj_shares^2 zone_ext_shares^2
134 134 *
135 135 * - usage - see (1b) for more details
136 136 * - pset_shares is the total of all *active* zone shares in the pset (by
137 137 * default there is only one pset)
138 138 * - kpj_shares is the individual project's share (project.cpu-shares rctl)
139 139 * - zone_int_shares is the sum of shares of all active projects within the
140 140 * zone (the zone-internal total)
141 141 * - zone_ext_shares is the share value for the zone (zone.cpu-shares rctl)
142 142 *
143 143 * The shusage is used in step (2b) to calculate the thread's new internal
144 144 * priority. A larger shusage value leads to a lower priority.
145 145 *
146 146 * 2) The fss_update function then calls fss_update_list to update the priority
147 147 * of all threads. This does two things.
148 148 *
149 149 * a) First the thread's internal priority is decayed using the following
150 150 * formula:
151 151 *
152 152 * fsspri * fss_nice_decay[nice_value])
153 153 * fsspri = ------------------------------------
154 154 * FSS_DECAY_BASE
155 155 *
156 156 * - FSS_DECAY_BASE is 128 as described above
157 157 *
158 158 * b) Second, if the thread is runnable (TS_RUN or TS_WAIT) calls fss_newpri
159 159 * to update the user-mode priority (umdpri) of the runnable thread.
160 160 * Threads that are running (TS_ONPROC) or waiting for an event (TS_SLEEP)
161 161 * are not updated at this time. The updated user-mode priority can cause
162 162 * threads to change their position in the run queue.
163 163 *
164 164 * The process's new internal fsspri is calculated using the following
165 165 * formula. All runnable threads in the project will use the same shusage
166 166 * and nrunnable values in their calculation.
167 167 *
168 168 * fsspri += shusage * nrunnable * ticks
169 169 *
170 170 * - shusage is the project's share usage, calculated in (1c)
171 171 * - nrunnable is the number of runnable threads in the project
172 172 * - ticks is the number of ticks this thread ran since the last fss_newpri
173 173 * invocation.
174 174 *
175 175 * Finally the process's new user-mode priority is calculated using the
176 176 * following formula:
177 177 *
178 178 * (fsspri * umdprirange)
179 179 * umdpri = maxumdpri - ------------------------
180 180 * maxfsspri
181 181 *
182 182 * - maxumdpri is MINCLSYSPRI - 1 (i.e. 59)
183 183 * - umdprirange is maxumdpri - 1 (i.e. 58)
184 184 * - maxfsspri is the largest fsspri seen so far, as we're iterating all
185 185 * runnable processes
186 186 *
187 187 * Thus, a higher internal priority (fsspri) leads to a lower user-mode
188 188 * priority which means the thread runs less. The fsspri is higher when
189 189 * the project's normalized share usage is higher, when the project has
190 190 * more runnable threads, or when the thread has accumulated more run-time.
191 191 *
192 192 * This code has various checks to ensure the resulting umdpri is in the
193 193 * range 1-59. See fss_newpri for more details.
194 194 *
195 195 * To reiterate, the above processing is performed once per second to recompute
196 196 * the runnable thread user-mode priorities.
197 197 *
198 198 * 3) The final major component in the priority calculation is the tick
199 199 * processing which occurs on a thread that is running when the clock
200 200 * calls fss_tick.
201 201 *
202 202 * A thread can run continuously in user-land (compute-bound) for the
203 203 * fss_quantum (see "dispadmin -c FSS -g" for the configurable properties).
204 204 * The fss_quantum defaults to 11 (i.e. 11 ticks).
205 205 *
206 206 * Once the quantum has been consumed, the thread will call fss_newpri to
207 207 * recompute its umdpri priority, as described above in (2b). Threads that
208 208 * were T_ONPROC at the one second interval when runnable thread priorities
209 209 * were recalculated will have their umdpri priority recalculated when their
210 210 * quanta expires.
211 211 *
212 212 * To ensure that runnable threads within a project see the expected
213 213 * round-robin behavior, there is a special case in fss_newpri for a thread
214 214 * that has run for its quanta within the one second update interval. See
215 215 * the handling for the quanta_up parameter within fss_newpri.
216 216 *
217 217 * Also of interest, the fss_tick code increments the project's tick value
218 218 * using the fss_nice_tick array entry for the thread's nice value. The idea
219 219 * behind the fss_nice_tick array is that the cost of a tick is lower at
220 220 * positive nice values (so that it doesn't increase the project's usage
221 221 * as much as normal) with a 50% drop at the maximum level and a 50%
222 222 * increase at the minimum level. See (1b). The fss_nice_tick array is
223 223 * initialized in fss_init using the following formula:
224 224 *
225 225 * FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2) - i)
226 226 * --------------------------------------------------
227 227 * FSS_NICE_RANGE
228 228 *
229 229 * - FSS_TICK_COST is 1000, the tick cost for threads with nice level 0
230 230 *
231 231 * FSS Data Structures:
232 232 *
233 233 * fsszone
234 234 * ----- -----
235 235 * ----- | | | |
236 236 * | |-------->| |<------->| |<---->...
237 237 * | | ----- -----
238 238 * | | ^ ^ ^
239 239 * | |--- | \ \
240 240 * ----- | | \ \
241 241 * fsspset | | \ \
242 242 * | | \ \
243 243 * | ----- ----- -----
244 244 * -->| |<--->| |<--->| |
245 245 * | | | | | |
246 246 * ----- ----- -----
247 247 * fssproj
248 248 *
249 249 * That is, fsspsets contain a list of fsszone's that are currently active in
250 250 * the pset, and a list of fssproj's, corresponding to projects with runnable
251 251 * threads on the pset. fssproj's in turn point to the fsszone which they
252 252 * are a member of.
253 253 *
254 254 * An fssproj_t is removed when there are no threads in it.
255 255 *
256 256 * An fsszone_t is removed when there are no projects with threads in it.
257 257 */
258 258
259 259 static pri_t fss_init(id_t, int, classfuncs_t **);
260 260
261 261 static struct sclass fss = {
262 262 "FSS",
263 263 fss_init,
264 264 0
265 265 };
266 266
267 267 extern struct mod_ops mod_schedops;
268 268
269 269 /*
270 270 * Module linkage information for the kernel.
271 271 */
272 272 static struct modlsched modlsched = {
273 273 &mod_schedops, "fair share scheduling class", &fss
274 274 };
275 275
276 276 static struct modlinkage modlinkage = {
277 277 MODREV_1, (void *)&modlsched, NULL
278 278 };
279 279
280 280 #define FSS_MAXUPRI 60
281 281
282 282 /*
283 283 * The fssproc_t structures are kept in an array of circular doubly linked
284 284 * lists. A hash on the thread pointer is used to determine which list each
285 285 * thread should be placed in. Each list has a dummy "head" which is never
286 286 * removed, so the list is never empty. fss_update traverses these lists to
287 287 * update the priorities of threads that have been waiting on the run queue.
288 288 */
289 289 #define FSS_LISTS 16 /* number of lists, must be power of 2 */
290 290 #define FSS_LIST_HASH(t) (((uintptr_t)(t) >> 9) & (FSS_LISTS - 1))
291 291 #define FSS_LIST_NEXT(i) (((i) + 1) & (FSS_LISTS - 1))
292 292
293 293 #define FSS_LIST_INSERT(fssproc) \
294 294 { \
295 295 int index = FSS_LIST_HASH(fssproc->fss_tp); \
296 296 kmutex_t *lockp = &fss_listlock[index]; \
297 297 fssproc_t *headp = &fss_listhead[index]; \
298 298 mutex_enter(lockp); \
299 299 fssproc->fss_next = headp->fss_next; \
300 300 fssproc->fss_prev = headp; \
301 301 headp->fss_next->fss_prev = fssproc; \
302 302 headp->fss_next = fssproc; \
303 303 mutex_exit(lockp); \
304 304 }
305 305
306 306 #define FSS_LIST_DELETE(fssproc) \
307 307 { \
308 308 int index = FSS_LIST_HASH(fssproc->fss_tp); \
309 309 kmutex_t *lockp = &fss_listlock[index]; \
310 310 mutex_enter(lockp); \
311 311 fssproc->fss_prev->fss_next = fssproc->fss_next; \
312 312 fssproc->fss_next->fss_prev = fssproc->fss_prev; \
313 313 mutex_exit(lockp); \
314 314 }
315 315
316 316 #define FSS_TICK_COST 1000 /* tick cost for threads with nice level = 0 */
317 317
318 318 /*
319 319 * Decay rate percentages are based on n/128 rather than n/100 so that
320 320 * calculations can avoid having to do an integer divide by 100 (divide
321 321 * by FSS_DECAY_BASE == 128 optimizes to an arithmetic shift).
322 322 *
323 323 * FSS_DECAY_MIN = 83/128 ~= 65%
324 324 * FSS_DECAY_MAX = 108/128 ~= 85%
325 325 * FSS_DECAY_USG = 96/128 ~= 75%
326 326 */
327 327 #define FSS_DECAY_MIN 83 /* fsspri decay pct for threads w/ nice -20 */
328 328 #define FSS_DECAY_MAX 108 /* fsspri decay pct for threads w/ nice +19 */
329 329 #define FSS_DECAY_USG 96 /* fssusage decay pct for projects */
330 330 #define FSS_DECAY_BASE 128 /* base for decay percentages above */
331 331
332 332 #define FSS_NICE_MIN 0
333 333 #define FSS_NICE_MAX (2 * NZERO - 1)
334 334 #define FSS_NICE_RANGE (FSS_NICE_MAX - FSS_NICE_MIN + 1)
335 335
336 336 static int fss_nice_tick[FSS_NICE_RANGE];
337 337 static int fss_nice_decay[FSS_NICE_RANGE];
338 338
339 339 static pri_t fss_maxupri = FSS_MAXUPRI; /* maximum FSS user priority */
340 340 static pri_t fss_maxumdpri; /* maximum user mode fss priority */
341 341 static pri_t fss_maxglobpri; /* maximum global priority used by fss class */
|
↓ open down ↓ |
341 lines elided |
↑ open up ↑ |
342 342 static pri_t fss_minglobpri; /* minimum global priority */
343 343
344 344 static fssproc_t fss_listhead[FSS_LISTS];
345 345 static kmutex_t fss_listlock[FSS_LISTS];
346 346
347 347 static fsspset_t *fsspsets;
348 348 static kmutex_t fsspsets_lock; /* protects fsspsets */
349 349
350 350 static id_t fss_cid;
351 351
352 -static time_t fss_minrun = 2; /* t_pri becomes 59 within 2 secs */
353 -static time_t fss_minslp = 2; /* min time on sleep queue for hardswap */
354 352 static int fss_quantum = 11;
355 353
356 354 static void fss_newpri(fssproc_t *, boolean_t);
357 355 static void fss_update(void *);
358 356 static int fss_update_list(int);
359 357 static void fss_change_priority(kthread_t *, fssproc_t *);
360 358
361 359 static int fss_admin(caddr_t, cred_t *);
362 360 static int fss_getclinfo(void *);
363 361 static int fss_parmsin(void *);
364 362 static int fss_parmsout(void *, pc_vaparms_t *);
365 363 static int fss_vaparmsin(void *, pc_vaparms_t *);
366 364 static int fss_vaparmsout(void *, pc_vaparms_t *);
367 365 static int fss_getclpri(pcpri_t *);
368 366 static int fss_alloc(void **, int);
369 367 static void fss_free(void *);
370 368
371 369 static int fss_enterclass(kthread_t *, id_t, void *, cred_t *, void *);
|
↓ open down ↓ |
8 lines elided |
↑ open up ↑ |
372 370 static void fss_exitclass(void *);
373 371 static int fss_canexit(kthread_t *, cred_t *);
374 372 static int fss_fork(kthread_t *, kthread_t *, void *);
375 373 static void fss_forkret(kthread_t *, kthread_t *);
376 374 static void fss_parmsget(kthread_t *, void *);
377 375 static int fss_parmsset(kthread_t *, void *, id_t, cred_t *);
378 376 static void fss_stop(kthread_t *, int, int);
379 377 static void fss_exit(kthread_t *);
380 378 static void fss_active(kthread_t *);
381 379 static void fss_inactive(kthread_t *);
382 -static pri_t fss_swapin(kthread_t *, int);
383 -static pri_t fss_swapout(kthread_t *, int);
384 380 static void fss_trapret(kthread_t *);
385 381 static void fss_preempt(kthread_t *);
386 382 static void fss_setrun(kthread_t *);
387 383 static void fss_sleep(kthread_t *);
388 384 static void fss_tick(kthread_t *);
389 385 static void fss_wakeup(kthread_t *);
390 386 static int fss_donice(kthread_t *, cred_t *, int, int *);
391 387 static int fss_doprio(kthread_t *, cred_t *, int, int *);
392 388 static pri_t fss_globpri(kthread_t *);
393 389 static void fss_yield(kthread_t *);
394 390 static void fss_nullsys();
395 391
396 392 static struct classfuncs fss_classfuncs = {
397 393 /* class functions */
398 394 fss_admin,
399 395 fss_getclinfo,
400 396 fss_parmsin,
401 397 fss_parmsout,
402 398 fss_vaparmsin,
403 399 fss_vaparmsout,
404 400 fss_getclpri,
405 401 fss_alloc,
406 402 fss_free,
407 403
408 404 /* thread functions */
409 405 fss_enterclass,
|
↓ open down ↓ |
16 lines elided |
↑ open up ↑ |
410 406 fss_exitclass,
411 407 fss_canexit,
412 408 fss_fork,
413 409 fss_forkret,
414 410 fss_parmsget,
415 411 fss_parmsset,
416 412 fss_stop,
417 413 fss_exit,
418 414 fss_active,
419 415 fss_inactive,
420 - fss_swapin,
421 - fss_swapout,
422 416 fss_trapret,
423 417 fss_preempt,
424 418 fss_setrun,
425 419 fss_sleep,
426 420 fss_tick,
427 421 fss_wakeup,
428 422 fss_donice,
429 423 fss_globpri,
430 424 fss_nullsys, /* set_process_group */
431 425 fss_yield,
432 426 fss_doprio,
433 427 };
434 428
435 429 int
436 430 _init()
437 431 {
438 432 return (mod_install(&modlinkage));
439 433 }
440 434
441 435 int
442 436 _fini()
443 437 {
444 438 return (EBUSY);
445 439 }
446 440
447 441 int
448 442 _info(struct modinfo *modinfop)
449 443 {
450 444 return (mod_info(&modlinkage, modinfop));
451 445 }
452 446
453 447 /*ARGSUSED*/
454 448 static int
455 449 fss_project_walker(kproject_t *kpj, void *buf)
456 450 {
457 451 return (0);
458 452 }
459 453
460 454 void *
461 455 fss_allocbuf(int op, int type)
462 456 {
463 457 fssbuf_t *fssbuf;
464 458 void **fsslist;
465 459 int cnt;
466 460 int i;
467 461 size_t size;
468 462
469 463 ASSERT(op == FSS_NPSET_BUF || op == FSS_NPROJ_BUF || op == FSS_ONE_BUF);
470 464 ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
471 465 ASSERT(MUTEX_HELD(&cpu_lock));
472 466
473 467 fssbuf = kmem_zalloc(sizeof (fssbuf_t), KM_SLEEP);
474 468 switch (op) {
475 469 case FSS_NPSET_BUF:
476 470 cnt = cpupart_list(NULL, 0, CP_NONEMPTY);
477 471 break;
478 472 case FSS_NPROJ_BUF:
479 473 cnt = project_walk_all(ALL_ZONES, fss_project_walker, NULL);
480 474 break;
481 475 case FSS_ONE_BUF:
482 476 cnt = 1;
483 477 break;
484 478 }
485 479
486 480 switch (type) {
487 481 case FSS_ALLOC_PROJ:
488 482 size = sizeof (fssproj_t);
489 483 break;
490 484 case FSS_ALLOC_ZONE:
491 485 size = sizeof (fsszone_t);
492 486 break;
493 487 }
494 488 fsslist = kmem_zalloc(cnt * sizeof (void *), KM_SLEEP);
495 489 fssbuf->fssb_size = cnt;
496 490 fssbuf->fssb_list = fsslist;
497 491 for (i = 0; i < cnt; i++)
498 492 fsslist[i] = kmem_zalloc(size, KM_SLEEP);
499 493 return (fssbuf);
500 494 }
501 495
502 496 void
503 497 fss_freebuf(fssbuf_t *fssbuf, int type)
504 498 {
505 499 void **fsslist;
506 500 int i;
507 501 size_t size;
508 502
509 503 ASSERT(fssbuf != NULL);
510 504 ASSERT(type == FSS_ALLOC_PROJ || type == FSS_ALLOC_ZONE);
511 505 fsslist = fssbuf->fssb_list;
512 506
513 507 switch (type) {
514 508 case FSS_ALLOC_PROJ:
515 509 size = sizeof (fssproj_t);
516 510 break;
517 511 case FSS_ALLOC_ZONE:
518 512 size = sizeof (fsszone_t);
519 513 break;
520 514 }
521 515
522 516 for (i = 0; i < fssbuf->fssb_size; i++) {
523 517 if (fsslist[i] != NULL)
524 518 kmem_free(fsslist[i], size);
525 519 }
526 520 kmem_free(fsslist, sizeof (void *) * fssbuf->fssb_size);
527 521 kmem_free(fssbuf, sizeof (fssbuf_t));
528 522 }
529 523
530 524 static fsspset_t *
531 525 fss_find_fsspset(cpupart_t *cpupart)
532 526 {
533 527 int i;
534 528 fsspset_t *fsspset = NULL;
535 529 int found = 0;
536 530
537 531 ASSERT(cpupart != NULL);
538 532 ASSERT(MUTEX_HELD(&fsspsets_lock));
539 533
540 534 /*
541 535 * Search for the cpupart pointer in the array of fsspsets.
542 536 */
543 537 for (i = 0; i < max_ncpus; i++) {
544 538 fsspset = &fsspsets[i];
545 539 if (fsspset->fssps_cpupart == cpupart) {
546 540 ASSERT(fsspset->fssps_nproj > 0);
547 541 found = 1;
548 542 break;
549 543 }
550 544 }
551 545 if (found == 0) {
552 546 /*
553 547 * If we didn't find anything, then use the first
554 548 * available slot in the fsspsets array.
555 549 */
556 550 for (i = 0; i < max_ncpus; i++) {
557 551 fsspset = &fsspsets[i];
558 552 if (fsspset->fssps_cpupart == NULL) {
559 553 ASSERT(fsspset->fssps_nproj == 0);
560 554 found = 1;
561 555 break;
562 556 }
563 557 }
564 558 fsspset->fssps_cpupart = cpupart;
565 559 }
566 560 ASSERT(found == 1);
567 561 return (fsspset);
568 562 }
569 563
570 564 static void
571 565 fss_del_fsspset(fsspset_t *fsspset)
572 566 {
573 567 ASSERT(MUTEX_HELD(&fsspsets_lock));
574 568 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
575 569 ASSERT(fsspset->fssps_nproj == 0);
576 570 ASSERT(fsspset->fssps_list == NULL);
577 571 ASSERT(fsspset->fssps_zones == NULL);
578 572 fsspset->fssps_cpupart = NULL;
579 573 fsspset->fssps_maxfsspri = 0;
580 574 fsspset->fssps_shares = 0;
581 575 }
582 576
583 577 /*
584 578 * The following routine returns a pointer to the fsszone structure which
585 579 * belongs to zone "zone" and cpu partition fsspset, if such structure exists.
586 580 */
587 581 static fsszone_t *
588 582 fss_find_fsszone(fsspset_t *fsspset, zone_t *zone)
589 583 {
590 584 fsszone_t *fsszone;
591 585
592 586 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
593 587
594 588 if (fsspset->fssps_list != NULL) {
595 589 /*
596 590 * There are projects/zones active on this cpu partition
597 591 * already. Try to find our zone among them.
598 592 */
599 593 fsszone = fsspset->fssps_zones;
600 594 do {
601 595 if (fsszone->fssz_zone == zone) {
602 596 return (fsszone);
603 597 }
604 598 fsszone = fsszone->fssz_next;
605 599 } while (fsszone != fsspset->fssps_zones);
606 600 }
607 601 return (NULL);
608 602 }
609 603
610 604 /*
611 605 * The following routine links new fsszone structure into doubly linked list of
612 606 * zones active on the specified cpu partition.
613 607 */
614 608 static void
615 609 fss_insert_fsszone(fsspset_t *fsspset, zone_t *zone, fsszone_t *fsszone)
616 610 {
617 611 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
618 612
619 613 fsszone->fssz_zone = zone;
620 614 fsszone->fssz_rshares = zone->zone_shares;
621 615
622 616 if (fsspset->fssps_zones == NULL) {
623 617 /*
624 618 * This will be the first fsszone for this fsspset
625 619 */
626 620 fsszone->fssz_next = fsszone->fssz_prev = fsszone;
627 621 fsspset->fssps_zones = fsszone;
628 622 } else {
629 623 /*
630 624 * Insert this fsszone to the doubly linked list.
631 625 */
632 626 fsszone_t *fssz_head = fsspset->fssps_zones;
633 627
634 628 fsszone->fssz_next = fssz_head;
635 629 fsszone->fssz_prev = fssz_head->fssz_prev;
636 630 fssz_head->fssz_prev->fssz_next = fsszone;
637 631 fssz_head->fssz_prev = fsszone;
638 632 fsspset->fssps_zones = fsszone;
639 633 }
640 634 }
641 635
642 636 /*
643 637 * The following routine removes a single fsszone structure from the doubly
644 638 * linked list of zones active on the specified cpu partition. Note that
645 639 * global fsspsets_lock must be held in case this fsszone structure is the last
646 640 * on the above mentioned list. Also note that the fsszone structure is not
647 641 * freed here, it is the responsibility of the caller to call kmem_free for it.
648 642 */
649 643 static void
650 644 fss_remove_fsszone(fsspset_t *fsspset, fsszone_t *fsszone)
651 645 {
652 646 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
653 647 ASSERT(fsszone->fssz_nproj == 0);
654 648 ASSERT(fsszone->fssz_shares == 0);
655 649 ASSERT(fsszone->fssz_runnable == 0);
656 650
657 651 if (fsszone->fssz_next != fsszone) {
658 652 /*
659 653 * This is not the last zone in the list.
660 654 */
661 655 fsszone->fssz_prev->fssz_next = fsszone->fssz_next;
662 656 fsszone->fssz_next->fssz_prev = fsszone->fssz_prev;
663 657 if (fsspset->fssps_zones == fsszone)
664 658 fsspset->fssps_zones = fsszone->fssz_next;
665 659 } else {
666 660 /*
667 661 * This was the last zone active in this cpu partition.
668 662 */
669 663 fsspset->fssps_zones = NULL;
670 664 }
671 665 }
672 666
673 667 /*
674 668 * The following routine returns a pointer to the fssproj structure
675 669 * which belongs to project kpj and cpu partition fsspset, if such structure
676 670 * exists.
677 671 */
678 672 static fssproj_t *
679 673 fss_find_fssproj(fsspset_t *fsspset, kproject_t *kpj)
680 674 {
681 675 fssproj_t *fssproj;
682 676
683 677 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
684 678
685 679 if (fsspset->fssps_list != NULL) {
686 680 /*
687 681 * There are projects running on this cpu partition already.
688 682 * Try to find our project among them.
689 683 */
690 684 fssproj = fsspset->fssps_list;
691 685 do {
692 686 if (fssproj->fssp_proj == kpj) {
693 687 ASSERT(fssproj->fssp_pset == fsspset);
694 688 return (fssproj);
695 689 }
696 690 fssproj = fssproj->fssp_next;
697 691 } while (fssproj != fsspset->fssps_list);
698 692 }
699 693 return (NULL);
700 694 }
701 695
702 696 /*
703 697 * The following routine links new fssproj structure into doubly linked list
704 698 * of projects running on the specified cpu partition.
705 699 */
706 700 static void
707 701 fss_insert_fssproj(fsspset_t *fsspset, kproject_t *kpj, fsszone_t *fsszone,
708 702 fssproj_t *fssproj)
709 703 {
710 704 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
711 705
712 706 fssproj->fssp_pset = fsspset;
713 707 fssproj->fssp_proj = kpj;
714 708 fssproj->fssp_shares = kpj->kpj_shares;
715 709
716 710 fsspset->fssps_nproj++;
717 711
718 712 if (fsspset->fssps_list == NULL) {
719 713 /*
720 714 * This will be the first fssproj for this fsspset
721 715 */
722 716 fssproj->fssp_next = fssproj->fssp_prev = fssproj;
723 717 fsspset->fssps_list = fssproj;
724 718 } else {
725 719 /*
726 720 * Insert this fssproj to the doubly linked list.
727 721 */
728 722 fssproj_t *fssp_head = fsspset->fssps_list;
729 723
730 724 fssproj->fssp_next = fssp_head;
731 725 fssproj->fssp_prev = fssp_head->fssp_prev;
732 726 fssp_head->fssp_prev->fssp_next = fssproj;
733 727 fssp_head->fssp_prev = fssproj;
734 728 fsspset->fssps_list = fssproj;
735 729 }
736 730 fssproj->fssp_fsszone = fsszone;
737 731 fsszone->fssz_nproj++;
738 732 ASSERT(fsszone->fssz_nproj != 0);
739 733 }
740 734
741 735 /*
742 736 * The following routine removes a single fssproj structure from the doubly
743 737 * linked list of projects running on the specified cpu partition. Note that
744 738 * global fsspsets_lock must be held in case if this fssproj structure is the
745 739 * last on the above mentioned list. Also note that the fssproj structure is
746 740 * not freed here, it is the responsibility of the caller to call kmem_free
747 741 * for it.
748 742 */
749 743 static void
750 744 fss_remove_fssproj(fsspset_t *fsspset, fssproj_t *fssproj)
751 745 {
752 746 fsszone_t *fsszone;
753 747
754 748 ASSERT(MUTEX_HELD(&fsspsets_lock));
755 749 ASSERT(MUTEX_HELD(&fsspset->fssps_lock));
756 750 ASSERT(fssproj->fssp_runnable == 0);
757 751
758 752 fsspset->fssps_nproj--;
759 753
760 754 fsszone = fssproj->fssp_fsszone;
761 755 fsszone->fssz_nproj--;
762 756
763 757 if (fssproj->fssp_next != fssproj) {
764 758 /*
765 759 * This is not the last part in the list.
766 760 */
767 761 fssproj->fssp_prev->fssp_next = fssproj->fssp_next;
768 762 fssproj->fssp_next->fssp_prev = fssproj->fssp_prev;
769 763 if (fsspset->fssps_list == fssproj)
770 764 fsspset->fssps_list = fssproj->fssp_next;
771 765 if (fsszone->fssz_nproj == 0)
772 766 fss_remove_fsszone(fsspset, fsszone);
773 767 } else {
774 768 /*
775 769 * This was the last project part running
776 770 * at this cpu partition.
777 771 */
778 772 fsspset->fssps_list = NULL;
779 773 ASSERT(fsspset->fssps_nproj == 0);
780 774 ASSERT(fsszone->fssz_nproj == 0);
781 775 fss_remove_fsszone(fsspset, fsszone);
782 776 fss_del_fsspset(fsspset);
783 777 }
784 778 }
785 779
786 780 static void
787 781 fss_inactive(kthread_t *t)
788 782 {
789 783 fssproc_t *fssproc;
790 784 fssproj_t *fssproj;
791 785 fsspset_t *fsspset;
792 786 fsszone_t *fsszone;
793 787
794 788 ASSERT(THREAD_LOCK_HELD(t));
795 789 fssproc = FSSPROC(t);
796 790 fssproj = FSSPROC2FSSPROJ(fssproc);
797 791 if (fssproj == NULL) /* if this thread already exited */
798 792 return;
799 793 fsspset = FSSPROJ2FSSPSET(fssproj);
800 794 fsszone = fssproj->fssp_fsszone;
801 795 disp_lock_enter_high(&fsspset->fssps_displock);
802 796 ASSERT(fssproj->fssp_runnable > 0);
803 797 if (--fssproj->fssp_runnable == 0) {
804 798 fsszone->fssz_shares -= fssproj->fssp_shares;
805 799 if (--fsszone->fssz_runnable == 0)
806 800 fsspset->fssps_shares -= fsszone->fssz_rshares;
807 801 }
808 802 ASSERT(fssproc->fss_runnable == 1);
809 803 fssproc->fss_runnable = 0;
810 804 disp_lock_exit_high(&fsspset->fssps_displock);
811 805 }
812 806
813 807 static void
814 808 fss_active(kthread_t *t)
815 809 {
816 810 fssproc_t *fssproc;
817 811 fssproj_t *fssproj;
818 812 fsspset_t *fsspset;
819 813 fsszone_t *fsszone;
820 814
821 815 ASSERT(THREAD_LOCK_HELD(t));
822 816 fssproc = FSSPROC(t);
823 817 fssproj = FSSPROC2FSSPROJ(fssproc);
824 818 if (fssproj == NULL) /* if this thread already exited */
825 819 return;
826 820 fsspset = FSSPROJ2FSSPSET(fssproj);
827 821 fsszone = fssproj->fssp_fsszone;
828 822 disp_lock_enter_high(&fsspset->fssps_displock);
829 823 if (++fssproj->fssp_runnable == 1) {
830 824 fsszone->fssz_shares += fssproj->fssp_shares;
831 825 if (++fsszone->fssz_runnable == 1)
832 826 fsspset->fssps_shares += fsszone->fssz_rshares;
833 827 }
834 828 ASSERT(fssproc->fss_runnable == 0);
835 829 fssproc->fss_runnable = 1;
836 830 disp_lock_exit_high(&fsspset->fssps_displock);
837 831 }
838 832
839 833 /*
840 834 * Fair share scheduler initialization. Called by dispinit() at boot time.
841 835 * We can ignore clparmsz argument since we know that the smallest possible
842 836 * parameter buffer is big enough for us.
843 837 */
844 838 /*ARGSUSED*/
845 839 static pri_t
846 840 fss_init(id_t cid, int clparmsz, classfuncs_t **clfuncspp)
847 841 {
848 842 int i;
849 843
850 844 ASSERT(MUTEX_HELD(&cpu_lock));
851 845
852 846 fss_cid = cid;
853 847 fss_maxumdpri = minclsyspri - 1;
854 848 fss_maxglobpri = minclsyspri;
855 849 fss_minglobpri = 0;
856 850 fsspsets = kmem_zalloc(sizeof (fsspset_t) * max_ncpus, KM_SLEEP);
857 851
858 852 /*
859 853 * Initialize the fssproc hash table.
860 854 */
861 855 for (i = 0; i < FSS_LISTS; i++)
862 856 fss_listhead[i].fss_next = fss_listhead[i].fss_prev =
863 857 &fss_listhead[i];
864 858
865 859 *clfuncspp = &fss_classfuncs;
866 860
867 861 /*
868 862 * Fill in fss_nice_tick and fss_nice_decay arrays:
869 863 * The cost of a tick is lower at positive nice values (so that it
870 864 * will not increase its project's usage as much as normal) with 50%
871 865 * drop at the maximum level and 50% increase at the minimum level.
872 866 * The fsspri decay is slower at positive nice values. fsspri values
873 867 * of processes with negative nice levels must decay faster to receive
874 868 * time slices more frequently than normal.
875 869 */
876 870 for (i = 0; i < FSS_NICE_RANGE; i++) {
877 871 fss_nice_tick[i] = (FSS_TICK_COST * (((3 * FSS_NICE_RANGE) / 2)
878 872 - i)) / FSS_NICE_RANGE;
879 873 fss_nice_decay[i] = FSS_DECAY_MIN +
880 874 ((FSS_DECAY_MAX - FSS_DECAY_MIN) * i) /
881 875 (FSS_NICE_RANGE - 1);
882 876 }
883 877
884 878 return (fss_maxglobpri);
885 879 }
886 880
887 881 /*
888 882 * Calculate the new fss_umdpri based on the usage, the normalized share usage
889 883 * and the number of active threads. Reset the tick counter for this thread.
890 884 *
891 885 * When calculating the new priority using the standard formula we can hit
892 886 * a scenario where we don't have good round-robin behavior. This would be
893 887 * most commonly seen when there is a zone with lots of runnable threads.
894 888 * In the bad scenario we will see the following behavior when using the
895 889 * standard formula and these conditions:
896 890 *
897 891 * - there are multiple runnable threads in the zone (project)
898 892 * - the fssps_maxfsspri is a very large value
899 893 * - (we also know all of these threads will use the project's
900 894 * fssp_shusage)
901 895 *
902 896 * Under these conditions, a thread with a low fss_fsspri value is chosen
903 897 * to run and the thread gets a high fss_umdpri. This thread can run for
904 898 * its full quanta (fss_timeleft) at which time fss_newpri is called to
905 899 * calculate the thread's new priority.
906 900 *
907 901 * In this case, because the newly calculated fsspri value is much smaller
908 902 * (orders of magnitude) than the fssps_maxfsspri value, if we used the
909 903 * standard formula the thread will still get a high fss_umdpri value and
910 904 * will run again for another quanta, even though there are other runnable
911 905 * threads in the project.
912 906 *
913 907 * For a thread that is runnable for a long time, the thread can continue
914 908 * to run for many quanta (totaling many seconds) before the thread's fsspri
915 909 * exceeds the fssps_maxfsspri and the thread's fss_umdpri is reset back
916 910 * down to 1. This behavior also keeps the fssps_maxfsspr at a high value,
917 911 * so that the next runnable thread might repeat this cycle.
918 912 *
919 913 * This leads to the case where we don't have round-robin behavior at quanta
920 914 * granularity, but instead, runnable threads within the project only run
921 915 * at several second intervals.
922 916 *
923 917 * To prevent this scenario from occuring, when a thread has consumed its
924 918 * quanta and there are multiple runnable threads in the project, we
925 919 * immediately cause the thread to hit fssps_maxfsspri so that it gets
926 920 * reset back to 1 and another runnable thread in the project can run.
927 921 */
928 922 static void
929 923 fss_newpri(fssproc_t *fssproc, boolean_t quanta_up)
930 924 {
931 925 kthread_t *tp;
932 926 fssproj_t *fssproj;
933 927 fsspset_t *fsspset;
934 928 fsszone_t *fsszone;
935 929 fsspri_t fsspri, maxfsspri;
936 930 uint32_t n_runnable;
937 931 pri_t invpri;
938 932 uint32_t ticks;
939 933
940 934 tp = fssproc->fss_tp;
941 935 ASSERT(tp != NULL);
942 936
943 937 if (tp->t_cid != fss_cid)
944 938 return;
945 939
946 940 ASSERT(THREAD_LOCK_HELD(tp));
947 941
948 942 fssproj = FSSPROC2FSSPROJ(fssproc);
949 943 fsszone = FSSPROJ2FSSZONE(fssproj);
950 944 if (fssproj == NULL)
951 945 /*
952 946 * No need to change priority of exited threads.
953 947 */
954 948 return;
955 949
956 950 fsspset = FSSPROJ2FSSPSET(fssproj);
957 951 disp_lock_enter_high(&fsspset->fssps_displock);
958 952
959 953 ticks = fssproc->fss_ticks;
960 954 fssproc->fss_ticks = 0;
961 955
962 956 if (fssproj->fssp_shares == 0 || fsszone->fssz_rshares == 0) {
963 957 /*
964 958 * Special case: threads with no shares.
965 959 */
966 960 fssproc->fss_umdpri = fss_minglobpri;
967 961 disp_lock_exit_high(&fsspset->fssps_displock);
968 962 return;
969 963 }
970 964
971 965 maxfsspri = fsspset->fssps_maxfsspri;
972 966 n_runnable = fssproj->fssp_runnable;
973 967
974 968 if (quanta_up && n_runnable > 1) {
975 969 fsspri = maxfsspri;
976 970 } else {
977 971 /*
978 972 * fsspri += fssp_shusage * nrunnable * ticks
979 973 * If all three values are non-0, this typically calculates to
980 974 * a large number (sometimes > 1M, sometimes > 100B) due to
981 975 * fssp_shusage which can be > 1T.
982 976 */
983 977 fsspri = fssproc->fss_fsspri;
984 978 fsspri += fssproj->fssp_shusage * n_runnable * ticks;
985 979 }
986 980
987 981 fssproc->fss_fsspri = fsspri;
988 982
989 983 /*
990 984 * fss_maxumdpri is normally 59, since FSS priorities are 0-59.
991 985 * If the previous calculation resulted in 0 (e.g. was 0 and added 0
992 986 * because ticks == 0), then instead of 0, we use the largest priority,
993 987 * which is still small in comparison to the large numbers we typically
994 988 * see.
995 989 */
996 990 if (fsspri < fss_maxumdpri)
997 991 fsspri = fss_maxumdpri; /* so that maxfsspri is != 0 */
998 992
999 993 /*
1000 994 * The general priority formula:
1001 995 *
1002 996 * (fsspri * umdprirange)
1003 997 * pri = maxumdpri - ------------------------
1004 998 * maxfsspri
1005 999 *
1006 1000 * If this thread's fsspri is greater than the previous largest
1007 1001 * fsspri, then record it as the new high and priority for this
1008 1002 * thread will be one (the lowest priority assigned to a thread
1009 1003 * that has non-zero shares). Because of this check, maxfsspri can
1010 1004 * change as this function is called via the
1011 1005 * fss_update -> fss_update_list -> fss_newpri code path to update
1012 1006 * all runnable threads. See the code in fss_update for how we
1013 1007 * mitigate this issue.
1014 1008 *
1015 1009 * Note that this formula cannot produce out of bounds priority
1016 1010 * values (0-59); if it is changed, additional checks may need to be
1017 1011 * added.
1018 1012 */
1019 1013 if (fsspri >= maxfsspri) {
1020 1014 fsspset->fssps_maxfsspri = fsspri;
1021 1015 disp_lock_exit_high(&fsspset->fssps_displock);
1022 1016 fssproc->fss_umdpri = 1;
1023 1017 } else {
1024 1018 disp_lock_exit_high(&fsspset->fssps_displock);
1025 1019 invpri = (fsspri * (fss_maxumdpri - 1)) / maxfsspri;
1026 1020 fssproc->fss_umdpri = fss_maxumdpri - invpri;
1027 1021 }
1028 1022 }
1029 1023
1030 1024 /*
1031 1025 * Decays usages of all running projects, resets their tick counters and
1032 1026 * calcluates the projects normalized share usage. Called once per second from
1033 1027 * fss_update().
1034 1028 */
1035 1029 static void
1036 1030 fss_decay_usage()
1037 1031 {
1038 1032 uint32_t zone_ext_shares, zone_int_shares;
1039 1033 uint32_t kpj_shares, pset_shares;
1040 1034 fsspset_t *fsspset;
1041 1035 fssproj_t *fssproj;
1042 1036 fsszone_t *fsszone;
1043 1037 fsspri_t maxfsspri;
1044 1038 int psetid;
1045 1039 struct zone *zp;
1046 1040
1047 1041 mutex_enter(&fsspsets_lock);
1048 1042 /*
1049 1043 * Go through all active processor sets and decay usages of projects
1050 1044 * running on them.
1051 1045 */
1052 1046 for (psetid = 0; psetid < max_ncpus; psetid++) {
1053 1047 fsspset = &fsspsets[psetid];
1054 1048 mutex_enter(&fsspset->fssps_lock);
1055 1049
1056 1050 fsspset->fssps_gen++;
1057 1051
1058 1052 if (fsspset->fssps_cpupart == NULL ||
1059 1053 (fssproj = fsspset->fssps_list) == NULL) {
1060 1054 mutex_exit(&fsspset->fssps_lock);
1061 1055 continue;
1062 1056 }
1063 1057
1064 1058 /*
1065 1059 * Decay maxfsspri for this cpu partition with the
1066 1060 * fastest possible decay rate.
1067 1061 */
1068 1062 disp_lock_enter(&fsspset->fssps_displock);
1069 1063
1070 1064 pset_shares = fsspset->fssps_shares;
1071 1065
1072 1066 maxfsspri = (fsspset->fssps_maxfsspri *
1073 1067 fss_nice_decay[NZERO]) / FSS_DECAY_BASE;
1074 1068 if (maxfsspri < fss_maxumdpri)
1075 1069 maxfsspri = fss_maxumdpri;
1076 1070 fsspset->fssps_maxfsspri = maxfsspri;
1077 1071
1078 1072 do {
1079 1073 fsszone = fssproj->fssp_fsszone;
1080 1074 zp = fsszone->fssz_zone;
1081 1075
1082 1076 /*
1083 1077 * Reset zone's FSS stats if they are from a
1084 1078 * previous cycle.
1085 1079 */
1086 1080 if (fsspset->fssps_gen != zp->zone_fss_gen) {
1087 1081 zp->zone_fss_gen = fsspset->fssps_gen;
1088 1082 zp->zone_run_ticks = 0;
1089 1083 }
1090 1084
1091 1085 /*
1092 1086 * Decay project usage, then add in this cycle's
1093 1087 * nice tick value.
1094 1088 */
1095 1089 fssproj->fssp_usage =
1096 1090 (fssproj->fssp_usage * FSS_DECAY_USG) /
1097 1091 FSS_DECAY_BASE +
1098 1092 fssproj->fssp_ticks;
1099 1093
1100 1094 fssproj->fssp_ticks = 0;
1101 1095 zp->zone_run_ticks += fssproj->fssp_tick_cnt;
1102 1096 fssproj->fssp_tick_cnt = 0;
1103 1097
1104 1098 /*
1105 1099 * Readjust the project's number of shares if it has
1106 1100 * changed since we checked it last time.
1107 1101 */
1108 1102 kpj_shares = fssproj->fssp_proj->kpj_shares;
1109 1103 if (fssproj->fssp_shares != kpj_shares) {
1110 1104 if (fssproj->fssp_runnable != 0) {
1111 1105 fsszone->fssz_shares -=
1112 1106 fssproj->fssp_shares;
1113 1107 fsszone->fssz_shares += kpj_shares;
1114 1108 }
1115 1109 fssproj->fssp_shares = kpj_shares;
1116 1110 }
1117 1111
1118 1112 /*
1119 1113 * Readjust the zone's number of shares if it
1120 1114 * has changed since we checked it last time.
1121 1115 */
1122 1116 zone_ext_shares = zp->zone_shares;
1123 1117 if (fsszone->fssz_rshares != zone_ext_shares) {
1124 1118 if (fsszone->fssz_runnable != 0) {
1125 1119 fsspset->fssps_shares -=
1126 1120 fsszone->fssz_rshares;
1127 1121 fsspset->fssps_shares +=
1128 1122 zone_ext_shares;
1129 1123 pset_shares = fsspset->fssps_shares;
1130 1124 }
1131 1125 fsszone->fssz_rshares = zone_ext_shares;
1132 1126 }
1133 1127 zone_int_shares = fsszone->fssz_shares;
1134 1128
1135 1129 /*
1136 1130 * If anything is runnable in the project, track the
1137 1131 * overall project share percent for monitoring useage.
1138 1132 */
1139 1133 if (fssproj->fssp_runnable > 0) {
1140 1134 uint32_t zone_shr_pct;
1141 1135 uint32_t int_shr_pct;
1142 1136
1143 1137 /*
1144 1138 * Times 1000 to get tenths of a percent
1145 1139 *
1146 1140 * zone_ext_shares
1147 1141 * zone_shr_pct = ---------------
1148 1142 * pset_shares
1149 1143 *
1150 1144 * kpj_shares
1151 1145 * int_shr_pct = ---------------
1152 1146 * zone_int_shares
1153 1147 */
1154 1148 if (pset_shares == 0 || zone_int_shares == 0) {
1155 1149 fssproj->fssp_shr_pct = 0;
1156 1150 } else {
1157 1151 zone_shr_pct =
1158 1152 (zone_ext_shares * 1000) /
1159 1153 pset_shares;
1160 1154 int_shr_pct = (kpj_shares * 1000) /
1161 1155 zone_int_shares;
1162 1156 fssproj->fssp_shr_pct =
1163 1157 (zone_shr_pct * int_shr_pct) /
1164 1158 1000;
1165 1159 }
1166 1160 } else {
1167 1161 DTRACE_PROBE1(fss__prj__norun, fssproj_t *,
1168 1162 fssproj);
1169 1163 }
1170 1164
1171 1165 /*
1172 1166 * Calculate fssp_shusage value to be used
1173 1167 * for fsspri increments for the next second.
1174 1168 */
1175 1169 if (kpj_shares == 0 || zone_ext_shares == 0) {
1176 1170 fssproj->fssp_shusage = 0;
1177 1171 } else if (FSSPROJ2KPROJ(fssproj) == proj0p) {
1178 1172 uint32_t zone_shr_pct;
1179 1173
1180 1174 /*
1181 1175 * Project 0 in the global zone has 50%
1182 1176 * of its zone. See calculation above for
1183 1177 * the zone's share percent.
1184 1178 */
1185 1179 if (pset_shares == 0)
1186 1180 zone_shr_pct = 1000;
1187 1181 else
1188 1182 zone_shr_pct =
1189 1183 (zone_ext_shares * 1000) /
1190 1184 pset_shares;
1191 1185
1192 1186 fssproj->fssp_shr_pct = zone_shr_pct / 2;
1193 1187
1194 1188 fssproj->fssp_shusage = (fssproj->fssp_usage *
1195 1189 zone_int_shares * zone_int_shares) /
1196 1190 (zone_ext_shares * zone_ext_shares);
1197 1191 } else {
1198 1192 /*
1199 1193 * Thread's priority is based on its project's
1200 1194 * normalized usage (shusage) value which gets
1201 1195 * calculated this way:
1202 1196 *
1203 1197 * pset_shares^2 zone_int_shares^2
1204 1198 * usage * ------------- * ------------------
1205 1199 * kpj_shares^2 zone_ext_shares^2
1206 1200 *
1207 1201 * Where zone_int_shares is the sum of shares
1208 1202 * of all active projects within the zone (and
1209 1203 * the pset), and zone_ext_shares is the number
1210 1204 * of zone shares (ie, zone.cpu-shares).
1211 1205 *
1212 1206 * If there is only one zone active on the pset
1213 1207 * the above reduces to:
1214 1208 *
1215 1209 * zone_int_shares^2
1216 1210 * shusage = usage * ---------------------
1217 1211 * kpj_shares^2
1218 1212 *
1219 1213 * If there's only one project active in the
1220 1214 * zone this formula reduces to:
1221 1215 *
1222 1216 * pset_shares^2
1223 1217 * shusage = usage * ----------------------
1224 1218 * zone_ext_shares^2
1225 1219 *
1226 1220 * shusage is one input to calculating fss_pri
1227 1221 * in fss_newpri(). Larger values tend toward
1228 1222 * lower priorities for processes in the proj.
1229 1223 */
1230 1224 fssproj->fssp_shusage = fssproj->fssp_usage *
1231 1225 pset_shares * zone_int_shares;
1232 1226 fssproj->fssp_shusage /=
1233 1227 kpj_shares * zone_ext_shares;
1234 1228 fssproj->fssp_shusage *=
1235 1229 pset_shares * zone_int_shares;
1236 1230 fssproj->fssp_shusage /=
1237 1231 kpj_shares * zone_ext_shares;
1238 1232 }
1239 1233 fssproj = fssproj->fssp_next;
1240 1234 } while (fssproj != fsspset->fssps_list);
1241 1235
1242 1236 disp_lock_exit(&fsspset->fssps_displock);
1243 1237 mutex_exit(&fsspset->fssps_lock);
1244 1238 }
1245 1239 mutex_exit(&fsspsets_lock);
1246 1240 }
1247 1241
1248 1242 static void
1249 1243 fss_change_priority(kthread_t *t, fssproc_t *fssproc)
1250 1244 {
1251 1245 pri_t new_pri;
1252 1246
1253 1247 ASSERT(THREAD_LOCK_HELD(t));
1254 1248 new_pri = fssproc->fss_umdpri;
1255 1249 ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
1256 1250
1257 1251 t->t_cpri = fssproc->fss_upri;
1258 1252 fssproc->fss_flags &= ~FSSRESTORE;
1259 1253 if (t == curthread || t->t_state == TS_ONPROC) {
1260 1254 /*
1261 1255 * curthread is always onproc
1262 1256 */
1263 1257 cpu_t *cp = t->t_disp_queue->disp_cpu;
1264 1258 THREAD_CHANGE_PRI(t, new_pri);
1265 1259 if (t == cp->cpu_dispthread)
1266 1260 cp->cpu_dispatch_pri = DISP_PRIO(t);
1267 1261 if (DISP_MUST_SURRENDER(t)) {
1268 1262 fssproc->fss_flags |= FSSBACKQ;
1269 1263 cpu_surrender(t);
1270 1264 } else {
1271 1265 fssproc->fss_timeleft = fss_quantum;
1272 1266 }
1273 1267 } else {
1274 1268 /*
1275 1269 * When the priority of a thread is changed, it may be
1276 1270 * necessary to adjust its position on a sleep queue or
1277 1271 * dispatch queue. The function thread_change_pri accomplishes
1278 1272 * this.
1279 1273 */
1280 1274 if (thread_change_pri(t, new_pri, 0)) {
1281 1275 /*
1282 1276 * The thread was on a run queue.
1283 1277 */
1284 1278 fssproc->fss_timeleft = fss_quantum;
1285 1279 } else {
1286 1280 fssproc->fss_flags |= FSSBACKQ;
1287 1281 }
1288 1282 }
1289 1283 }
1290 1284
1291 1285 /*
1292 1286 * Update priorities of all fair-sharing threads that are currently runnable
1293 1287 * at a user mode priority based on the number of shares and current usage.
1294 1288 * Called once per second via timeout which we reset here.
1295 1289 *
1296 1290 * There are several lists of fair-sharing threads broken up by a hash on the
1297 1291 * thread pointer. Each list has its own lock. This avoids blocking all
1298 1292 * fss_enterclass, fss_fork, and fss_exitclass operations while fss_update runs.
1299 1293 * fss_update traverses each list in turn.
1300 1294 *
1301 1295 * Each time we're run (once/second) we may start at the next list and iterate
1302 1296 * through all of the lists. By starting with a different list, we mitigate any
1303 1297 * effects we would see updating the fssps_maxfsspri value in fss_newpri.
1304 1298 */
1305 1299 static void
1306 1300 fss_update(void *arg)
1307 1301 {
1308 1302 int i;
1309 1303 int new_marker = -1;
1310 1304 static int fss_update_marker;
1311 1305
1312 1306 /*
1313 1307 * Decay and update usages for all projects.
1314 1308 */
1315 1309 fss_decay_usage();
1316 1310
1317 1311 /*
1318 1312 * Start with the fss_update_marker list, then do the rest.
1319 1313 */
1320 1314 i = fss_update_marker;
1321 1315
1322 1316 /*
1323 1317 * Go around all threads, set new priorities and decay
1324 1318 * per-thread CPU usages.
1325 1319 */
1326 1320 do {
1327 1321 /*
1328 1322 * If this is the first list after the current marker to have
1329 1323 * threads with priority updates, advance the marker to this
1330 1324 * list for the next time fss_update runs.
1331 1325 */
1332 1326 if (fss_update_list(i) &&
1333 1327 new_marker == -1 && i != fss_update_marker)
1334 1328 new_marker = i;
1335 1329 } while ((i = FSS_LIST_NEXT(i)) != fss_update_marker);
1336 1330
1337 1331 /*
1338 1332 * Advance marker for the next fss_update call
1339 1333 */
1340 1334 if (new_marker != -1)
1341 1335 fss_update_marker = new_marker;
1342 1336
1343 1337 (void) timeout(fss_update, arg, hz);
1344 1338 }
1345 1339
1346 1340 /*
1347 1341 * Updates priority for a list of threads. Returns 1 if the priority of one
1348 1342 * of the threads was actually updated, 0 if none were for various reasons
1349 1343 * (thread is no longer in the FSS class, is not runnable, has the preemption
1350 1344 * control no-preempt bit set, etc.)
1351 1345 */
1352 1346 static int
1353 1347 fss_update_list(int i)
1354 1348 {
1355 1349 fssproc_t *fssproc;
1356 1350 fssproj_t *fssproj;
1357 1351 fsspri_t fsspri;
1358 1352 pri_t fss_umdpri;
1359 1353 kthread_t *t;
1360 1354 int updated = 0;
1361 1355
1362 1356 mutex_enter(&fss_listlock[i]);
1363 1357 for (fssproc = fss_listhead[i].fss_next; fssproc != &fss_listhead[i];
1364 1358 fssproc = fssproc->fss_next) {
1365 1359 t = fssproc->fss_tp;
1366 1360 /*
1367 1361 * Lock the thread and verify the state.
1368 1362 */
1369 1363 thread_lock(t);
1370 1364 /*
1371 1365 * Skip the thread if it is no longer in the FSS class or
1372 1366 * is running with kernel mode priority.
1373 1367 */
1374 1368 if (t->t_cid != fss_cid)
1375 1369 goto next;
1376 1370 if ((fssproc->fss_flags & FSSKPRI) != 0)
1377 1371 goto next;
1378 1372
1379 1373 fssproj = FSSPROC2FSSPROJ(fssproc);
1380 1374 if (fssproj == NULL)
1381 1375 goto next;
1382 1376
1383 1377 if (fssproj->fssp_shares != 0) {
1384 1378 /*
1385 1379 * Decay fsspri value.
1386 1380 */
1387 1381 fsspri = fssproc->fss_fsspri;
1388 1382 fsspri = (fsspri * fss_nice_decay[fssproc->fss_nice]) /
1389 1383 FSS_DECAY_BASE;
1390 1384 fssproc->fss_fsspri = fsspri;
1391 1385 }
1392 1386
1393 1387 if (t->t_schedctl && schedctl_get_nopreempt(t))
1394 1388 goto next;
1395 1389 if (t->t_state != TS_RUN && t->t_state != TS_WAIT) {
1396 1390 /*
1397 1391 * Make next syscall/trap call fss_trapret
1398 1392 */
1399 1393 t->t_trapret = 1;
1400 1394 aston(t);
1401 1395 if (t->t_state == TS_ONPROC)
1402 1396 DTRACE_PROBE1(fss__onproc, fssproc_t *,
1403 1397 fssproc);
1404 1398 goto next;
1405 1399 }
1406 1400 fss_newpri(fssproc, B_FALSE);
1407 1401 updated = 1;
1408 1402
1409 1403 fss_umdpri = fssproc->fss_umdpri;
1410 1404
1411 1405 /*
1412 1406 * Only dequeue the thread if it needs to be moved; otherwise
1413 1407 * it should just round-robin here.
1414 1408 */
1415 1409 if (t->t_pri != fss_umdpri)
1416 1410 fss_change_priority(t, fssproc);
1417 1411 next:
1418 1412 thread_unlock(t);
1419 1413 }
1420 1414 mutex_exit(&fss_listlock[i]);
1421 1415 return (updated);
1422 1416 }
1423 1417
1424 1418 /*ARGSUSED*/
1425 1419 static int
1426 1420 fss_admin(caddr_t uaddr, cred_t *reqpcredp)
1427 1421 {
1428 1422 fssadmin_t fssadmin;
1429 1423
1430 1424 if (copyin(uaddr, &fssadmin, sizeof (fssadmin_t)))
1431 1425 return (EFAULT);
1432 1426
1433 1427 switch (fssadmin.fss_cmd) {
1434 1428 case FSS_SETADMIN:
1435 1429 if (secpolicy_dispadm(reqpcredp) != 0)
1436 1430 return (EPERM);
1437 1431 if (fssadmin.fss_quantum <= 0 || fssadmin.fss_quantum >= hz)
1438 1432 return (EINVAL);
1439 1433 fss_quantum = fssadmin.fss_quantum;
1440 1434 break;
1441 1435 case FSS_GETADMIN:
1442 1436 fssadmin.fss_quantum = fss_quantum;
1443 1437 if (copyout(&fssadmin, uaddr, sizeof (fssadmin_t)))
1444 1438 return (EFAULT);
1445 1439 break;
1446 1440 default:
1447 1441 return (EINVAL);
1448 1442 }
1449 1443 return (0);
1450 1444 }
1451 1445
1452 1446 static int
1453 1447 fss_getclinfo(void *infop)
1454 1448 {
1455 1449 fssinfo_t *fssinfo = (fssinfo_t *)infop;
1456 1450 fssinfo->fss_maxupri = fss_maxupri;
1457 1451 return (0);
1458 1452 }
1459 1453
1460 1454 static int
1461 1455 fss_parmsin(void *parmsp)
1462 1456 {
1463 1457 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1464 1458
1465 1459 /*
1466 1460 * Check validity of parameters.
1467 1461 */
1468 1462 if ((fssparmsp->fss_uprilim > fss_maxupri ||
1469 1463 fssparmsp->fss_uprilim < -fss_maxupri) &&
1470 1464 fssparmsp->fss_uprilim != FSS_NOCHANGE)
1471 1465 return (EINVAL);
1472 1466
1473 1467 if ((fssparmsp->fss_upri > fss_maxupri ||
1474 1468 fssparmsp->fss_upri < -fss_maxupri) &&
1475 1469 fssparmsp->fss_upri != FSS_NOCHANGE)
1476 1470 return (EINVAL);
1477 1471
1478 1472 return (0);
1479 1473 }
1480 1474
1481 1475 /*ARGSUSED*/
1482 1476 static int
1483 1477 fss_parmsout(void *parmsp, pc_vaparms_t *vaparmsp)
1484 1478 {
1485 1479 return (0);
1486 1480 }
1487 1481
1488 1482 static int
1489 1483 fss_vaparmsin(void *parmsp, pc_vaparms_t *vaparmsp)
1490 1484 {
1491 1485 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1492 1486 int priflag = 0;
1493 1487 int limflag = 0;
1494 1488 uint_t cnt;
1495 1489 pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
1496 1490
1497 1491 /*
1498 1492 * FSS_NOCHANGE (-32768) is outside of the range of values for
1499 1493 * fss_uprilim and fss_upri. If the structure fssparms_t is changed,
1500 1494 * FSS_NOCHANGE should be replaced by a flag word.
1501 1495 */
1502 1496 fssparmsp->fss_uprilim = FSS_NOCHANGE;
1503 1497 fssparmsp->fss_upri = FSS_NOCHANGE;
1504 1498
1505 1499 /*
1506 1500 * Get the varargs parameter and check validity of parameters.
1507 1501 */
1508 1502 if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
1509 1503 return (EINVAL);
1510 1504
1511 1505 for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
1512 1506 switch (vpp->pc_key) {
1513 1507 case FSS_KY_UPRILIM:
1514 1508 if (limflag++)
1515 1509 return (EINVAL);
1516 1510 fssparmsp->fss_uprilim = (pri_t)vpp->pc_parm;
1517 1511 if (fssparmsp->fss_uprilim > fss_maxupri ||
1518 1512 fssparmsp->fss_uprilim < -fss_maxupri)
1519 1513 return (EINVAL);
1520 1514 break;
1521 1515 case FSS_KY_UPRI:
1522 1516 if (priflag++)
1523 1517 return (EINVAL);
1524 1518 fssparmsp->fss_upri = (pri_t)vpp->pc_parm;
1525 1519 if (fssparmsp->fss_upri > fss_maxupri ||
1526 1520 fssparmsp->fss_upri < -fss_maxupri)
1527 1521 return (EINVAL);
1528 1522 break;
1529 1523 default:
1530 1524 return (EINVAL);
1531 1525 }
1532 1526 }
1533 1527
1534 1528 if (vaparmsp->pc_vaparmscnt == 0) {
1535 1529 /*
1536 1530 * Use default parameters.
1537 1531 */
1538 1532 fssparmsp->fss_upri = fssparmsp->fss_uprilim = 0;
1539 1533 }
1540 1534
1541 1535 return (0);
1542 1536 }
1543 1537
1544 1538 /*
1545 1539 * Copy all selected fair-sharing class parameters to the user. The parameters
1546 1540 * are specified by a key.
1547 1541 */
1548 1542 static int
1549 1543 fss_vaparmsout(void *parmsp, pc_vaparms_t *vaparmsp)
1550 1544 {
1551 1545 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1552 1546 int priflag = 0;
1553 1547 int limflag = 0;
1554 1548 uint_t cnt;
1555 1549 pc_vaparm_t *vpp = &vaparmsp->pc_parms[0];
1556 1550
1557 1551 ASSERT(MUTEX_NOT_HELD(&curproc->p_lock));
1558 1552
1559 1553 if (vaparmsp->pc_vaparmscnt > PC_VAPARMCNT)
1560 1554 return (EINVAL);
1561 1555
1562 1556 for (cnt = 0; cnt < vaparmsp->pc_vaparmscnt; cnt++, vpp++) {
1563 1557 switch (vpp->pc_key) {
1564 1558 case FSS_KY_UPRILIM:
1565 1559 if (limflag++)
1566 1560 return (EINVAL);
1567 1561 if (copyout(&fssparmsp->fss_uprilim,
1568 1562 (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
1569 1563 return (EFAULT);
1570 1564 break;
1571 1565 case FSS_KY_UPRI:
1572 1566 if (priflag++)
1573 1567 return (EINVAL);
1574 1568 if (copyout(&fssparmsp->fss_upri,
1575 1569 (caddr_t)(uintptr_t)vpp->pc_parm, sizeof (pri_t)))
1576 1570 return (EFAULT);
1577 1571 break;
1578 1572 default:
1579 1573 return (EINVAL);
1580 1574 }
1581 1575 }
1582 1576
1583 1577 return (0);
1584 1578 }
1585 1579
1586 1580 /*
1587 1581 * Return the user mode scheduling priority range.
1588 1582 */
1589 1583 static int
1590 1584 fss_getclpri(pcpri_t *pcprip)
1591 1585 {
1592 1586 pcprip->pc_clpmax = fss_maxupri;
1593 1587 pcprip->pc_clpmin = -fss_maxupri;
1594 1588 return (0);
1595 1589 }
1596 1590
1597 1591 static int
1598 1592 fss_alloc(void **p, int flag)
1599 1593 {
1600 1594 void *bufp;
1601 1595
1602 1596 if ((bufp = kmem_zalloc(sizeof (fssproc_t), flag)) == NULL) {
1603 1597 return (ENOMEM);
1604 1598 } else {
1605 1599 *p = bufp;
1606 1600 return (0);
1607 1601 }
1608 1602 }
1609 1603
1610 1604 static void
1611 1605 fss_free(void *bufp)
1612 1606 {
1613 1607 if (bufp)
1614 1608 kmem_free(bufp, sizeof (fssproc_t));
1615 1609 }
1616 1610
1617 1611 /*
1618 1612 * Thread functions
1619 1613 */
1620 1614 static int
1621 1615 fss_enterclass(kthread_t *t, id_t cid, void *parmsp, cred_t *reqpcredp,
1622 1616 void *bufp)
1623 1617 {
1624 1618 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1625 1619 fssproc_t *fssproc;
1626 1620 pri_t reqfssuprilim;
1627 1621 pri_t reqfssupri;
1628 1622 static uint32_t fssexists = 0;
1629 1623 fsspset_t *fsspset;
1630 1624 fssproj_t *fssproj;
1631 1625 fsszone_t *fsszone;
1632 1626 kproject_t *kpj;
1633 1627 zone_t *zone;
1634 1628 int fsszone_allocated = 0;
1635 1629
1636 1630 fssproc = (fssproc_t *)bufp;
1637 1631 ASSERT(fssproc != NULL);
1638 1632
1639 1633 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1640 1634
1641 1635 /*
1642 1636 * Only root can move threads to FSS class.
1643 1637 */
1644 1638 if (reqpcredp != NULL && secpolicy_setpriority(reqpcredp) != 0)
1645 1639 return (EPERM);
1646 1640 /*
1647 1641 * Initialize the fssproc structure.
1648 1642 */
1649 1643 fssproc->fss_umdpri = fss_maxumdpri / 2;
1650 1644
1651 1645 if (fssparmsp == NULL) {
1652 1646 /*
1653 1647 * Use default values.
1654 1648 */
1655 1649 fssproc->fss_nice = NZERO;
1656 1650 fssproc->fss_uprilim = fssproc->fss_upri = 0;
1657 1651 } else {
1658 1652 /*
1659 1653 * Use supplied values.
1660 1654 */
1661 1655 if (fssparmsp->fss_uprilim == FSS_NOCHANGE) {
1662 1656 reqfssuprilim = 0;
1663 1657 } else {
1664 1658 if (fssparmsp->fss_uprilim > 0 &&
1665 1659 secpolicy_setpriority(reqpcredp) != 0)
1666 1660 return (EPERM);
1667 1661 reqfssuprilim = fssparmsp->fss_uprilim;
1668 1662 }
1669 1663 if (fssparmsp->fss_upri == FSS_NOCHANGE) {
1670 1664 reqfssupri = reqfssuprilim;
1671 1665 } else {
1672 1666 if (fssparmsp->fss_upri > 0 &&
1673 1667 secpolicy_setpriority(reqpcredp) != 0)
1674 1668 return (EPERM);
1675 1669 /*
1676 1670 * Set the user priority to the requested value or
1677 1671 * the upri limit, whichever is lower.
1678 1672 */
1679 1673 reqfssupri = fssparmsp->fss_upri;
1680 1674 if (reqfssupri > reqfssuprilim)
1681 1675 reqfssupri = reqfssuprilim;
1682 1676 }
1683 1677 fssproc->fss_uprilim = reqfssuprilim;
1684 1678 fssproc->fss_upri = reqfssupri;
1685 1679 fssproc->fss_nice = NZERO - (NZERO * reqfssupri) / fss_maxupri;
1686 1680 if (fssproc->fss_nice > FSS_NICE_MAX)
1687 1681 fssproc->fss_nice = FSS_NICE_MAX;
1688 1682 }
1689 1683
1690 1684 fssproc->fss_timeleft = fss_quantum;
1691 1685 fssproc->fss_tp = t;
1692 1686 cpucaps_sc_init(&fssproc->fss_caps);
1693 1687
1694 1688 /*
1695 1689 * Put a lock on our fsspset structure.
1696 1690 */
1697 1691 mutex_enter(&fsspsets_lock);
1698 1692 fsspset = fss_find_fsspset(t->t_cpupart);
1699 1693 mutex_enter(&fsspset->fssps_lock);
1700 1694 mutex_exit(&fsspsets_lock);
1701 1695
1702 1696 zone = ttoproc(t)->p_zone;
1703 1697 if ((fsszone = fss_find_fsszone(fsspset, zone)) == NULL) {
1704 1698 if ((fsszone = kmem_zalloc(sizeof (fsszone_t), KM_NOSLEEP))
1705 1699 == NULL) {
1706 1700 mutex_exit(&fsspset->fssps_lock);
1707 1701 return (ENOMEM);
1708 1702 } else {
1709 1703 fsszone_allocated = 1;
1710 1704 fss_insert_fsszone(fsspset, zone, fsszone);
1711 1705 }
1712 1706 }
1713 1707 kpj = ttoproj(t);
1714 1708 if ((fssproj = fss_find_fssproj(fsspset, kpj)) == NULL) {
1715 1709 if ((fssproj = kmem_zalloc(sizeof (fssproj_t), KM_NOSLEEP))
1716 1710 == NULL) {
1717 1711 if (fsszone_allocated) {
1718 1712 fss_remove_fsszone(fsspset, fsszone);
1719 1713 kmem_free(fsszone, sizeof (fsszone_t));
1720 1714 }
1721 1715 mutex_exit(&fsspset->fssps_lock);
1722 1716 return (ENOMEM);
1723 1717 } else {
1724 1718 fss_insert_fssproj(fsspset, kpj, fsszone, fssproj);
1725 1719 }
1726 1720 }
1727 1721 fssproj->fssp_threads++;
1728 1722 fssproc->fss_proj = fssproj;
1729 1723
1730 1724 /*
1731 1725 * Reset priority. Process goes to a "user mode" priority here
1732 1726 * regardless of whether or not it has slept since entering the kernel.
1733 1727 */
1734 1728 thread_lock(t);
1735 1729 t->t_clfuncs = &(sclass[cid].cl_funcs->thread);
1736 1730 t->t_cid = cid;
1737 1731 t->t_cldata = (void *)fssproc;
1738 1732 t->t_schedflag |= TS_RUNQMATCH;
1739 1733 fss_change_priority(t, fssproc);
1740 1734 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
1741 1735 t->t_state == TS_WAIT)
1742 1736 fss_active(t);
1743 1737 thread_unlock(t);
1744 1738
1745 1739 mutex_exit(&fsspset->fssps_lock);
1746 1740
1747 1741 /*
1748 1742 * Link new structure into fssproc list.
1749 1743 */
1750 1744 FSS_LIST_INSERT(fssproc);
1751 1745
1752 1746 /*
1753 1747 * If this is the first fair-sharing thread to occur since boot,
1754 1748 * we set up the initial call to fss_update() here. Use an atomic
1755 1749 * compare-and-swap since that's easier and faster than a mutex
1756 1750 * (but check with an ordinary load first since most of the time
1757 1751 * this will already be done).
1758 1752 */
1759 1753 if (fssexists == 0 && atomic_cas_32(&fssexists, 0, 1) == 0)
1760 1754 (void) timeout(fss_update, NULL, hz);
1761 1755
1762 1756 return (0);
1763 1757 }
1764 1758
1765 1759 /*
1766 1760 * Remove fssproc_t from the list.
1767 1761 */
1768 1762 static void
1769 1763 fss_exitclass(void *procp)
1770 1764 {
1771 1765 fssproc_t *fssproc = (fssproc_t *)procp;
1772 1766 fssproj_t *fssproj;
1773 1767 fsspset_t *fsspset;
1774 1768 fsszone_t *fsszone;
1775 1769 kthread_t *t = fssproc->fss_tp;
1776 1770
1777 1771 /*
1778 1772 * We should be either getting this thread off the deathrow or
1779 1773 * this thread has already moved to another scheduling class and
1780 1774 * we're being called with its old cldata buffer pointer. In both
1781 1775 * cases, the content of this buffer can not be changed while we're
1782 1776 * here.
1783 1777 */
1784 1778 mutex_enter(&fsspsets_lock);
1785 1779 thread_lock(t);
1786 1780 if (t->t_cid != fss_cid) {
1787 1781 /*
1788 1782 * We're being called as a result of the priocntl() system
1789 1783 * call -- someone is trying to move our thread to another
1790 1784 * scheduling class. We can't call fss_inactive() here
1791 1785 * because our thread's t_cldata pointer already points
1792 1786 * to another scheduling class specific data.
1793 1787 */
1794 1788 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
1795 1789
1796 1790 fssproj = FSSPROC2FSSPROJ(fssproc);
1797 1791 fsspset = FSSPROJ2FSSPSET(fssproj);
1798 1792 fsszone = fssproj->fssp_fsszone;
1799 1793
1800 1794 if (fssproc->fss_runnable) {
1801 1795 disp_lock_enter_high(&fsspset->fssps_displock);
1802 1796 if (--fssproj->fssp_runnable == 0) {
1803 1797 fsszone->fssz_shares -= fssproj->fssp_shares;
1804 1798 if (--fsszone->fssz_runnable == 0)
1805 1799 fsspset->fssps_shares -=
1806 1800 fsszone->fssz_rshares;
1807 1801 }
1808 1802 disp_lock_exit_high(&fsspset->fssps_displock);
1809 1803 }
1810 1804 thread_unlock(t);
1811 1805
1812 1806 mutex_enter(&fsspset->fssps_lock);
1813 1807 if (--fssproj->fssp_threads == 0) {
1814 1808 fss_remove_fssproj(fsspset, fssproj);
1815 1809 if (fsszone->fssz_nproj == 0)
1816 1810 kmem_free(fsszone, sizeof (fsszone_t));
1817 1811 kmem_free(fssproj, sizeof (fssproj_t));
1818 1812 }
1819 1813 mutex_exit(&fsspset->fssps_lock);
1820 1814
1821 1815 } else {
1822 1816 ASSERT(t->t_state == TS_FREE);
1823 1817 /*
1824 1818 * We're being called from thread_free() when our thread
1825 1819 * is removed from the deathrow. There is nothing we need
1826 1820 * do here since everything should've been done earlier
1827 1821 * in fss_exit().
1828 1822 */
1829 1823 thread_unlock(t);
1830 1824 }
1831 1825 mutex_exit(&fsspsets_lock);
1832 1826
1833 1827 FSS_LIST_DELETE(fssproc);
1834 1828 fss_free(fssproc);
1835 1829 }
1836 1830
1837 1831 /*ARGSUSED*/
1838 1832 static int
1839 1833 fss_canexit(kthread_t *t, cred_t *credp)
1840 1834 {
1841 1835 /*
1842 1836 * A thread is allowed to exit FSS only if we have sufficient
1843 1837 * privileges.
1844 1838 */
1845 1839 if (credp != NULL && secpolicy_setpriority(credp) != 0)
1846 1840 return (EPERM);
1847 1841 else
1848 1842 return (0);
1849 1843 }
1850 1844
1851 1845 /*
1852 1846 * Initialize fair-share class specific proc structure for a child.
1853 1847 */
1854 1848 static int
1855 1849 fss_fork(kthread_t *pt, kthread_t *ct, void *bufp)
1856 1850 {
1857 1851 fssproc_t *pfssproc; /* ptr to parent's fssproc structure */
1858 1852 fssproc_t *cfssproc; /* ptr to child's fssproc structure */
1859 1853 fssproj_t *fssproj;
1860 1854 fsspset_t *fsspset;
1861 1855
1862 1856 ASSERT(MUTEX_HELD(&ttoproc(pt)->p_lock));
1863 1857 ASSERT(ct->t_state == TS_STOPPED);
1864 1858
1865 1859 cfssproc = (fssproc_t *)bufp;
1866 1860 ASSERT(cfssproc != NULL);
1867 1861 bzero(cfssproc, sizeof (fssproc_t));
1868 1862
1869 1863 thread_lock(pt);
1870 1864 pfssproc = FSSPROC(pt);
1871 1865 fssproj = FSSPROC2FSSPROJ(pfssproc);
1872 1866 fsspset = FSSPROJ2FSSPSET(fssproj);
1873 1867 thread_unlock(pt);
1874 1868
1875 1869 mutex_enter(&fsspset->fssps_lock);
1876 1870 /*
1877 1871 * Initialize child's fssproc structure.
1878 1872 */
1879 1873 thread_lock(pt);
1880 1874 ASSERT(FSSPROJ(pt) == fssproj);
1881 1875 cfssproc->fss_proj = fssproj;
1882 1876 cfssproc->fss_timeleft = fss_quantum;
1883 1877 cfssproc->fss_umdpri = pfssproc->fss_umdpri;
1884 1878 cfssproc->fss_fsspri = 0;
1885 1879 cfssproc->fss_uprilim = pfssproc->fss_uprilim;
1886 1880 cfssproc->fss_upri = pfssproc->fss_upri;
1887 1881 cfssproc->fss_tp = ct;
1888 1882 cfssproc->fss_nice = pfssproc->fss_nice;
1889 1883 cpucaps_sc_init(&cfssproc->fss_caps);
1890 1884
1891 1885 cfssproc->fss_flags =
1892 1886 pfssproc->fss_flags & ~(FSSKPRI | FSSBACKQ | FSSRESTORE);
1893 1887 ct->t_cldata = (void *)cfssproc;
1894 1888 ct->t_schedflag |= TS_RUNQMATCH;
1895 1889 thread_unlock(pt);
1896 1890
1897 1891 fssproj->fssp_threads++;
1898 1892 mutex_exit(&fsspset->fssps_lock);
1899 1893
1900 1894 /*
1901 1895 * Link new structure into fssproc hash table.
1902 1896 */
1903 1897 FSS_LIST_INSERT(cfssproc);
1904 1898 return (0);
1905 1899 }
1906 1900
1907 1901 /*
1908 1902 * Child is placed at back of dispatcher queue and parent gives up processor
1909 1903 * so that the child runs first after the fork. This allows the child
1910 1904 * immediately execing to break the multiple use of copy on write pages with no
1911 1905 * disk home. The parent will get to steal them back rather than uselessly
1912 1906 * copying them.
1913 1907 */
1914 1908 static void
1915 1909 fss_forkret(kthread_t *t, kthread_t *ct)
1916 1910 {
1917 1911 proc_t *pp = ttoproc(t);
1918 1912 proc_t *cp = ttoproc(ct);
1919 1913 fssproc_t *fssproc;
1920 1914
1921 1915 ASSERT(t == curthread);
1922 1916 ASSERT(MUTEX_HELD(&pidlock));
1923 1917
1924 1918 /*
1925 1919 * Grab the child's p_lock before dropping pidlock to ensure the
1926 1920 * process does not disappear before we set it running.
1927 1921 */
1928 1922 mutex_enter(&cp->p_lock);
1929 1923 continuelwps(cp);
1930 1924 mutex_exit(&cp->p_lock);
1931 1925
1932 1926 mutex_enter(&pp->p_lock);
1933 1927 mutex_exit(&pidlock);
1934 1928 continuelwps(pp);
1935 1929
1936 1930 thread_lock(t);
1937 1931
1938 1932 fssproc = FSSPROC(t);
1939 1933 fss_newpri(fssproc, B_FALSE);
1940 1934 fssproc->fss_timeleft = fss_quantum;
1941 1935 t->t_pri = fssproc->fss_umdpri;
1942 1936 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
1943 1937 fssproc->fss_flags &= ~FSSKPRI;
1944 1938 THREAD_TRANSITION(t);
1945 1939
1946 1940 /*
1947 1941 * We don't want to call fss_setrun(t) here because it may call
1948 1942 * fss_active, which we don't need.
1949 1943 */
1950 1944 fssproc->fss_flags &= ~FSSBACKQ;
1951 1945
1952 1946 if (t->t_disp_time != ddi_get_lbolt())
1953 1947 setbackdq(t);
1954 1948 else
1955 1949 setfrontdq(t);
1956 1950
1957 1951 thread_unlock(t);
1958 1952 /*
1959 1953 * Safe to drop p_lock now since it is safe to change
1960 1954 * the scheduling class after this point.
1961 1955 */
1962 1956 mutex_exit(&pp->p_lock);
1963 1957
1964 1958 swtch();
1965 1959 }
1966 1960
1967 1961 /*
1968 1962 * Get the fair-sharing parameters of the thread pointed to by fssprocp into
1969 1963 * the buffer pointed by fssparmsp.
1970 1964 */
1971 1965 static void
1972 1966 fss_parmsget(kthread_t *t, void *parmsp)
1973 1967 {
1974 1968 fssproc_t *fssproc = FSSPROC(t);
1975 1969 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1976 1970
1977 1971 fssparmsp->fss_uprilim = fssproc->fss_uprilim;
1978 1972 fssparmsp->fss_upri = fssproc->fss_upri;
1979 1973 }
1980 1974
1981 1975 /*ARGSUSED*/
1982 1976 static int
1983 1977 fss_parmsset(kthread_t *t, void *parmsp, id_t reqpcid, cred_t *reqpcredp)
1984 1978 {
1985 1979 char nice;
1986 1980 pri_t reqfssuprilim;
1987 1981 pri_t reqfssupri;
1988 1982 fssproc_t *fssproc = FSSPROC(t);
1989 1983 fssparms_t *fssparmsp = (fssparms_t *)parmsp;
1990 1984
1991 1985 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
1992 1986
1993 1987 if (fssparmsp->fss_uprilim == FSS_NOCHANGE)
1994 1988 reqfssuprilim = fssproc->fss_uprilim;
1995 1989 else
1996 1990 reqfssuprilim = fssparmsp->fss_uprilim;
1997 1991
1998 1992 if (fssparmsp->fss_upri == FSS_NOCHANGE)
1999 1993 reqfssupri = fssproc->fss_upri;
2000 1994 else
2001 1995 reqfssupri = fssparmsp->fss_upri;
2002 1996
2003 1997 /*
2004 1998 * Make sure the user priority doesn't exceed the upri limit.
2005 1999 */
2006 2000 if (reqfssupri > reqfssuprilim)
2007 2001 reqfssupri = reqfssuprilim;
2008 2002
2009 2003 /*
2010 2004 * Basic permissions enforced by generic kernel code for all classes
2011 2005 * require that a thread attempting to change the scheduling parameters
2012 2006 * of a target thread be privileged or have a real or effective UID
2013 2007 * matching that of the target thread. We are not called unless these
2014 2008 * basic permission checks have already passed. The fair-sharing class
2015 2009 * requires in addition that the calling thread be privileged if it
2016 2010 * is attempting to raise the upri limit above its current value.
2017 2011 * This may have been checked previously but if our caller passed us
2018 2012 * a non-NULL credential pointer we assume it hasn't and we check it
2019 2013 * here.
2020 2014 */
2021 2015 if ((reqpcredp != NULL) &&
2022 2016 (reqfssuprilim > fssproc->fss_uprilim) &&
2023 2017 secpolicy_raisepriority(reqpcredp) != 0)
2024 2018 return (EPERM);
2025 2019
2026 2020 /*
2027 2021 * Set fss_nice to the nice value corresponding to the user priority we
2028 2022 * are setting. Note that setting the nice field of the parameter
2029 2023 * struct won't affect upri or nice.
2030 2024 */
2031 2025 nice = NZERO - (reqfssupri * NZERO) / fss_maxupri;
2032 2026 if (nice > FSS_NICE_MAX)
2033 2027 nice = FSS_NICE_MAX;
2034 2028
2035 2029 thread_lock(t);
2036 2030
2037 2031 fssproc->fss_uprilim = reqfssuprilim;
2038 2032 fssproc->fss_upri = reqfssupri;
2039 2033 fssproc->fss_nice = nice;
2040 2034 fss_newpri(fssproc, B_FALSE);
2041 2035
2042 2036 if ((fssproc->fss_flags & FSSKPRI) != 0) {
2043 2037 thread_unlock(t);
2044 2038 return (0);
2045 2039 }
2046 2040
2047 2041 fss_change_priority(t, fssproc);
2048 2042 thread_unlock(t);
2049 2043 return (0);
2050 2044
2051 2045 }
2052 2046
2053 2047 /*
2054 2048 * The thread is being stopped.
2055 2049 */
2056 2050 /*ARGSUSED*/
2057 2051 static void
2058 2052 fss_stop(kthread_t *t, int why, int what)
2059 2053 {
2060 2054 ASSERT(THREAD_LOCK_HELD(t));
2061 2055 ASSERT(t == curthread);
2062 2056
2063 2057 fss_inactive(t);
2064 2058 }
2065 2059
2066 2060 /*
2067 2061 * The current thread is exiting, do necessary adjustments to its project
2068 2062 */
2069 2063 static void
2070 2064 fss_exit(kthread_t *t)
2071 2065 {
2072 2066 fsspset_t *fsspset;
2073 2067 fssproj_t *fssproj;
2074 2068 fssproc_t *fssproc;
2075 2069 fsszone_t *fsszone;
2076 2070 int free = 0;
2077 2071
2078 2072 /*
2079 2073 * Thread t here is either a current thread (in which case we hold
2080 2074 * its process' p_lock), or a thread being destroyed by forklwp_fail(),
2081 2075 * in which case we hold pidlock and thread is no longer on the
2082 2076 * thread list.
2083 2077 */
2084 2078 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock) || MUTEX_HELD(&pidlock));
2085 2079
2086 2080 fssproc = FSSPROC(t);
2087 2081 fssproj = FSSPROC2FSSPROJ(fssproc);
2088 2082 fsspset = FSSPROJ2FSSPSET(fssproj);
2089 2083 fsszone = fssproj->fssp_fsszone;
2090 2084
2091 2085 mutex_enter(&fsspsets_lock);
2092 2086 mutex_enter(&fsspset->fssps_lock);
2093 2087
2094 2088 thread_lock(t);
2095 2089 disp_lock_enter_high(&fsspset->fssps_displock);
2096 2090 if (t->t_state == TS_ONPROC || t->t_state == TS_RUN) {
2097 2091 if (--fssproj->fssp_runnable == 0) {
2098 2092 fsszone->fssz_shares -= fssproj->fssp_shares;
2099 2093 if (--fsszone->fssz_runnable == 0)
2100 2094 fsspset->fssps_shares -= fsszone->fssz_rshares;
2101 2095 }
2102 2096 ASSERT(fssproc->fss_runnable == 1);
2103 2097 fssproc->fss_runnable = 0;
2104 2098 }
2105 2099 if (--fssproj->fssp_threads == 0) {
2106 2100 fss_remove_fssproj(fsspset, fssproj);
2107 2101 free = 1;
2108 2102 }
2109 2103 disp_lock_exit_high(&fsspset->fssps_displock);
2110 2104 fssproc->fss_proj = NULL; /* mark this thread as already exited */
2111 2105 thread_unlock(t);
2112 2106
2113 2107 if (free) {
2114 2108 if (fsszone->fssz_nproj == 0)
2115 2109 kmem_free(fsszone, sizeof (fsszone_t));
2116 2110 kmem_free(fssproj, sizeof (fssproj_t));
2117 2111 }
2118 2112 mutex_exit(&fsspset->fssps_lock);
2119 2113 mutex_exit(&fsspsets_lock);
2120 2114
2121 2115 /*
2122 2116 * A thread could be exiting in between clock ticks, so we need to
2123 2117 * calculate how much CPU time it used since it was charged last time.
2124 2118 *
2125 2119 * CPU caps are not enforced on exiting processes - it is usually
2126 2120 * desirable to exit as soon as possible to free resources.
2127 2121 */
2128 2122 if (CPUCAPS_ON()) {
2129 2123 thread_lock(t);
2130 2124 fssproc = FSSPROC(t);
2131 2125 (void) cpucaps_charge(t, &fssproc->fss_caps,
2132 2126 CPUCAPS_CHARGE_ONLY);
|
↓ open down ↓ |
1701 lines elided |
↑ open up ↑ |
2133 2127 thread_unlock(t);
2134 2128 }
2135 2129 }
2136 2130
2137 2131 static void
2138 2132 fss_nullsys()
2139 2133 {
2140 2134 }
2141 2135
2142 2136 /*
2143 - * fss_swapin() returns -1 if the thread is loaded or is not eligible to be
2144 - * swapped in. Otherwise, it returns the thread's effective priority based
2145 - * on swapout time and size of process (0 <= epri <= 0 SHRT_MAX).
2146 - */
2147 -/*ARGSUSED*/
2148 -static pri_t
2149 -fss_swapin(kthread_t *t, int flags)
2150 -{
2151 - fssproc_t *fssproc = FSSPROC(t);
2152 - long epri = -1;
2153 - proc_t *pp = ttoproc(t);
2154 -
2155 - ASSERT(THREAD_LOCK_HELD(t));
2156 -
2157 - if (t->t_state == TS_RUN && (t->t_schedflag & TS_LOAD) == 0) {
2158 - time_t swapout_time;
2159 -
2160 - swapout_time = (ddi_get_lbolt() - t->t_stime) / hz;
2161 - if (INHERITED(t) || (fssproc->fss_flags & FSSKPRI)) {
2162 - epri = (long)DISP_PRIO(t) + swapout_time;
2163 - } else {
2164 - /*
2165 - * Threads which have been out for a long time,
2166 - * have high user mode priority and are associated
2167 - * with a small address space are more deserving.
2168 - */
2169 - epri = fssproc->fss_umdpri;
2170 - ASSERT(epri >= 0 && epri <= fss_maxumdpri);
2171 - epri += swapout_time - pp->p_swrss / nz(maxpgio)/2;
2172 - }
2173 - /*
2174 - * Scale epri so that SHRT_MAX / 2 represents zero priority.
2175 - */
2176 - epri += SHRT_MAX / 2;
2177 - if (epri < 0)
2178 - epri = 0;
2179 - else if (epri > SHRT_MAX)
2180 - epri = SHRT_MAX;
2181 - }
2182 - return ((pri_t)epri);
2183 -}
2184 -
2185 -/*
2186 - * fss_swapout() returns -1 if the thread isn't loaded or is not eligible to
2187 - * be swapped out. Otherwise, it returns the thread's effective priority
2188 - * based on if the swapper is in softswap or hardswap mode.
2189 - */
2190 -static pri_t
2191 -fss_swapout(kthread_t *t, int flags)
2192 -{
2193 - fssproc_t *fssproc = FSSPROC(t);
2194 - long epri = -1;
2195 - proc_t *pp = ttoproc(t);
2196 - time_t swapin_time;
2197 -
2198 - ASSERT(THREAD_LOCK_HELD(t));
2199 -
2200 - if (INHERITED(t) ||
2201 - (fssproc->fss_flags & FSSKPRI) ||
2202 - (t->t_proc_flag & TP_LWPEXIT) ||
2203 - (t->t_state & (TS_ZOMB|TS_FREE|TS_STOPPED|TS_ONPROC|TS_WAIT)) ||
2204 - !(t->t_schedflag & TS_LOAD) ||
2205 - !(SWAP_OK(t)))
2206 - return (-1);
2207 -
2208 - ASSERT(t->t_state & (TS_SLEEP | TS_RUN));
2209 -
2210 - swapin_time = (ddi_get_lbolt() - t->t_stime) / hz;
2211 -
2212 - if (flags == SOFTSWAP) {
2213 - if (t->t_state == TS_SLEEP && swapin_time > maxslp) {
2214 - epri = 0;
2215 - } else {
2216 - return ((pri_t)epri);
2217 - }
2218 - } else {
2219 - pri_t pri;
2220 -
2221 - if ((t->t_state == TS_SLEEP && swapin_time > fss_minslp) ||
2222 - (t->t_state == TS_RUN && swapin_time > fss_minrun)) {
2223 - pri = fss_maxumdpri;
2224 - epri = swapin_time -
2225 - (rm_asrss(pp->p_as) / nz(maxpgio)/2) - (long)pri;
2226 - } else {
2227 - return ((pri_t)epri);
2228 - }
2229 - }
2230 -
2231 - /*
2232 - * Scale epri so that SHRT_MAX / 2 represents zero priority.
2233 - */
2234 - epri += SHRT_MAX / 2;
2235 - if (epri < 0)
2236 - epri = 0;
2237 - else if (epri > SHRT_MAX)
2238 - epri = SHRT_MAX;
2239 -
2240 - return ((pri_t)epri);
2241 -}
2242 -
2243 -/*
2244 2137 * If thread is currently at a kernel mode priority (has slept) and is
2245 2138 * returning to the userland we assign it the appropriate user mode priority
2246 2139 * and time quantum here. If we're lowering the thread's priority below that
2247 2140 * of other runnable threads then we will set runrun via cpu_surrender() to
2248 2141 * cause preemption.
2249 2142 */
2250 2143 static void
2251 2144 fss_trapret(kthread_t *t)
2252 2145 {
2253 2146 fssproc_t *fssproc = FSSPROC(t);
2254 2147 cpu_t *cp = CPU;
2255 2148
2256 2149 ASSERT(THREAD_LOCK_HELD(t));
2257 2150 ASSERT(t == curthread);
2258 2151 ASSERT(cp->cpu_dispthread == t);
2259 2152 ASSERT(t->t_state == TS_ONPROC);
2260 2153
2261 2154 t->t_kpri_req = 0;
2262 2155 if (fssproc->fss_flags & FSSKPRI) {
2263 2156 /*
|
↓ open down ↓ |
10 lines elided |
↑ open up ↑ |
2264 2157 * If thread has blocked in the kernel
2265 2158 */
2266 2159 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2267 2160 cp->cpu_dispatch_pri = DISP_PRIO(t);
2268 2161 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
2269 2162 fssproc->fss_flags &= ~FSSKPRI;
2270 2163
2271 2164 if (DISP_MUST_SURRENDER(t))
2272 2165 cpu_surrender(t);
2273 2166 }
2274 -
2275 - /*
2276 - * Swapout lwp if the swapper is waiting for this thread to reach
2277 - * a safe point.
2278 - */
2279 - if (t->t_schedflag & TS_SWAPENQ) {
2280 - thread_unlock(t);
2281 - swapout_lwp(ttolwp(t));
2282 - thread_lock(t);
2283 - }
2284 2167 }
2285 2168
2286 2169 /*
2287 2170 * Arrange for thread to be placed in appropriate location on dispatcher queue.
2288 2171 * This is called with the current thread in TS_ONPROC and locked.
2289 2172 */
2290 2173 static void
2291 2174 fss_preempt(kthread_t *t)
2292 2175 {
2293 2176 fssproc_t *fssproc = FSSPROC(t);
2294 2177 klwp_t *lwp;
2295 2178 uint_t flags;
2296 2179
2297 2180 ASSERT(t == curthread);
2298 2181 ASSERT(THREAD_LOCK_HELD(curthread));
2299 2182 ASSERT(t->t_state == TS_ONPROC);
2300 2183
2301 2184 /*
2302 2185 * If preempted in the kernel, make sure the thread has a kernel
2303 2186 * priority if needed.
2304 2187 */
2305 2188 lwp = curthread->t_lwp;
2306 2189 if (!(fssproc->fss_flags & FSSKPRI) && lwp != NULL && t->t_kpri_req) {
2307 2190 fssproc->fss_flags |= FSSKPRI;
2308 2191 THREAD_CHANGE_PRI(t, minclsyspri);
2309 2192 ASSERT(t->t_pri >= 0 && t->t_pri <= fss_maxglobpri);
2310 2193 t->t_trapret = 1; /* so that fss_trapret will run */
2311 2194 aston(t);
2312 2195 }
2313 2196
2314 2197 /*
2315 2198 * This thread may be placed on wait queue by CPU Caps. In this case we
2316 2199 * do not need to do anything until it is removed from the wait queue.
2317 2200 * Do not enforce CPU caps on threads running at a kernel priority
|
↓ open down ↓ |
24 lines elided |
↑ open up ↑ |
2318 2201 */
2319 2202 if (CPUCAPS_ON()) {
2320 2203 (void) cpucaps_charge(t, &fssproc->fss_caps,
2321 2204 CPUCAPS_CHARGE_ENFORCE);
2322 2205
2323 2206 if (!(fssproc->fss_flags & FSSKPRI) && CPUCAPS_ENFORCE(t))
2324 2207 return;
2325 2208 }
2326 2209
2327 2210 /*
2328 - * If preempted in user-land mark the thread as swappable because it
2329 - * cannot be holding any kernel locks.
2330 - */
2331 - ASSERT(t->t_schedflag & TS_DONT_SWAP);
2332 - if (lwp != NULL && lwp->lwp_state == LWP_USER)
2333 - t->t_schedflag &= ~TS_DONT_SWAP;
2334 -
2335 - /*
2336 2211 * Check to see if we're doing "preemption control" here. If
2337 2212 * we are, and if the user has requested that this thread not
2338 2213 * be preempted, and if preemptions haven't been put off for
2339 2214 * too long, let the preemption happen here but try to make
2340 2215 * sure the thread is rescheduled as soon as possible. We do
2341 2216 * this by putting it on the front of the highest priority run
2342 2217 * queue in the FSS class. If the preemption has been put off
2343 2218 * for too long, clear the "nopreempt" bit and let the thread
2344 2219 * be preempted.
2345 2220 */
2346 2221 if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2347 2222 if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2348 2223 DTRACE_SCHED1(schedctl__nopreempt, kthread_t *, t);
|
↓ open down ↓ |
3 lines elided |
↑ open up ↑ |
2349 2224 if (!(fssproc->fss_flags & FSSKPRI)) {
2350 2225 /*
2351 2226 * If not already remembered, remember current
2352 2227 * priority for restoration in fss_yield().
2353 2228 */
2354 2229 if (!(fssproc->fss_flags & FSSRESTORE)) {
2355 2230 fssproc->fss_scpri = t->t_pri;
2356 2231 fssproc->fss_flags |= FSSRESTORE;
2357 2232 }
2358 2233 THREAD_CHANGE_PRI(t, fss_maxumdpri);
2359 - t->t_schedflag |= TS_DONT_SWAP;
2360 2234 }
2361 2235 schedctl_set_yield(t, 1);
2362 2236 setfrontdq(t);
2363 2237 return;
2364 2238 } else {
2365 2239 if (fssproc->fss_flags & FSSRESTORE) {
2366 2240 THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
2367 2241 fssproc->fss_flags &= ~FSSRESTORE;
2368 2242 }
2369 2243 schedctl_set_nopreempt(t, 0);
2370 2244 DTRACE_SCHED1(schedctl__preempt, kthread_t *, t);
2371 2245 /*
2372 2246 * Fall through and be preempted below.
2373 2247 */
2374 2248 }
2375 2249 }
2376 2250
2377 2251 flags = fssproc->fss_flags & (FSSBACKQ | FSSKPRI);
2378 2252
2379 2253 if (flags == FSSBACKQ) {
2380 2254 fssproc->fss_timeleft = fss_quantum;
2381 2255 fssproc->fss_flags &= ~FSSBACKQ;
2382 2256 setbackdq(t);
2383 2257 } else if (flags == (FSSBACKQ | FSSKPRI)) {
2384 2258 fssproc->fss_flags &= ~FSSBACKQ;
2385 2259 setbackdq(t);
2386 2260 } else {
2387 2261 setfrontdq(t);
2388 2262 }
2389 2263 }
2390 2264
2391 2265 /*
2392 2266 * Called when a thread is waking up and is to be placed on the run queue.
2393 2267 */
2394 2268 static void
2395 2269 fss_setrun(kthread_t *t)
2396 2270 {
2397 2271 fssproc_t *fssproc = FSSPROC(t);
2398 2272
2399 2273 ASSERT(THREAD_LOCK_HELD(t)); /* t should be in transition */
2400 2274
2401 2275 if (t->t_state == TS_SLEEP || t->t_state == TS_STOPPED)
2402 2276 fss_active(t);
2403 2277
2404 2278 fssproc->fss_timeleft = fss_quantum;
2405 2279
2406 2280 fssproc->fss_flags &= ~FSSBACKQ;
2407 2281 /*
2408 2282 * If previously were running at the kernel priority then keep that
2409 2283 * priority and the fss_timeleft doesn't matter.
2410 2284 */
2411 2285 if ((fssproc->fss_flags & FSSKPRI) == 0)
2412 2286 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2413 2287
2414 2288 if (t->t_disp_time != ddi_get_lbolt())
2415 2289 setbackdq(t);
2416 2290 else
2417 2291 setfrontdq(t);
2418 2292 }
2419 2293
2420 2294 /*
2421 2295 * Prepare thread for sleep. We reset the thread priority so it will run at the
2422 2296 * kernel priority level when it wakes up.
2423 2297 */
2424 2298 static void
2425 2299 fss_sleep(kthread_t *t)
2426 2300 {
2427 2301 fssproc_t *fssproc = FSSPROC(t);
2428 2302
2429 2303 ASSERT(t == curthread);
2430 2304 ASSERT(THREAD_LOCK_HELD(t));
2431 2305
2432 2306 ASSERT(t->t_state == TS_ONPROC);
2433 2307
2434 2308 /*
2435 2309 * Account for time spent on CPU before going to sleep.
2436 2310 */
2437 2311 (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2438 2312
2439 2313 fss_inactive(t);
2440 2314
2441 2315 /*
2442 2316 * Assign a system priority to the thread and arrange for it to be
2443 2317 * retained when the thread is next placed on the run queue (i.e.,
2444 2318 * when it wakes up) instead of being given a new pri. Also arrange
2445 2319 * for trapret processing as the thread leaves the system call so it
2446 2320 * will drop back to normal priority range.
2447 2321 */
2448 2322 if (t->t_kpri_req) {
2449 2323 THREAD_CHANGE_PRI(t, minclsyspri);
2450 2324 fssproc->fss_flags |= FSSKPRI;
2451 2325 t->t_trapret = 1; /* so that fss_trapret will run */
2452 2326 aston(t);
2453 2327 } else if (fssproc->fss_flags & FSSKPRI) {
2454 2328 /*
|
↓ open down ↓ |
85 lines elided |
↑ open up ↑ |
2455 2329 * The thread has done a THREAD_KPRI_REQUEST(), slept, then
2456 2330 * done THREAD_KPRI_RELEASE() (so no t_kpri_req is 0 again),
2457 2331 * then slept again all without finishing the current system
2458 2332 * call so trapret won't have cleared FSSKPRI
2459 2333 */
2460 2334 fssproc->fss_flags &= ~FSSKPRI;
2461 2335 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2462 2336 if (DISP_MUST_SURRENDER(curthread))
2463 2337 cpu_surrender(t);
2464 2338 }
2465 - t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */
2466 2339 }
2467 2340
2468 2341 /*
2469 2342 * A tick interrupt has ocurrend on a running thread. Check to see if our
2470 - * time slice has expired. We must also clear the TS_DONT_SWAP flag in
2471 - * t_schedflag if the thread is eligible to be swapped out.
2343 + * time slice has expired.
2472 2344 */
2473 2345 static void
2474 2346 fss_tick(kthread_t *t)
2475 2347 {
2476 2348 fssproc_t *fssproc;
2477 2349 fssproj_t *fssproj;
2478 - klwp_t *lwp;
2479 2350 boolean_t call_cpu_surrender = B_FALSE;
2480 2351 boolean_t cpucaps_enforce = B_FALSE;
2481 2352
2482 2353 ASSERT(MUTEX_HELD(&(ttoproc(t))->p_lock));
2483 2354
2484 2355 /*
2485 2356 * It's safe to access fsspset and fssproj structures because we're
2486 2357 * holding our p_lock here.
2487 2358 */
2488 2359 thread_lock(t);
2489 2360 fssproc = FSSPROC(t);
2490 2361 fssproj = FSSPROC2FSSPROJ(fssproc);
2491 2362 if (fssproj != NULL) {
2492 2363 fsspset_t *fsspset = FSSPROJ2FSSPSET(fssproj);
2493 2364 disp_lock_enter_high(&fsspset->fssps_displock);
2494 2365 fssproj->fssp_ticks += fss_nice_tick[fssproc->fss_nice];
2495 2366 fssproj->fssp_tick_cnt++;
2496 2367 fssproc->fss_ticks++;
2497 2368 disp_lock_exit_high(&fsspset->fssps_displock);
2498 2369 }
2499 2370
2500 2371 /*
2501 2372 * Keep track of thread's project CPU usage. Note that projects
2502 2373 * get charged even when threads are running in the kernel.
2503 2374 * Do not surrender CPU if running in the SYS class.
2504 2375 */
2505 2376 if (CPUCAPS_ON()) {
2506 2377 cpucaps_enforce = cpucaps_charge(t,
2507 2378 &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE) &&
2508 2379 !(fssproc->fss_flags & FSSKPRI);
2509 2380 }
2510 2381
2511 2382 /*
2512 2383 * A thread's execution time for threads running in the SYS class
2513 2384 * is not tracked.
2514 2385 */
2515 2386 if ((fssproc->fss_flags & FSSKPRI) == 0) {
2516 2387 /*
2517 2388 * If thread is not in kernel mode, decrement its fss_timeleft
2518 2389 */
2519 2390 if (--fssproc->fss_timeleft <= 0) {
2520 2391 pri_t new_pri;
2521 2392
2522 2393 /*
2523 2394 * If we're doing preemption control and trying to
2524 2395 * avoid preempting this thread, just note that the
2525 2396 * thread should yield soon and let it keep running
2526 2397 * (unless it's been a while).
2527 2398 */
2528 2399 if (t->t_schedctl && schedctl_get_nopreempt(t)) {
2529 2400 if (fssproc->fss_timeleft > -SC_MAX_TICKS) {
2530 2401 DTRACE_SCHED1(schedctl__nopreempt,
2531 2402 kthread_t *, t);
2532 2403 schedctl_set_yield(t, 1);
2533 2404 thread_unlock_nopreempt(t);
2534 2405 return;
2535 2406 }
2536 2407 }
2537 2408 fssproc->fss_flags &= ~FSSRESTORE;
2538 2409
2539 2410 fss_newpri(fssproc, B_TRUE);
|
↓ open down ↓ |
51 lines elided |
↑ open up ↑ |
2540 2411 new_pri = fssproc->fss_umdpri;
2541 2412 ASSERT(new_pri >= 0 && new_pri <= fss_maxglobpri);
2542 2413
2543 2414 /*
2544 2415 * When the priority of a thread is changed, it may
2545 2416 * be necessary to adjust its position on a sleep queue
2546 2417 * or dispatch queue. The function thread_change_pri
2547 2418 * accomplishes this.
2548 2419 */
2549 2420 if (thread_change_pri(t, new_pri, 0)) {
2550 - if ((t->t_schedflag & TS_LOAD) &&
2551 - (lwp = t->t_lwp) &&
2552 - lwp->lwp_state == LWP_USER)
2553 - t->t_schedflag &= ~TS_DONT_SWAP;
2554 2421 fssproc->fss_timeleft = fss_quantum;
2555 2422 } else {
2556 2423 call_cpu_surrender = B_TRUE;
2557 2424 }
2558 2425 } else if (t->t_state == TS_ONPROC &&
2559 2426 t->t_pri < t->t_disp_queue->disp_maxrunpri) {
2560 2427 /*
2561 2428 * If there is a higher-priority thread which is
2562 2429 * waiting for a processor, then thread surrenders
2563 2430 * the processor.
2564 2431 */
2565 2432 call_cpu_surrender = B_TRUE;
2566 2433 }
2567 2434 }
2568 2435
2569 2436 if (cpucaps_enforce && 2 * fssproc->fss_timeleft > fss_quantum) {
2570 2437 /*
2571 2438 * The thread used more than half of its quantum, so assume that
2572 2439 * it used the whole quantum.
2573 2440 *
2574 2441 * Update thread's priority just before putting it on the wait
2575 2442 * queue so that it gets charged for the CPU time from its
2576 2443 * quantum even before that quantum expires.
2577 2444 */
2578 2445 fss_newpri(fssproc, B_FALSE);
2579 2446 if (t->t_pri != fssproc->fss_umdpri)
2580 2447 fss_change_priority(t, fssproc);
2581 2448
2582 2449 /*
2583 2450 * We need to call cpu_surrender for this thread due to cpucaps
2584 2451 * enforcement, but fss_change_priority may have already done
2585 2452 * so. In this case FSSBACKQ is set and there is no need to call
2586 2453 * cpu-surrender again.
2587 2454 */
2588 2455 if (!(fssproc->fss_flags & FSSBACKQ))
2589 2456 call_cpu_surrender = B_TRUE;
2590 2457 }
2591 2458
2592 2459 if (call_cpu_surrender) {
2593 2460 fssproc->fss_flags |= FSSBACKQ;
2594 2461 cpu_surrender(t);
2595 2462 }
2596 2463
2597 2464 thread_unlock_nopreempt(t); /* clock thread can't be preempted */
2598 2465 }
2599 2466
2600 2467 /*
2601 2468 * Processes waking up go to the back of their queue. We don't need to assign
2602 2469 * a time quantum here because thread is still at a kernel mode priority and
2603 2470 * the time slicing is not done for threads running in the kernel after
2604 2471 * sleeping. The proper time quantum will be assigned by fss_trapret before the
2605 2472 * thread returns to user mode.
2606 2473 */
|
↓ open down ↓ |
43 lines elided |
↑ open up ↑ |
2607 2474 static void
2608 2475 fss_wakeup(kthread_t *t)
2609 2476 {
2610 2477 fssproc_t *fssproc;
2611 2478
2612 2479 ASSERT(THREAD_LOCK_HELD(t));
2613 2480 ASSERT(t->t_state == TS_SLEEP);
2614 2481
2615 2482 fss_active(t);
2616 2483
2617 - t->t_stime = ddi_get_lbolt(); /* time stamp for the swapper */
2618 2484 fssproc = FSSPROC(t);
2619 2485 fssproc->fss_flags &= ~FSSBACKQ;
2620 2486
2621 2487 if (fssproc->fss_flags & FSSKPRI) {
2622 2488 /*
2623 2489 * If we already have a kernel priority assigned, then we
2624 2490 * just use it.
2625 2491 */
2626 2492 setbackdq(t);
2627 2493 } else if (t->t_kpri_req) {
2628 2494 /*
2629 2495 * Give thread a priority boost if we were asked.
2630 2496 */
2631 2497 fssproc->fss_flags |= FSSKPRI;
2632 2498 THREAD_CHANGE_PRI(t, minclsyspri);
2633 2499 setbackdq(t);
2634 2500 t->t_trapret = 1; /* so that fss_trapret will run */
2635 2501 aston(t);
2636 2502 } else {
2637 2503 /*
2638 2504 * Otherwise, we recalculate the priority.
2639 2505 */
2640 2506 if (t->t_disp_time == ddi_get_lbolt()) {
2641 2507 setfrontdq(t);
2642 2508 } else {
2643 2509 fssproc->fss_timeleft = fss_quantum;
2644 2510 THREAD_CHANGE_PRI(t, fssproc->fss_umdpri);
2645 2511 setbackdq(t);
2646 2512 }
2647 2513 }
2648 2514 }
2649 2515
2650 2516 /*
2651 2517 * fss_donice() is called when a nice(1) command is issued on the thread to
2652 2518 * alter the priority. The nice(1) command exists in Solaris for compatibility.
2653 2519 * Thread priority adjustments should be done via priocntl(1).
2654 2520 */
2655 2521 static int
2656 2522 fss_donice(kthread_t *t, cred_t *cr, int incr, int *retvalp)
2657 2523 {
2658 2524 int newnice;
2659 2525 fssproc_t *fssproc = FSSPROC(t);
2660 2526 fssparms_t fssparms;
2661 2527
2662 2528 /*
2663 2529 * If there is no change to priority, just return current setting.
2664 2530 */
2665 2531 if (incr == 0) {
2666 2532 if (retvalp)
2667 2533 *retvalp = fssproc->fss_nice - NZERO;
2668 2534 return (0);
2669 2535 }
2670 2536
2671 2537 if ((incr < 0 || incr > 2 * NZERO) && secpolicy_raisepriority(cr) != 0)
2672 2538 return (EPERM);
2673 2539
2674 2540 /*
2675 2541 * Specifying a nice increment greater than the upper limit of
2676 2542 * FSS_NICE_MAX (== 2 * NZERO - 1) will result in the thread's nice
2677 2543 * value being set to the upper limit. We check for this before
2678 2544 * computing the new value because otherwise we could get overflow
2679 2545 * if a privileged user specified some ridiculous increment.
2680 2546 */
2681 2547 if (incr > FSS_NICE_MAX)
2682 2548 incr = FSS_NICE_MAX;
2683 2549
2684 2550 newnice = fssproc->fss_nice + incr;
2685 2551 if (newnice > FSS_NICE_MAX)
2686 2552 newnice = FSS_NICE_MAX;
2687 2553 else if (newnice < FSS_NICE_MIN)
2688 2554 newnice = FSS_NICE_MIN;
2689 2555
2690 2556 fssparms.fss_uprilim = fssparms.fss_upri =
2691 2557 -((newnice - NZERO) * fss_maxupri) / NZERO;
2692 2558
2693 2559 /*
2694 2560 * Reset the uprilim and upri values of the thread.
2695 2561 */
2696 2562 (void) fss_parmsset(t, (void *)&fssparms, (id_t)0, (cred_t *)NULL);
2697 2563
2698 2564 /*
2699 2565 * Although fss_parmsset already reset fss_nice it may not have been
2700 2566 * set to precisely the value calculated above because fss_parmsset
2701 2567 * determines the nice value from the user priority and we may have
2702 2568 * truncated during the integer conversion from nice value to user
2703 2569 * priority and back. We reset fss_nice to the value we calculated
2704 2570 * above.
2705 2571 */
2706 2572 fssproc->fss_nice = (char)newnice;
2707 2573
2708 2574 if (retvalp)
2709 2575 *retvalp = newnice - NZERO;
2710 2576 return (0);
2711 2577 }
2712 2578
2713 2579 /*
2714 2580 * Increment the priority of the specified thread by incr and
2715 2581 * return the new value in *retvalp.
2716 2582 */
2717 2583 static int
2718 2584 fss_doprio(kthread_t *t, cred_t *cr, int incr, int *retvalp)
2719 2585 {
2720 2586 int newpri;
2721 2587 fssproc_t *fssproc = FSSPROC(t);
2722 2588 fssparms_t fssparms;
2723 2589
2724 2590 /*
2725 2591 * If there is no change to priority, just return current setting.
2726 2592 */
2727 2593 if (incr == 0) {
2728 2594 *retvalp = fssproc->fss_upri;
2729 2595 return (0);
2730 2596 }
2731 2597
2732 2598 newpri = fssproc->fss_upri + incr;
2733 2599 if (newpri > fss_maxupri || newpri < -fss_maxupri)
2734 2600 return (EINVAL);
2735 2601
2736 2602 *retvalp = newpri;
2737 2603 fssparms.fss_uprilim = fssparms.fss_upri = newpri;
2738 2604
2739 2605 /*
2740 2606 * Reset the uprilim and upri values of the thread.
2741 2607 */
2742 2608 return (fss_parmsset(t, &fssparms, (id_t)0, cr));
2743 2609 }
2744 2610
2745 2611 /*
2746 2612 * Return the global scheduling priority that would be assigned to a thread
2747 2613 * entering the fair-sharing class with the fss_upri.
2748 2614 */
2749 2615 /*ARGSUSED*/
2750 2616 static pri_t
2751 2617 fss_globpri(kthread_t *t)
2752 2618 {
2753 2619 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2754 2620
2755 2621 return (fss_maxumdpri / 2);
2756 2622 }
2757 2623
2758 2624 /*
2759 2625 * Called from the yield(2) system call when a thread is yielding (surrendering)
2760 2626 * the processor. The kernel thread is placed at the back of a dispatch queue.
2761 2627 */
2762 2628 static void
2763 2629 fss_yield(kthread_t *t)
2764 2630 {
2765 2631 fssproc_t *fssproc = FSSPROC(t);
2766 2632
2767 2633 ASSERT(t == curthread);
2768 2634 ASSERT(THREAD_LOCK_HELD(t));
2769 2635
2770 2636 /*
2771 2637 * Collect CPU usage spent before yielding
2772 2638 */
2773 2639 (void) CPUCAPS_CHARGE(t, &fssproc->fss_caps, CPUCAPS_CHARGE_ENFORCE);
2774 2640
2775 2641 /*
2776 2642 * Clear the preemption control "yield" bit since the user is
2777 2643 * doing a yield.
2778 2644 */
2779 2645 if (t->t_schedctl)
2780 2646 schedctl_set_yield(t, 0);
2781 2647 /*
2782 2648 * If fss_preempt() artifically increased the thread's priority
2783 2649 * to avoid preemption, restore the original priority now.
2784 2650 */
2785 2651 if (fssproc->fss_flags & FSSRESTORE) {
2786 2652 THREAD_CHANGE_PRI(t, fssproc->fss_scpri);
2787 2653 fssproc->fss_flags &= ~FSSRESTORE;
2788 2654 }
2789 2655 if (fssproc->fss_timeleft < 0) {
2790 2656 /*
2791 2657 * Time slice was artificially extended to avoid preemption,
2792 2658 * so pretend we're preempting it now.
2793 2659 */
2794 2660 DTRACE_SCHED1(schedctl__yield, int, -fssproc->fss_timeleft);
2795 2661 fssproc->fss_timeleft = fss_quantum;
2796 2662 }
2797 2663 fssproc->fss_flags &= ~FSSBACKQ;
2798 2664 setbackdq(t);
2799 2665 }
2800 2666
2801 2667 void
2802 2668 fss_changeproj(kthread_t *t, void *kp, void *zp, fssbuf_t *projbuf,
2803 2669 fssbuf_t *zonebuf)
2804 2670 {
2805 2671 kproject_t *kpj_new = kp;
2806 2672 zone_t *zone = zp;
2807 2673 fssproj_t *fssproj_old, *fssproj_new;
2808 2674 fsspset_t *fsspset;
2809 2675 kproject_t *kpj_old;
2810 2676 fssproc_t *fssproc;
2811 2677 fsszone_t *fsszone_old, *fsszone_new;
2812 2678 int free = 0;
2813 2679 int id;
2814 2680
2815 2681 ASSERT(MUTEX_HELD(&cpu_lock));
2816 2682 ASSERT(MUTEX_HELD(&pidlock));
2817 2683 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2818 2684
2819 2685 if (t->t_cid != fss_cid)
2820 2686 return;
2821 2687
2822 2688 fssproc = FSSPROC(t);
2823 2689 mutex_enter(&fsspsets_lock);
2824 2690 fssproj_old = FSSPROC2FSSPROJ(fssproc);
2825 2691 if (fssproj_old == NULL) {
2826 2692 mutex_exit(&fsspsets_lock);
2827 2693 return;
2828 2694 }
2829 2695
2830 2696 fsspset = FSSPROJ2FSSPSET(fssproj_old);
2831 2697 mutex_enter(&fsspset->fssps_lock);
2832 2698 kpj_old = FSSPROJ2KPROJ(fssproj_old);
2833 2699 fsszone_old = fssproj_old->fssp_fsszone;
2834 2700
2835 2701 ASSERT(t->t_cpupart == fsspset->fssps_cpupart);
2836 2702
2837 2703 if (kpj_old == kpj_new) {
2838 2704 mutex_exit(&fsspset->fssps_lock);
2839 2705 mutex_exit(&fsspsets_lock);
2840 2706 return;
2841 2707 }
2842 2708
2843 2709 if ((fsszone_new = fss_find_fsszone(fsspset, zone)) == NULL) {
2844 2710 /*
2845 2711 * If the zone for the new project is not currently active on
2846 2712 * the cpu partition we're on, get one of the pre-allocated
2847 2713 * buffers and link it in our per-pset zone list. Such buffers
2848 2714 * should already exist.
2849 2715 */
2850 2716 for (id = 0; id < zonebuf->fssb_size; id++) {
2851 2717 if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
2852 2718 fss_insert_fsszone(fsspset, zone, fsszone_new);
2853 2719 zonebuf->fssb_list[id] = NULL;
2854 2720 break;
2855 2721 }
2856 2722 }
2857 2723 }
2858 2724 ASSERT(fsszone_new != NULL);
2859 2725 if ((fssproj_new = fss_find_fssproj(fsspset, kpj_new)) == NULL) {
2860 2726 /*
2861 2727 * If our new project is not currently running
2862 2728 * on the cpu partition we're on, get one of the
2863 2729 * pre-allocated buffers and link it in our new cpu
2864 2730 * partition doubly linked list. Such buffers should already
2865 2731 * exist.
2866 2732 */
2867 2733 for (id = 0; id < projbuf->fssb_size; id++) {
2868 2734 if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
2869 2735 fss_insert_fssproj(fsspset, kpj_new,
2870 2736 fsszone_new, fssproj_new);
2871 2737 projbuf->fssb_list[id] = NULL;
2872 2738 break;
2873 2739 }
2874 2740 }
2875 2741 }
2876 2742 ASSERT(fssproj_new != NULL);
2877 2743
2878 2744 thread_lock(t);
2879 2745 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2880 2746 t->t_state == TS_WAIT)
2881 2747 fss_inactive(t);
2882 2748 ASSERT(fssproj_old->fssp_threads > 0);
2883 2749 if (--fssproj_old->fssp_threads == 0) {
2884 2750 fss_remove_fssproj(fsspset, fssproj_old);
2885 2751 free = 1;
2886 2752 }
2887 2753 fssproc->fss_proj = fssproj_new;
2888 2754 fssproc->fss_fsspri = 0;
2889 2755 fssproj_new->fssp_threads++;
2890 2756 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2891 2757 t->t_state == TS_WAIT)
2892 2758 fss_active(t);
2893 2759 thread_unlock(t);
2894 2760 if (free) {
2895 2761 if (fsszone_old->fssz_nproj == 0)
2896 2762 kmem_free(fsszone_old, sizeof (fsszone_t));
2897 2763 kmem_free(fssproj_old, sizeof (fssproj_t));
2898 2764 }
2899 2765
2900 2766 mutex_exit(&fsspset->fssps_lock);
2901 2767 mutex_exit(&fsspsets_lock);
2902 2768 }
2903 2769
2904 2770 void
2905 2771 fss_changepset(kthread_t *t, void *newcp, fssbuf_t *projbuf,
2906 2772 fssbuf_t *zonebuf)
2907 2773 {
2908 2774 fsspset_t *fsspset_old, *fsspset_new;
2909 2775 fssproj_t *fssproj_old, *fssproj_new;
2910 2776 fsszone_t *fsszone_old, *fsszone_new;
2911 2777 fssproc_t *fssproc;
2912 2778 kproject_t *kpj;
2913 2779 zone_t *zone;
2914 2780 int id;
2915 2781
2916 2782 ASSERT(MUTEX_HELD(&cpu_lock));
2917 2783 ASSERT(MUTEX_HELD(&pidlock));
2918 2784 ASSERT(MUTEX_HELD(&ttoproc(t)->p_lock));
2919 2785
2920 2786 if (t->t_cid != fss_cid)
2921 2787 return;
2922 2788
2923 2789 fssproc = FSSPROC(t);
2924 2790 zone = ttoproc(t)->p_zone;
2925 2791 mutex_enter(&fsspsets_lock);
2926 2792 fssproj_old = FSSPROC2FSSPROJ(fssproc);
2927 2793 if (fssproj_old == NULL) {
2928 2794 mutex_exit(&fsspsets_lock);
2929 2795 return;
2930 2796 }
2931 2797 fsszone_old = fssproj_old->fssp_fsszone;
2932 2798 fsspset_old = FSSPROJ2FSSPSET(fssproj_old);
2933 2799 kpj = FSSPROJ2KPROJ(fssproj_old);
2934 2800
2935 2801 if (fsspset_old->fssps_cpupart == newcp) {
2936 2802 mutex_exit(&fsspsets_lock);
2937 2803 return;
2938 2804 }
2939 2805
2940 2806 ASSERT(ttoproj(t) == kpj);
2941 2807
2942 2808 fsspset_new = fss_find_fsspset(newcp);
2943 2809
2944 2810 mutex_enter(&fsspset_new->fssps_lock);
2945 2811 if ((fsszone_new = fss_find_fsszone(fsspset_new, zone)) == NULL) {
2946 2812 for (id = 0; id < zonebuf->fssb_size; id++) {
2947 2813 if ((fsszone_new = zonebuf->fssb_list[id]) != NULL) {
2948 2814 fss_insert_fsszone(fsspset_new, zone,
2949 2815 fsszone_new);
2950 2816 zonebuf->fssb_list[id] = NULL;
2951 2817 break;
2952 2818 }
2953 2819 }
2954 2820 }
2955 2821 ASSERT(fsszone_new != NULL);
2956 2822 if ((fssproj_new = fss_find_fssproj(fsspset_new, kpj)) == NULL) {
2957 2823 for (id = 0; id < projbuf->fssb_size; id++) {
2958 2824 if ((fssproj_new = projbuf->fssb_list[id]) != NULL) {
2959 2825 fss_insert_fssproj(fsspset_new, kpj,
2960 2826 fsszone_new, fssproj_new);
2961 2827 projbuf->fssb_list[id] = NULL;
2962 2828 break;
2963 2829 }
2964 2830 }
2965 2831 }
2966 2832 ASSERT(fssproj_new != NULL);
2967 2833
2968 2834 fssproj_new->fssp_threads++;
2969 2835 thread_lock(t);
2970 2836 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2971 2837 t->t_state == TS_WAIT)
2972 2838 fss_inactive(t);
2973 2839 fssproc->fss_proj = fssproj_new;
2974 2840 fssproc->fss_fsspri = 0;
2975 2841 if (t->t_state == TS_RUN || t->t_state == TS_ONPROC ||
2976 2842 t->t_state == TS_WAIT)
2977 2843 fss_active(t);
2978 2844 thread_unlock(t);
2979 2845 mutex_exit(&fsspset_new->fssps_lock);
2980 2846
2981 2847 mutex_enter(&fsspset_old->fssps_lock);
2982 2848 if (--fssproj_old->fssp_threads == 0) {
2983 2849 fss_remove_fssproj(fsspset_old, fssproj_old);
2984 2850 if (fsszone_old->fssz_nproj == 0)
2985 2851 kmem_free(fsszone_old, sizeof (fsszone_t));
2986 2852 kmem_free(fssproj_old, sizeof (fssproj_t));
2987 2853 }
2988 2854 mutex_exit(&fsspset_old->fssps_lock);
2989 2855
2990 2856 mutex_exit(&fsspsets_lock);
2991 2857 }
|
↓ open down ↓ |
364 lines elided |
↑ open up ↑ |
XXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXXX