1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright (c) 2009-2010, Intel Corporation.
  23  * All rights reserved.
  24  */
  25 
  26 /*
  27  * Introduction
  28  * This file implements a CPU event notification mechanism to signal clients
  29  * which are interested in CPU related events.
  30  * Currently it only supports CPU idle state change events which will be
  31  * triggered just before CPU entering hardware idle state and just after CPU
  32  * wakes up from hardware idle state.
  33  * Please refer to PSARC/2009/115 for detail information.
  34  *
  35  * Lock Strategy
  36  * 1) cpu_idle_prop_busy/free are protected by cpu_idle_prop_lock.
  37  * 2) No protection for cpu_idle_cb_state because it's per-CPU data.
  38  * 3) cpu_idle_cb_busy is protected by cpu_idle_cb_lock.
  39  * 4) cpu_idle_cb_array is protected by pause_cpus/start_cpus logic.
  40  * 5) cpu_idle_cb_max/curr are protected by both cpu_idle_cb_lock and
  41  *    pause_cpus/start_cpus logic.
  42  * We have optimized the algorithm for hot path on read side access.
  43  * In the current algorithm, it's lock free on read side access.
  44  * On write side, we use pause_cpus() to keep other CPUs in the pause thread,
  45  * which will guarantee that no other threads will access
  46  * cpu_idle_cb_max/curr/array data structure.
  47  */
  48 
  49 #include <sys/types.h>
  50 #include <sys/cmn_err.h>
  51 #include <sys/cpuvar.h>
  52 #include <sys/cpu.h>
  53 #include <sys/kmem.h>
  54 #include <sys/machcpuvar.h>
  55 #include <sys/sdt.h>
  56 #include <sys/sysmacros.h>
  57 #include <sys/synch.h>
  58 #include <sys/systm.h>
  59 #include <sys/sunddi.h>
  60 #if defined(__sparc)
  61 #include <sys/machsystm.h>
  62 #elif defined(__x86)
  63 #include <sys/archsystm.h>
  64 #endif
  65 #include <sys/cpu_event.h>
  66 
  67 /* Define normal state for CPU on different platforms. */
  68 #if defined(__x86)
  69 #define CPU_IDLE_STATE_NORMAL           IDLE_STATE_C0
  70 #elif defined(__sparc)
  71 /*
  72  * At the time of this implementation IDLE_STATE_NORMAL is defined
  73  * in mach_startup.c, and not in a header file.  So if we find it is
  74  * undefined, then we set it to the value as defined in mach_startup.c
  75  * Should it eventually be defined, we will pick it up.
  76  */
  77 #ifndef IDLE_STATE_NORMAL
  78 #define IDLE_STATE_NORMAL       0
  79 #endif
  80 #define CPU_IDLE_STATE_NORMAL   IDLE_STATE_NORMAL
  81 #endif
  82 
  83 /*
  84  * To improve cache efficiency and avoid cache false sharing, CPU idle
  85  * properties are grouped into cache lines as below:
  86  * |     CPU0      |     CPU1      |.........|     CPUn      |
  87  * | cache line 0  | cache line 1  |.........| cache line n  |
  88  * | v0 | ... | vm | v0 | ... | vm |.........| v0 | ... | vm |
  89  * To access value of property m for CPU n, using following value as index:
  90  *    index = seq_id_of_CPUn * CPU_IDLE_VALUE_GROUP_SIZE + m.
  91  */
  92 #define CPU_IDLE_VALUE_GROUP_SIZE       \
  93         (CPU_CACHE_COHERENCE_SIZE / sizeof (cpu_idle_prop_value_t))
  94 
  95 /* Get callback context handle for current CPU. */
  96 #define CPU_IDLE_GET_CTX(cp)            \
  97         ((cpu_idle_callback_context_t)(intptr_t)((cp)->cpu_seqid))
  98 
  99 /* Get CPU sequential id from ctx. */
 100 #define CPU_IDLE_CTX2CPUID(ctx)         ((processorid_t)(intptr_t)(ctx))
 101 
 102 /* Compute index from callback context handle. */
 103 #define CPU_IDLE_CTX2IDX(ctx)           \
 104         (((int)(intptr_t)(ctx)) * CPU_IDLE_VALUE_GROUP_SIZE)
 105 
 106 #define CPU_IDLE_HDL2VALP(hdl, idx)     \
 107         (&((cpu_idle_prop_impl_t *)(hdl))->value[(idx)])
 108 
 109 /*
 110  * When cpu_idle_cb_array is NULL or full, increase CPU_IDLE_ARRAY_CAPACITY_INC
 111  * entries every time. Here we prefer linear growth instead of exponential.
 112  */
 113 #define CPU_IDLE_ARRAY_CAPACITY_INC     0x10
 114 
 115 typedef struct cpu_idle_prop_impl {
 116         cpu_idle_prop_value_t           *value;
 117         struct cpu_idle_prop_impl       *next;
 118         char                            *name;
 119         cpu_idle_prop_update_t          update;
 120         void                            *private;
 121         cpu_idle_prop_type_t            type;
 122         uint32_t                        refcnt;
 123 } cpu_idle_prop_impl_t;
 124 
 125 typedef struct cpu_idle_prop_item {
 126         cpu_idle_prop_type_t            type;
 127         char                            *name;
 128         cpu_idle_prop_update_t          update;
 129         void                            *arg;
 130         cpu_idle_prop_handle_t          handle;
 131 } cpu_idle_prop_item_t;
 132 
 133 /* Structure to maintain registered callbacks in list. */
 134 typedef struct cpu_idle_cb_impl {
 135         struct cpu_idle_cb_impl         *next;
 136         cpu_idle_callback_t             *callback;
 137         void                            *argument;
 138         int                             priority;
 139 } cpu_idle_cb_impl_t;
 140 
 141 /*
 142  * Structure to maintain registered callbacks in priority order and also
 143  * optimized for cache efficiency for reading access.
 144  */
 145 typedef struct cpu_idle_cb_item {
 146         cpu_idle_enter_cbfn_t           enter;
 147         cpu_idle_exit_cbfn_t            exit;
 148         void                            *arg;
 149         cpu_idle_cb_impl_t              *impl;
 150 } cpu_idle_cb_item_t;
 151 
 152 /* Per-CPU state aligned to CPU_CACHE_COHERENCE_SIZE to avoid false sharing. */
 153 typedef union cpu_idle_cb_state {
 154         struct {
 155                 /* Index of already invoked callbacks. */
 156                 int                     index;
 157                 /* Invoke registered callbacks if true. */
 158                 boolean_t               enabled;
 159                 /* Property values are valid if true. */
 160                 boolean_t               ready;
 161                 /* Pointers to per-CPU properties. */
 162                 cpu_idle_prop_value_t   *idle_state;
 163                 cpu_idle_prop_value_t   *enter_ts;
 164                 cpu_idle_prop_value_t   *exit_ts;
 165                 cpu_idle_prop_value_t   *last_idle;
 166                 cpu_idle_prop_value_t   *last_busy;
 167                 cpu_idle_prop_value_t   *total_idle;
 168                 cpu_idle_prop_value_t   *total_busy;
 169                 cpu_idle_prop_value_t   *intr_cnt;
 170         } v;
 171 #ifdef _LP64
 172         char                            align[2 * CPU_CACHE_COHERENCE_SIZE];
 173 #else
 174         char                            align[CPU_CACHE_COHERENCE_SIZE];
 175 #endif
 176 } cpu_idle_cb_state_t;
 177 
 178 static kmutex_t                         cpu_idle_prop_lock;
 179 static cpu_idle_prop_impl_t             *cpu_idle_prop_busy = NULL;
 180 static cpu_idle_prop_impl_t             *cpu_idle_prop_free = NULL;
 181 
 182 static kmutex_t                         cpu_idle_cb_lock;
 183 static cpu_idle_cb_impl_t               *cpu_idle_cb_busy = NULL;
 184 static cpu_idle_cb_item_t               *cpu_idle_cb_array = NULL;
 185 static int                              cpu_idle_cb_curr = 0;
 186 static int                              cpu_idle_cb_max = 0;
 187 
 188 static cpu_idle_cb_state_t              *cpu_idle_cb_state;
 189 
 190 #ifdef  __x86
 191 /*
 192  * cpuset used to intercept CPUs before powering them off.
 193  * The control CPU sets the bit corresponding to the target CPU and waits
 194  * until the bit is cleared.
 195  * The target CPU disables interrupts before clearing corresponding bit and
 196  * then loops for ever.
 197  */
 198 static cpuset_t                         cpu_idle_intercept_set;
 199 #endif
 200 
 201 static int cpu_idle_prop_update_intr_cnt(void *arg, uint64_t seqnum,
 202     cpu_idle_prop_value_t *valp);
 203 
 204 static cpu_idle_prop_item_t cpu_idle_prop_array[] = {
 205         {
 206             CPU_IDLE_PROP_TYPE_INTPTR, CPU_IDLE_PROP_IDLE_STATE,
 207             NULL, NULL, NULL
 208         },
 209         {
 210             CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_ENTER_TIMESTAMP,
 211             NULL, NULL, NULL
 212         },
 213         {
 214             CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_EXIT_TIMESTAMP,
 215             NULL, NULL, NULL
 216         },
 217         {
 218             CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_LAST_IDLE_TIME,
 219             NULL, NULL, NULL
 220         },
 221         {
 222             CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_LAST_BUSY_TIME,
 223             NULL, NULL, NULL
 224         },
 225         {
 226             CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_TOTAL_IDLE_TIME,
 227             NULL, NULL, NULL
 228         },
 229         {
 230             CPU_IDLE_PROP_TYPE_HRTIME, CPU_IDLE_PROP_TOTAL_BUSY_TIME,
 231             NULL, NULL, NULL
 232         },
 233         {
 234             CPU_IDLE_PROP_TYPE_UINT64, CPU_IDLE_PROP_INTERRUPT_COUNT,
 235             cpu_idle_prop_update_intr_cnt, NULL, NULL
 236         },
 237 };
 238 
 239 #define CPU_IDLE_PROP_IDX_IDLE_STATE    0
 240 #define CPU_IDLE_PROP_IDX_ENTER_TS      1
 241 #define CPU_IDLE_PROP_IDX_EXIT_TS       2
 242 #define CPU_IDLE_PROP_IDX_LAST_IDLE     3
 243 #define CPU_IDLE_PROP_IDX_LAST_BUSY     4
 244 #define CPU_IDLE_PROP_IDX_TOTAL_IDLE    5
 245 #define CPU_IDLE_PROP_IDX_TOTAL_BUSY    6
 246 #define CPU_IDLE_PROP_IDX_INTR_CNT      7
 247 
 248 /*ARGSUSED*/
 249 static void
 250 cpu_idle_dtrace_enter(void *arg, cpu_idle_callback_context_t ctx,
 251     cpu_idle_check_wakeup_t check_func, void *check_arg)
 252 {
 253         int state;
 254 
 255         state = cpu_idle_prop_get_intptr(
 256             cpu_idle_prop_array[CPU_IDLE_PROP_IDX_IDLE_STATE].handle, ctx);
 257         DTRACE_PROBE1(idle__state__transition, uint_t, state);
 258 }
 259 
 260 /*ARGSUSED*/
 261 static void
 262 cpu_idle_dtrace_exit(void *arg, cpu_idle_callback_context_t ctx, int flag)
 263 {
 264         DTRACE_PROBE1(idle__state__transition, uint_t, CPU_IDLE_STATE_NORMAL);
 265 }
 266 
 267 static cpu_idle_callback_handle_t cpu_idle_cb_handle_dtrace;
 268 static cpu_idle_callback_t cpu_idle_callback_dtrace = {
 269         CPU_IDLE_CALLBACK_VERS,
 270         cpu_idle_dtrace_enter,
 271         cpu_idle_dtrace_exit,
 272 };
 273 
 274 #if defined(__x86) && !defined(__xpv)
 275 extern void tlb_going_idle(void);
 276 extern void tlb_service(void);
 277 
 278 static cpu_idle_callback_handle_t cpu_idle_cb_handle_tlb;
 279 static cpu_idle_callback_t cpu_idle_callback_tlb = {
 280         CPU_IDLE_CALLBACK_VERS,
 281         (cpu_idle_enter_cbfn_t)tlb_going_idle,
 282         (cpu_idle_exit_cbfn_t)tlb_service,
 283 };
 284 #endif
 285 
 286 void
 287 cpu_event_init(void)
 288 {
 289         int i, idx;
 290         size_t sz;
 291         intptr_t buf;
 292         cpu_idle_cb_state_t *sp;
 293         cpu_idle_prop_item_t *ip;
 294 
 295         mutex_init(&cpu_idle_cb_lock, NULL, MUTEX_DRIVER, NULL);
 296         mutex_init(&cpu_idle_prop_lock, NULL, MUTEX_DRIVER, NULL);
 297 
 298         /* Create internal properties. */
 299         for (i = 0, ip = cpu_idle_prop_array;
 300             i < sizeof (cpu_idle_prop_array) / sizeof (cpu_idle_prop_array[0]);
 301             i++, ip++) {
 302                 (void) cpu_idle_prop_create_property(ip->name, ip->type,
 303                     ip->update, ip->arg, &ip->handle);
 304                 ASSERT(ip->handle != NULL);
 305         }
 306 
 307         /* Allocate buffer and align to CPU_CACHE_COHERENCE_SIZE. */
 308         sz = sizeof (cpu_idle_cb_state_t) * max_ncpus;
 309         sz += CPU_CACHE_COHERENCE_SIZE;
 310         buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
 311         cpu_idle_cb_state = (cpu_idle_cb_state_t *)P2ROUNDUP(buf,
 312             CPU_CACHE_COHERENCE_SIZE);
 313 
 314         /* Cache frequently used property value pointers. */
 315         for (sp = cpu_idle_cb_state, i = 0; i < max_ncpus; i++, sp++) {
 316                 idx = CPU_IDLE_CTX2IDX(i);
 317 #define ___INIT_P(f, i) \
 318         sp->v.f = CPU_IDLE_HDL2VALP(cpu_idle_prop_array[(i)].handle, idx)
 319                 ___INIT_P(idle_state, CPU_IDLE_PROP_IDX_IDLE_STATE);
 320                 ___INIT_P(enter_ts, CPU_IDLE_PROP_IDX_ENTER_TS);
 321                 ___INIT_P(exit_ts, CPU_IDLE_PROP_IDX_EXIT_TS);
 322                 ___INIT_P(last_idle, CPU_IDLE_PROP_IDX_LAST_IDLE);
 323                 ___INIT_P(last_busy, CPU_IDLE_PROP_IDX_LAST_BUSY);
 324                 ___INIT_P(total_idle, CPU_IDLE_PROP_IDX_TOTAL_IDLE);
 325                 ___INIT_P(total_busy, CPU_IDLE_PROP_IDX_TOTAL_BUSY);
 326                 ___INIT_P(last_idle, CPU_IDLE_PROP_IDX_INTR_CNT);
 327 #undef  ___INIT_P
 328         }
 329 
 330         /* Register built-in callbacks. */
 331         if (cpu_idle_register_callback(CPU_IDLE_CB_PRIO_DTRACE,
 332             &cpu_idle_callback_dtrace, NULL, &cpu_idle_cb_handle_dtrace) != 0) {
 333                 cmn_err(CE_PANIC,
 334                     "cpu_idle: failed to register callback for dtrace.");
 335         }
 336 #if defined(__x86) && !defined(__xpv)
 337         if (cpu_idle_register_callback(CPU_IDLE_CB_PRIO_TLB,
 338             &cpu_idle_callback_tlb, NULL, &cpu_idle_cb_handle_tlb) != 0) {
 339                 cmn_err(CE_PANIC,
 340                     "cpu_idle: failed to register callback for tlb_flush.");
 341         }
 342 #endif
 343 }
 344 
 345 /*
 346  * This function is called to initialize per CPU state when starting CPUs.
 347  */
 348 void
 349 cpu_event_init_cpu(cpu_t *cp)
 350 {
 351         ASSERT(cp->cpu_seqid < max_ncpus);
 352         cpu_idle_cb_state[cp->cpu_seqid].v.index = 0;
 353         cpu_idle_cb_state[cp->cpu_seqid].v.ready = B_FALSE;
 354         cpu_idle_cb_state[cp->cpu_seqid].v.enabled = B_TRUE;
 355 }
 356 
 357 /*
 358  * This function is called to clean up per CPU state when stopping CPUs.
 359  */
 360 void
 361 cpu_event_fini_cpu(cpu_t *cp)
 362 {
 363         ASSERT(cp->cpu_seqid < max_ncpus);
 364         cpu_idle_cb_state[cp->cpu_seqid].v.enabled = B_FALSE;
 365         cpu_idle_cb_state[cp->cpu_seqid].v.ready = B_FALSE;
 366 }
 367 
 368 static void
 369 cpu_idle_insert_callback(cpu_idle_cb_impl_t *cip)
 370 {
 371         int unlock = 0, unpause = 0;
 372         int i, cnt_new = 0, cnt_old = 0;
 373         char *buf_new = NULL, *buf_old = NULL;
 374 
 375         ASSERT(MUTEX_HELD(&cpu_idle_cb_lock));
 376 
 377         /*
 378          * Expand array if it's full.
 379          * Memory must be allocated out of pause/start_cpus() scope because
 380          * kmem_zalloc() can't be called with KM_SLEEP flag within that scope.
 381          */
 382         if (cpu_idle_cb_curr == cpu_idle_cb_max) {
 383                 cnt_new = cpu_idle_cb_max + CPU_IDLE_ARRAY_CAPACITY_INC;
 384                 buf_new = (char *)kmem_zalloc(cnt_new *
 385                     sizeof (cpu_idle_cb_item_t), KM_SLEEP);
 386         }
 387 
 388         /* Try to acquire cpu_lock if not held yet. */
 389         if (!MUTEX_HELD(&cpu_lock)) {
 390                 mutex_enter(&cpu_lock);
 391                 unlock = 1;
 392         }
 393         /*
 394          * Pause all other CPUs (and let them run pause thread).
 395          * It's guaranteed that no other threads will access cpu_idle_cb_array
 396          * after pause_cpus().
 397          */
 398         if (!cpus_paused()) {
 399                 pause_cpus(NULL, NULL);
 400                 unpause = 1;
 401         }
 402 
 403         /* Copy content to new buffer if needed. */
 404         if (buf_new != NULL) {
 405                 buf_old = (char *)cpu_idle_cb_array;
 406                 cnt_old = cpu_idle_cb_max;
 407                 if (buf_old != NULL) {
 408                         ASSERT(cnt_old != 0);
 409                         bcopy(cpu_idle_cb_array, buf_new,
 410                             sizeof (cpu_idle_cb_item_t) * cnt_old);
 411                 }
 412                 cpu_idle_cb_array = (cpu_idle_cb_item_t *)buf_new;
 413                 cpu_idle_cb_max = cnt_new;
 414         }
 415 
 416         /* Insert into array according to priority. */
 417         ASSERT(cpu_idle_cb_curr < cpu_idle_cb_max);
 418         for (i = cpu_idle_cb_curr; i > 0; i--) {
 419                 if (cpu_idle_cb_array[i - 1].impl->priority >= cip->priority) {
 420                         break;
 421                 }
 422                 cpu_idle_cb_array[i] = cpu_idle_cb_array[i - 1];
 423         }
 424         cpu_idle_cb_array[i].arg = cip->argument;
 425         cpu_idle_cb_array[i].enter = cip->callback->idle_enter;
 426         cpu_idle_cb_array[i].exit = cip->callback->idle_exit;
 427         cpu_idle_cb_array[i].impl = cip;
 428         cpu_idle_cb_curr++;
 429 
 430         /* Resume other CPUs from paused state if needed. */
 431         if (unpause) {
 432                 start_cpus();
 433         }
 434         if (unlock) {
 435                 mutex_exit(&cpu_lock);
 436         }
 437 
 438         /* Free old resource if needed. */
 439         if (buf_old != NULL) {
 440                 ASSERT(cnt_old != 0);
 441                 kmem_free(buf_old, cnt_old * sizeof (cpu_idle_cb_item_t));
 442         }
 443 }
 444 
 445 static void
 446 cpu_idle_remove_callback(cpu_idle_cb_impl_t *cip)
 447 {
 448         int i, found = 0;
 449         int unlock = 0, unpause = 0;
 450         cpu_idle_cb_state_t *sp;
 451 
 452         ASSERT(MUTEX_HELD(&cpu_idle_cb_lock));
 453 
 454         /* Try to acquire cpu_lock if not held yet. */
 455         if (!MUTEX_HELD(&cpu_lock)) {
 456                 mutex_enter(&cpu_lock);
 457                 unlock = 1;
 458         }
 459         /*
 460          * Pause all other CPUs.
 461          * It's guaranteed that no other threads will access cpu_idle_cb_array
 462          * after pause_cpus().
 463          */
 464         if (!cpus_paused()) {
 465                 pause_cpus(NULL, NULL);
 466                 unpause = 1;
 467         }
 468 
 469         /* Remove cip from array. */
 470         for (i = 0; i < cpu_idle_cb_curr; i++) {
 471                 if (found == 0) {
 472                         if (cpu_idle_cb_array[i].impl == cip) {
 473                                 found = 1;
 474                         }
 475                 } else {
 476                         cpu_idle_cb_array[i - 1] = cpu_idle_cb_array[i];
 477                 }
 478         }
 479         ASSERT(found != 0);
 480         cpu_idle_cb_curr--;
 481 
 482         /*
 483          * Reset property ready flag for all CPUs if no registered callback
 484          * left because cpu_idle_enter/exit will stop updating property if
 485          * there's no callback registered.
 486          */
 487         if (cpu_idle_cb_curr == 0) {
 488                 for (sp = cpu_idle_cb_state, i = 0; i < max_ncpus; i++, sp++) {
 489                         sp->v.ready = B_FALSE;
 490                 }
 491         }
 492 
 493         /* Resume other CPUs from paused state if needed. */
 494         if (unpause) {
 495                 start_cpus();
 496         }
 497         if (unlock) {
 498                 mutex_exit(&cpu_lock);
 499         }
 500 }
 501 
 502 int
 503 cpu_idle_register_callback(uint_t prio, cpu_idle_callback_t *cbp,
 504     void *arg, cpu_idle_callback_handle_t *hdlp)
 505 {
 506         cpu_idle_cb_state_t *sp;
 507         cpu_idle_cb_impl_t *cip = NULL;
 508 
 509         /* First validate parameters. */
 510         ASSERT(!CPU_ON_INTR(CPU));
 511         ASSERT(CPU->cpu_seqid < max_ncpus);
 512         sp = &cpu_idle_cb_state[CPU->cpu_seqid];
 513         if (sp->v.index != 0) {
 514                 cmn_err(CE_NOTE,
 515                     "!cpu_event: register_callback called from callback.");
 516                 return (EBUSY);
 517         } else if (cbp == NULL || hdlp == NULL) {
 518                 cmn_err(CE_NOTE,
 519                     "!cpu_event: NULL parameters in register_callback.");
 520                 return (EINVAL);
 521         } else if (prio < CPU_IDLE_CB_PRIO_LOW_BASE ||
 522             prio >= CPU_IDLE_CB_PRIO_RESV_BASE) {
 523                 cmn_err(CE_NOTE,
 524                     "!cpu_event: priority 0x%x out of range.", prio);
 525                 return (EINVAL);
 526         } else if (cbp->version != CPU_IDLE_CALLBACK_VERS) {
 527                 cmn_err(CE_NOTE,
 528                     "!cpu_event: callback version %d is not supported.",
 529                     cbp->version);
 530                 return (EINVAL);
 531         }
 532 
 533         mutex_enter(&cpu_idle_cb_lock);
 534         /* Check whether callback with priority exists if not dynamic. */
 535         if (prio != CPU_IDLE_CB_PRIO_DYNAMIC) {
 536                 for (cip = cpu_idle_cb_busy; cip != NULL;
 537                     cip = cip->next) {
 538                         if (cip->priority == prio) {
 539                                 mutex_exit(&cpu_idle_cb_lock);
 540                                 cmn_err(CE_NOTE, "!cpu_event: callback with "
 541                                     "priority 0x%x already exists.", prio);
 542                                 return (EEXIST);
 543                         }
 544                 }
 545         }
 546 
 547         cip = kmem_zalloc(sizeof (*cip), KM_SLEEP);
 548         cip->callback = cbp;
 549         cip->argument = arg;
 550         cip->priority = prio;
 551         cip->next = cpu_idle_cb_busy;
 552         cpu_idle_cb_busy = cip;
 553         cpu_idle_insert_callback(cip);
 554         mutex_exit(&cpu_idle_cb_lock);
 555 
 556         *hdlp = (cpu_idle_callback_handle_t)cip;
 557 
 558         return (0);
 559 }
 560 
 561 int
 562 cpu_idle_unregister_callback(cpu_idle_callback_handle_t hdl)
 563 {
 564         int rc = ENODEV;
 565         cpu_idle_cb_state_t *sp;
 566         cpu_idle_cb_impl_t *ip, **ipp;
 567 
 568         ASSERT(!CPU_ON_INTR(CPU));
 569         ASSERT(CPU->cpu_seqid < max_ncpus);
 570         sp = &cpu_idle_cb_state[CPU->cpu_seqid];
 571         if (sp->v.index != 0) {
 572                 cmn_err(CE_NOTE,
 573                     "!cpu_event: unregister_callback called from callback.");
 574                 return (EBUSY);
 575         } else if (hdl == NULL) {
 576                 cmn_err(CE_NOTE,
 577                     "!cpu_event: hdl is NULL in unregister_callback.");
 578                 return (EINVAL);
 579         }
 580 
 581         ip = (cpu_idle_cb_impl_t *)hdl;
 582         mutex_enter(&cpu_idle_cb_lock);
 583         for (ipp = &cpu_idle_cb_busy; *ipp != NULL; ipp = &(*ipp)->next) {
 584                 if (*ipp == ip) {
 585                         *ipp = ip->next;
 586                         cpu_idle_remove_callback(ip);
 587                         rc = 0;
 588                         break;
 589                 }
 590         }
 591         mutex_exit(&cpu_idle_cb_lock);
 592 
 593         if (rc == 0) {
 594                 kmem_free(ip, sizeof (*ip));
 595         } else {
 596                 cmn_err(CE_NOTE,
 597                     "!cpu_event: callback handle %p not found.", (void *)hdl);
 598         }
 599 
 600         return (rc);
 601 }
 602 
 603 static int
 604 cpu_idle_enter_state(cpu_idle_cb_state_t *sp, intptr_t state)
 605 {
 606         sp->v.idle_state->cipv_intptr = state;
 607         sp->v.enter_ts->cipv_hrtime = gethrtime_unscaled();
 608         sp->v.last_busy->cipv_hrtime = sp->v.enter_ts->cipv_hrtime -
 609             sp->v.exit_ts->cipv_hrtime;
 610         sp->v.total_busy->cipv_hrtime += sp->v.last_busy->cipv_hrtime;
 611         if (sp->v.ready == B_FALSE) {
 612                 sp->v.ready = B_TRUE;
 613                 return (0);
 614         }
 615 
 616         return (1);
 617 }
 618 
 619 static void
 620 cpu_idle_exit_state(cpu_idle_cb_state_t *sp)
 621 {
 622         sp->v.idle_state->cipv_intptr = CPU_IDLE_STATE_NORMAL;
 623         sp->v.exit_ts->cipv_hrtime = gethrtime_unscaled();
 624         sp->v.last_idle->cipv_hrtime = sp->v.exit_ts->cipv_hrtime -
 625             sp->v.enter_ts->cipv_hrtime;
 626         sp->v.total_idle->cipv_hrtime += sp->v.last_idle->cipv_hrtime;
 627 }
 628 
 629 /*ARGSUSED*/
 630 int
 631 cpu_idle_enter(int state, int flag,
 632     cpu_idle_check_wakeup_t check_func, void *check_arg)
 633 {
 634         int i;
 635         cpu_idle_cb_item_t *cip;
 636         cpu_idle_cb_state_t *sp;
 637         cpu_idle_callback_context_t ctx;
 638 #if defined(__x86)
 639         ulong_t iflags;
 640 #endif
 641 
 642         ctx = CPU_IDLE_GET_CTX(CPU);
 643         ASSERT(CPU->cpu_seqid < max_ncpus);
 644         sp = &cpu_idle_cb_state[CPU->cpu_seqid];
 645         ASSERT(sp->v.index == 0);
 646         if (sp->v.enabled == B_FALSE) {
 647 #if defined(__x86)
 648                 /* Intercept CPU at a safe point before powering off it. */
 649                 if (CPU_IN_SET(cpu_idle_intercept_set, CPU->cpu_id)) {
 650                         iflags = intr_clear();
 651                         CPUSET_ATOMIC_DEL(cpu_idle_intercept_set, CPU->cpu_id);
 652                         /*CONSTCOND*/
 653                         while (1) {
 654                                 SMT_PAUSE();
 655                         }
 656                 }
 657 #endif
 658 
 659                 return (0);
 660         }
 661 
 662         /*
 663          * On x86, cpu_idle_enter can be called from idle thread with either
 664          * interrupts enabled or disabled, so we need to make sure interrupts
 665          * are disabled here.
 666          * On SPARC, cpu_idle_enter will be called from idle thread with
 667          * interrupt disabled, so no special handling necessary.
 668          */
 669 #if defined(__x86)
 670         iflags = intr_clear();
 671 #endif
 672 
 673         /* Skip calling callback if state is not ready for current CPU. */
 674         if (cpu_idle_enter_state(sp, state) == 0) {
 675 #if defined(__x86)
 676                 intr_restore(iflags);
 677 #endif
 678                 return (0);
 679         }
 680 
 681         for (i = 0, cip = cpu_idle_cb_array; i < cpu_idle_cb_curr; i++, cip++) {
 682                 /*
 683                  * Increase index so corresponding idle_exit callback
 684                  * will be invoked should interrupt happen during
 685                  * idle_enter callback.
 686                  */
 687                 sp->v.index++;
 688 
 689                 /* Call idle_enter callback function if it's not NULL. */
 690                 if (cip->enter != NULL) {
 691                         cip->enter(cip->arg, ctx, check_func, check_arg);
 692 
 693                         /*
 694                          * cpu_idle_enter runs with interrupts
 695                          * disabled, so the idle_enter callbacks will
 696                          * also be called with interrupts disabled.
 697                          * It is permissible for the callbacks to
 698                          * enable the interrupts, if they can also
 699                          * handle the condition if the interrupt
 700                          * occurs.
 701                          *
 702                          * However, if an interrupt occurs and we
 703                          * return here without dealing with it, we
 704                          * return to the cpu_idle_enter() caller
 705                          * with an EBUSY, and the caller will not
 706                          * enter the idle state.
 707                          *
 708                          * We detect the interrupt, by checking the
 709                          * index value of the state pointer.  If it
 710                          * is not the index we incremented above,
 711                          * then it was cleared while processing
 712                          * the interrupt.
 713                          *
 714                          * Also note, that at this point of the code
 715                          * the normal index value will be one greater
 716                          * than the variable 'i' in the loop, as it
 717                          * hasn't yet been incremented.
 718                          */
 719                         if (sp->v.index != i + 1) {
 720 #if defined(__x86)
 721                                 intr_restore(iflags);
 722 #endif
 723                                 return (EBUSY);
 724                         }
 725                 }
 726         }
 727 #if defined(__x86)
 728         intr_restore(iflags);
 729 #endif
 730 
 731         return (0);
 732 }
 733 
 734 void
 735 cpu_idle_exit(int flag)
 736 {
 737         int i;
 738         cpu_idle_cb_item_t *cip;
 739         cpu_idle_cb_state_t *sp;
 740         cpu_idle_callback_context_t ctx;
 741 #if defined(__x86)
 742         ulong_t iflags;
 743 #endif
 744 
 745         ASSERT(CPU->cpu_seqid < max_ncpus);
 746         sp = &cpu_idle_cb_state[CPU->cpu_seqid];
 747 
 748 #if defined(__sparc)
 749         /*
 750          * On SPARC, cpu_idle_exit will only be called from idle thread
 751          * with interrupt disabled.
 752          */
 753 
 754         if (sp->v.index != 0) {
 755                 ctx = CPU_IDLE_GET_CTX(CPU);
 756                 cpu_idle_exit_state(sp);
 757                 for (i = sp->v.index - 1; i >= 0; i--) {
 758                         cip = &cpu_idle_cb_array[i];
 759                         if (cip->exit != NULL) {
 760                                 cip->exit(cip->arg, ctx, flag);
 761                         }
 762                 }
 763                 sp->v.index = 0;
 764         }
 765 #elif defined(__x86)
 766         /*
 767          * On x86, cpu_idle_exit will be called from idle thread or interrupt
 768          * handler. When called from interrupt handler, interrupts will be
 769          * disabled. When called from idle thread, interrupts may be disabled
 770          * or enabled.
 771          */
 772 
 773         /* Called from interrupt, interrupts are already disabled. */
 774         if (flag & CPU_IDLE_CB_FLAG_INTR) {
 775                 /*
 776                  * return if cpu_idle_exit already called or
 777                  * there is no registered callback.
 778                  */
 779                 if (sp->v.index == 0) {
 780                         return;
 781                 }
 782                 ctx = CPU_IDLE_GET_CTX(CPU);
 783                 cpu_idle_exit_state(sp);
 784                 for (i = sp->v.index - 1; i >= 0; i--) {
 785                         cip = &cpu_idle_cb_array[i];
 786                         if (cip->exit != NULL) {
 787                                 cip->exit(cip->arg, ctx, flag);
 788                         }
 789                 }
 790                 sp->v.index = 0;
 791 
 792         /* Called from idle thread, need to disable interrupt. */
 793         } else {
 794                 iflags = intr_clear();
 795                 if (sp->v.index != 0) {
 796                         ctx = CPU_IDLE_GET_CTX(CPU);
 797                         cpu_idle_exit_state(sp);
 798                         for (i = sp->v.index - 1; i >= 0; i--) {
 799                                 cip = &cpu_idle_cb_array[i];
 800                                 if (cip->exit != NULL) {
 801                                         cip->exit(cip->arg, ctx, flag);
 802                                 }
 803                         }
 804                         sp->v.index = 0;
 805                 }
 806                 intr_restore(iflags);
 807         }
 808 #endif
 809 }
 810 
 811 cpu_idle_callback_context_t
 812 cpu_idle_get_context(void)
 813 {
 814         return (CPU_IDLE_GET_CTX(CPU));
 815 }
 816 
 817 /*
 818  * Allocate property structure in group of CPU_IDLE_VALUE_GROUP_SIZE to improve
 819  * cache efficiency. To simplify implementation, allocated memory for property
 820  * structure won't be freed.
 821  */
 822 static void
 823 cpu_idle_prop_allocate_impl(void)
 824 {
 825         int i;
 826         size_t sz;
 827         intptr_t buf;
 828         cpu_idle_prop_impl_t *prop;
 829         cpu_idle_prop_value_t *valp;
 830 
 831         ASSERT(!CPU_ON_INTR(CPU));
 832         prop = kmem_zalloc(sizeof (*prop) * CPU_IDLE_VALUE_GROUP_SIZE,
 833             KM_SLEEP);
 834         sz = sizeof (*valp) * CPU_IDLE_VALUE_GROUP_SIZE * max_ncpus;
 835         sz += CPU_CACHE_COHERENCE_SIZE;
 836         buf = (intptr_t)kmem_zalloc(sz, KM_SLEEP);
 837         valp = (cpu_idle_prop_value_t *)P2ROUNDUP(buf,
 838             CPU_CACHE_COHERENCE_SIZE);
 839 
 840         for (i = 0; i < CPU_IDLE_VALUE_GROUP_SIZE; i++, prop++, valp++) {
 841                 prop->value = valp;
 842                 prop->next = cpu_idle_prop_free;
 843                 cpu_idle_prop_free = prop;
 844         }
 845 }
 846 
 847 int
 848 cpu_idle_prop_create_property(const char *name, cpu_idle_prop_type_t type,
 849     cpu_idle_prop_update_t update, void *arg, cpu_idle_prop_handle_t *hdlp)
 850 {
 851         int rc = EEXIST;
 852         cpu_idle_prop_impl_t *prop;
 853 
 854         ASSERT(!CPU_ON_INTR(CPU));
 855         if (name == NULL || hdlp == NULL) {
 856                 cmn_err(CE_WARN,
 857                     "!cpu_event: NULL parameters in create_property.");
 858                 return (EINVAL);
 859         }
 860 
 861         mutex_enter(&cpu_idle_prop_lock);
 862         for (prop = cpu_idle_prop_busy; prop != NULL; prop = prop->next) {
 863                 if (strcmp(prop->name, name) == 0) {
 864                         cmn_err(CE_NOTE,
 865                             "!cpu_event: property %s already exists.", name);
 866                         break;
 867                 }
 868         }
 869         if (prop == NULL) {
 870                 if (cpu_idle_prop_free == NULL) {
 871                         cpu_idle_prop_allocate_impl();
 872                 }
 873                 ASSERT(cpu_idle_prop_free != NULL);
 874                 prop = cpu_idle_prop_free;
 875                 cpu_idle_prop_free = prop->next;
 876                 prop->next = cpu_idle_prop_busy;
 877                 cpu_idle_prop_busy = prop;
 878 
 879                 ASSERT(prop->value != NULL);
 880                 prop->name = strdup(name);
 881                 prop->type = type;
 882                 prop->update = update;
 883                 prop->private = arg;
 884                 prop->refcnt = 1;
 885                 *hdlp = prop;
 886                 rc = 0;
 887         }
 888         mutex_exit(&cpu_idle_prop_lock);
 889 
 890         return (rc);
 891 }
 892 
 893 int
 894 cpu_idle_prop_destroy_property(cpu_idle_prop_handle_t hdl)
 895 {
 896         int rc = ENODEV;
 897         cpu_idle_prop_impl_t *prop, **propp;
 898         cpu_idle_prop_value_t *valp;
 899 
 900         ASSERT(!CPU_ON_INTR(CPU));
 901         if (hdl == NULL) {
 902                 cmn_err(CE_WARN,
 903                     "!cpu_event: hdl is NULL in destroy_property.");
 904                 return (EINVAL);
 905         }
 906 
 907         prop = (cpu_idle_prop_impl_t *)hdl;
 908         mutex_enter(&cpu_idle_prop_lock);
 909         for (propp = &cpu_idle_prop_busy; *propp != NULL;
 910             propp = &(*propp)->next) {
 911                 if (*propp == prop) {
 912                         ASSERT(prop->refcnt > 0);
 913                         if (atomic_cas_32(&prop->refcnt, 1, 0) == 1) {
 914                                 *propp = prop->next;
 915                                 strfree(prop->name);
 916                                 valp = prop->value;
 917                                 bzero(prop, sizeof (*prop));
 918                                 prop->value = valp;
 919                                 prop->next = cpu_idle_prop_free;
 920                                 cpu_idle_prop_free = prop;
 921                                 rc = 0;
 922                         } else {
 923                                 rc = EBUSY;
 924                         }
 925                         break;
 926                 }
 927         }
 928         mutex_exit(&cpu_idle_prop_lock);
 929 
 930         return (rc);
 931 }
 932 
 933 int
 934 cpu_idle_prop_create_handle(const char *name, cpu_idle_prop_handle_t *hdlp)
 935 {
 936         int rc = ENODEV;
 937         cpu_idle_prop_impl_t *prop;
 938 
 939         ASSERT(!CPU_ON_INTR(CPU));
 940         if (name == NULL || hdlp == NULL) {
 941                 cmn_err(CE_WARN,
 942                     "!cpu_event: NULL parameters in create_handle.");
 943                 return (EINVAL);
 944         }
 945 
 946         mutex_enter(&cpu_idle_prop_lock);
 947         for (prop = cpu_idle_prop_busy; prop != NULL; prop = prop->next) {
 948                 if (strcmp(prop->name, name) == 0) {
 949                         /* Hold one refcount on object. */
 950                         ASSERT(prop->refcnt > 0);
 951                         atomic_inc_32(&prop->refcnt);
 952                         *hdlp = (cpu_idle_prop_handle_t)prop;
 953                         rc = 0;
 954                         break;
 955                 }
 956         }
 957         mutex_exit(&cpu_idle_prop_lock);
 958 
 959         return (rc);
 960 }
 961 
 962 int
 963 cpu_idle_prop_destroy_handle(cpu_idle_prop_handle_t hdl)
 964 {
 965         int rc = ENODEV;
 966         cpu_idle_prop_impl_t *prop;
 967 
 968         ASSERT(!CPU_ON_INTR(CPU));
 969         if (hdl == NULL) {
 970                 cmn_err(CE_WARN,
 971                     "!cpu_event: hdl is NULL in destroy_handle.");
 972                 return (EINVAL);
 973         }
 974 
 975         mutex_enter(&cpu_idle_prop_lock);
 976         for (prop = cpu_idle_prop_busy; prop != NULL; prop = prop->next) {
 977                 if (prop == hdl) {
 978                         /* Release refcnt held in create_handle. */
 979                         ASSERT(prop->refcnt > 1);
 980                         atomic_dec_32(&prop->refcnt);
 981                         rc = 0;
 982                         break;
 983                 }
 984         }
 985         mutex_exit(&cpu_idle_prop_lock);
 986 
 987         return (rc);
 988 }
 989 
 990 cpu_idle_prop_type_t
 991 cpu_idle_prop_get_type(cpu_idle_prop_handle_t hdl)
 992 {
 993         ASSERT(hdl != NULL);
 994         return (((cpu_idle_prop_impl_t *)hdl)->type);
 995 }
 996 
 997 const char *
 998 cpu_idle_prop_get_name(cpu_idle_prop_handle_t hdl)
 999 {
1000         ASSERT(hdl != NULL);
1001         return (((cpu_idle_prop_impl_t *)hdl)->name);
1002 }
1003 
1004 int
1005 cpu_idle_prop_get_value(cpu_idle_prop_handle_t hdl,
1006     cpu_idle_callback_context_t ctx, cpu_idle_prop_value_t *valp)
1007 {
1008         int idx, rc = 0;
1009         cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
1010 
1011         ASSERT(CPU_IDLE_CTX2CPUID(ctx) < max_ncpus);
1012         if (hdl == NULL || valp == NULL) {
1013                 cmn_err(CE_NOTE, "!cpu_event: NULL parameters in prop_get.");
1014                 return (EINVAL);
1015         }
1016         idx = CPU_IDLE_CTX2IDX(ctx);
1017         if (prop->update != NULL) {
1018                 cpu_idle_cb_state_t *sp;
1019 
1020                 ASSERT(CPU->cpu_seqid < max_ncpus);
1021                 sp = &cpu_idle_cb_state[CPU->cpu_seqid];
1022                 /* CPU's idle enter timestamp as sequence number. */
1023                 rc = prop->update(prop->private,
1024                     (uint64_t)sp->v.enter_ts->cipv_hrtime, &prop->value[idx]);
1025         }
1026         if (rc == 0) {
1027                 *valp = prop->value[idx];
1028         }
1029 
1030         return (rc);
1031 }
1032 
1033 uint32_t
1034 cpu_idle_prop_get_uint32(cpu_idle_prop_handle_t hdl,
1035     cpu_idle_callback_context_t ctx)
1036 {
1037         int idx;
1038         cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
1039 
1040         ASSERT(hdl != NULL);
1041         ASSERT(CPU_IDLE_CTX2CPUID(ctx) < max_ncpus);
1042         idx = CPU_IDLE_CTX2IDX(ctx);
1043         return (prop->value[idx].cipv_uint32);
1044 }
1045 
1046 uint64_t
1047 cpu_idle_prop_get_uint64(cpu_idle_prop_handle_t hdl,
1048     cpu_idle_callback_context_t ctx)
1049 {
1050         int idx;
1051         cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
1052 
1053         ASSERT(hdl != NULL);
1054         ASSERT(CPU_IDLE_CTX2CPUID(ctx) < max_ncpus);
1055         idx = CPU_IDLE_CTX2IDX(ctx);
1056         return (prop->value[idx].cipv_uint64);
1057 }
1058 
1059 intptr_t
1060 cpu_idle_prop_get_intptr(cpu_idle_prop_handle_t hdl,
1061     cpu_idle_callback_context_t ctx)
1062 {
1063         int idx;
1064         cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
1065 
1066         ASSERT(hdl != NULL);
1067         ASSERT(CPU_IDLE_CTX2CPUID(ctx) < max_ncpus);
1068         idx = CPU_IDLE_CTX2IDX(ctx);
1069         return (prop->value[idx].cipv_intptr);
1070 }
1071 
1072 hrtime_t
1073 cpu_idle_prop_get_hrtime(cpu_idle_prop_handle_t hdl,
1074     cpu_idle_callback_context_t ctx)
1075 {
1076         int idx;
1077         cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
1078 
1079         ASSERT(hdl != NULL);
1080         ASSERT(CPU_IDLE_CTX2CPUID(ctx) < max_ncpus);
1081         idx = CPU_IDLE_CTX2IDX(ctx);
1082         return (prop->value[idx].cipv_hrtime);
1083 }
1084 
1085 void
1086 cpu_idle_prop_set_value(cpu_idle_prop_handle_t hdl,
1087     cpu_idle_callback_context_t ctx, cpu_idle_prop_value_t val)
1088 {
1089         int idx;
1090         cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
1091 
1092         ASSERT(hdl != NULL);
1093         ASSERT(CPU_IDLE_CTX2CPUID(ctx) < max_ncpus);
1094         idx = CPU_IDLE_CTX2IDX(ctx);
1095         prop->value[idx] = val;
1096 }
1097 
1098 void
1099 cpu_idle_prop_set_all(cpu_idle_prop_handle_t hdl, cpu_idle_prop_value_t val)
1100 {
1101         int i, idx;
1102         cpu_idle_prop_impl_t *prop = (cpu_idle_prop_impl_t *)hdl;
1103 
1104         ASSERT(hdl != NULL);
1105         for (i = 0; i < max_ncpus; i++) {
1106                 idx = CPU_IDLE_CTX2IDX(i);
1107                 prop->value[idx] = val;
1108         }
1109 }
1110 
1111 /*ARGSUSED*/
1112 static int cpu_idle_prop_update_intr_cnt(void *arg, uint64_t seqnum,
1113     cpu_idle_prop_value_t *valp)
1114 {
1115         int i;
1116         uint64_t val;
1117 
1118         for (val = 0, i = 0; i < PIL_MAX; i++) {
1119                 val += CPU->cpu_stats.sys.intr[i];
1120         }
1121         valp->cipv_uint64 = val;
1122 
1123         return (0);
1124 }
1125 
1126 uint_t
1127 cpu_idle_get_cpu_state(cpu_t *cp)
1128 {
1129         ASSERT(cp != NULL && cp->cpu_seqid < max_ncpus);
1130         return ((uint_t)cpu_idle_prop_get_uint32(
1131             cpu_idle_prop_array[CPU_IDLE_PROP_IDX_IDLE_STATE].handle,
1132             CPU_IDLE_GET_CTX(cp)));
1133 }
1134 
1135 #if defined(__x86)
1136 /*
1137  * Intercept CPU at a safe point in idle() before powering it off.
1138  */
1139 void
1140 cpu_idle_intercept_cpu(cpu_t *cp)
1141 {
1142         ASSERT(cp->cpu_seqid < max_ncpus);
1143         ASSERT(cpu_idle_cb_state[cp->cpu_seqid].v.enabled == B_FALSE);
1144 
1145         /* Set flag to intercept CPU. */
1146         CPUSET_ATOMIC_ADD(cpu_idle_intercept_set, cp->cpu_id);
1147         /* Wake up CPU from possible sleep state. */
1148         poke_cpu(cp->cpu_id);
1149         while (CPU_IN_SET(cpu_idle_intercept_set, cp->cpu_id)) {
1150                 DELAY(1);
1151         }
1152         /*
1153          * Now target CPU is spinning in a pause loop with interrupts disabled.
1154          */
1155 }
1156 #endif