1 /* 2 * CDDL HEADER START 3 * 4 * The contents of this file are subject to the terms of the 5 * Common Development and Distribution License (the "License"). 6 * You may not use this file except in compliance with the License. 7 * 8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE 9 * or http://www.opensolaris.org/os/licensing. 10 * See the License for the specific language governing permissions 11 * and limitations under the License. 12 * 13 * When distributing Covered Code, include this CDDL HEADER in each 14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE. 15 * If applicable, add the following below this CDDL HEADER, with the 16 * fields enclosed by brackets "[]" replaced with your own identifying 17 * information: Portions Copyright [yyyy] [name of copyright owner] 18 * 19 * CDDL HEADER END 20 */ 21 22 /* 23 * Copyright (c) 1992, 2010, Oracle and/or its affiliates. All rights reserved. 24 */ 25 26 #include <sys/param.h> 27 #include <sys/vmparam.h> 28 #include <sys/types.h> 29 #include <sys/sysmacros.h> 30 #include <sys/systm.h> 31 #include <sys/signal.h> 32 #include <sys/stack.h> 33 #include <sys/cred.h> 34 #include <sys/cmn_err.h> 35 #include <sys/user.h> 36 #include <sys/privregs.h> 37 #include <sys/psw.h> 38 #include <sys/debug.h> 39 #include <sys/errno.h> 40 #include <sys/proc.h> 41 #include <sys/modctl.h> 42 #include <sys/var.h> 43 #include <sys/inline.h> 44 #include <sys/syscall.h> 45 #include <sys/ucontext.h> 46 #include <sys/cpuvar.h> 47 #include <sys/siginfo.h> 48 #include <sys/trap.h> 49 #include <sys/vtrace.h> 50 #include <sys/sysinfo.h> 51 #include <sys/procfs.h> 52 #include <sys/prsystm.h> 53 #include <c2/audit.h> 54 #include <sys/modctl.h> 55 #include <sys/aio_impl.h> 56 #include <sys/tnf.h> 57 #include <sys/tnf_probe.h> 58 #include <sys/copyops.h> 59 #include <sys/priv.h> 60 #include <sys/msacct.h> 61 62 int syscalltrace = 0; 63 #ifdef SYSCALLTRACE 64 static kmutex_t systrace_lock; /* syscall tracing lock */ 65 #else 66 #define syscalltrace 0 67 #endif /* SYSCALLTRACE */ 68 69 typedef int64_t (*llfcn_t)(); /* function returning long long */ 70 71 int pre_syscall(void); 72 void post_syscall(long rval1, long rval2); 73 static krwlock_t *lock_syscall(struct sysent *, uint_t); 74 void deferred_singlestep_trap(caddr_t); 75 76 #ifdef _SYSCALL32_IMPL 77 #define LWP_GETSYSENT(lwp) \ 78 (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE ? sysent : sysent32) 79 #else 80 #define LWP_GETSYSENT(lwp) (sysent) 81 #endif 82 83 /* 84 * If watchpoints are active, don't make copying in of 85 * system call arguments take a read watchpoint trap. 86 */ 87 static int 88 copyin_args(struct regs *rp, long *ap, uint_t nargs) 89 { 90 greg_t *sp = 1 + (greg_t *)rp->r_sp; /* skip ret addr */ 91 92 ASSERT(nargs <= MAXSYSARGS); 93 94 return (copyin_nowatch(sp, ap, nargs * sizeof (*sp))); 95 } 96 97 #if defined(_SYSCALL32_IMPL) 98 static int 99 copyin_args32(struct regs *rp, long *ap, uint_t nargs) 100 { 101 greg32_t *sp = 1 + (greg32_t *)rp->r_sp; /* skip ret addr */ 102 uint32_t a32[MAXSYSARGS]; 103 int rc; 104 105 ASSERT(nargs <= MAXSYSARGS); 106 107 if ((rc = copyin_nowatch(sp, a32, nargs * sizeof (*sp))) == 0) { 108 uint32_t *a32p = &a32[0]; 109 110 while (nargs--) 111 *ap++ = (ulong_t)*a32p++; 112 } 113 return (rc); 114 } 115 #define COPYIN_ARGS32 copyin_args32 116 #else 117 #define COPYIN_ARGS32 copyin_args 118 #endif 119 120 /* 121 * Error handler for system calls where arg copy gets fault. 122 */ 123 static longlong_t 124 syscall_err() 125 { 126 return (0); 127 } 128 129 /* 130 * Corresponding sysent entry to allow syscall_entry caller 131 * to invoke syscall_err. 132 */ 133 static struct sysent sysent_err = { 134 0, SE_32RVAL1, NULL, NULL, (llfcn_t)syscall_err 135 }; 136 137 /* 138 * Called from syscall() when a non-trivial 32-bit system call occurs. 139 * Sets up the args and returns a pointer to the handler. 140 */ 141 struct sysent * 142 syscall_entry(kthread_t *t, long *argp) 143 { 144 klwp_t *lwp = ttolwp(t); 145 struct regs *rp = lwptoregs(lwp); 146 unsigned int code; 147 struct sysent *callp; 148 struct sysent *se = LWP_GETSYSENT(lwp); 149 int error = 0; 150 uint_t nargs; 151 152 ASSERT(t == curthread); 153 154 lwp->lwp_ru.sysc++; 155 lwp->lwp_eosys = NORMALRETURN; /* assume this will be normal */ 156 157 /* 158 * Set lwp_ap to point to the args, even if none are needed for this 159 * system call. This is for the loadable-syscall case where the 160 * number of args won't be known until the system call is loaded, and 161 * also maintains a non-NULL lwp_ap setup for get_syscall_args(). Note 162 * that lwp_ap MUST be set to a non-NULL value _BEFORE_ t_sysnum is 163 * set to non-zero; otherwise get_syscall_args(), seeing a non-zero 164 * t_sysnum for this thread, will charge ahead and dereference lwp_ap. 165 */ 166 lwp->lwp_ap = argp; /* for get_syscall_args */ 167 168 code = rp->r_r0; 169 t->t_sysnum = (short)code; 170 callp = code >= NSYSCALL ? &nosys_ent : se + code; 171 172 if ((t->t_pre_sys | syscalltrace) != 0) { 173 error = pre_syscall(); 174 175 /* 176 * pre_syscall() has taken care so that lwp_ap is current; 177 * it either points to syscall-entry-saved amd64 regs, 178 * or it points to lwp_arg[], which has been re-copied from 179 * the ia32 ustack, but either way, it's a current copy after 180 * /proc has possibly mucked with the syscall args. 181 */ 182 183 if (error) 184 return (&sysent_err); /* use dummy handler */ 185 } 186 187 /* 188 * Fetch the system call arguments to the kernel stack copy used 189 * for syscall handling. 190 * Note: for loadable system calls the number of arguments required 191 * may not be known at this point, and will be zero if the system call 192 * was never loaded. Once the system call has been loaded, the number 193 * of args is not allowed to be changed. 194 */ 195 if ((nargs = (uint_t)callp->sy_narg) != 0 && 196 COPYIN_ARGS32(rp, argp, nargs)) { 197 (void) set_errno(EFAULT); 198 return (&sysent_err); /* use dummy handler */ 199 } 200 201 return (callp); /* return sysent entry for caller */ 202 } 203 204 void 205 syscall_exit(kthread_t *t, long rval1, long rval2) 206 { 207 /* 208 * Handle signals and other post-call events if necessary. 209 */ 210 if ((t->t_post_sys_ast | syscalltrace) == 0) { 211 klwp_t *lwp = ttolwp(t); 212 struct regs *rp = lwptoregs(lwp); 213 214 /* 215 * Normal return. 216 * Clear error indication and set return values. 217 */ 218 rp->r_ps &= ~PS_C; /* reset carry bit */ 219 rp->r_r0 = rval1; 220 rp->r_r1 = rval2; 221 lwp->lwp_state = LWP_USER; 222 } else 223 post_syscall(rval1, rval2); 224 t->t_sysnum = 0; /* invalidate args */ 225 } 226 227 /* 228 * Perform pre-system-call processing, including stopping for tracing, 229 * auditing, etc. 230 * 231 * This routine is called only if the t_pre_sys flag is set. Any condition 232 * requiring pre-syscall handling must set the t_pre_sys flag. If the 233 * condition is persistent, this routine will repost t_pre_sys. 234 */ 235 int 236 pre_syscall() 237 { 238 kthread_t *t = curthread; 239 unsigned code = t->t_sysnum; 240 klwp_t *lwp = ttolwp(t); 241 proc_t *p = ttoproc(t); 242 int repost; 243 244 t->t_pre_sys = repost = 0; /* clear pre-syscall processing flag */ 245 246 #if defined(DEBUG) 247 /* 248 * On the i386 kernel, lwp_ap points at the piece of the thread 249 * stack that we copy the users arguments into. 250 * 251 * On the amd64 kernel, the syscall arguments in the rdi..r9 252 * registers should be pointed at by lwp_ap. If the args need to 253 * be copied so that those registers can be changed without losing 254 * the ability to get the args for /proc, they can be saved by 255 * save_syscall_args(), and lwp_ap will be restored by post_syscall(). 256 */ 257 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) { 258 #if defined(_LP64) 259 ASSERT(lwp->lwp_ap == (long *)&lwptoregs(lwp)->r_rdi); 260 } else { 261 #endif 262 ASSERT((caddr_t)lwp->lwp_ap > t->t_stkbase && 263 (caddr_t)lwp->lwp_ap < t->t_stk); 264 } 265 #endif /* DEBUG */ 266 267 /* 268 * Make sure the thread is holding the latest credentials for the 269 * process. The credentials in the process right now apply to this 270 * thread for the entire system call. 271 */ 272 if (t->t_cred != p->p_cred) { 273 cred_t *oldcred = t->t_cred; 274 /* 275 * DTrace accesses t_cred in probe context. t_cred must 276 * always be either NULL, or point to a valid, allocated cred 277 * structure. 278 */ 279 t->t_cred = crgetcred(); 280 crfree(oldcred); 281 } 282 283 /* 284 * From the proc(4) manual page: 285 * When entry to a system call is being traced, the traced process 286 * stops after having begun the call to the system but before the 287 * system call arguments have been fetched from the process. 288 */ 289 if (PTOU(p)->u_systrap) { 290 if (prismember(&PTOU(p)->u_entrymask, code)) { 291 mutex_enter(&p->p_lock); 292 /* 293 * Recheck stop condition, now that lock is held. 294 */ 295 if (PTOU(p)->u_systrap && 296 prismember(&PTOU(p)->u_entrymask, code)) { 297 stop(PR_SYSENTRY, code); 298 299 /* 300 * /proc may have modified syscall args, 301 * either in regs for amd64 or on ustack 302 * for ia32. Either way, arrange to 303 * copy them again, both for the syscall 304 * handler and for other consumers in 305 * post_syscall (like audit). Here, we 306 * only do amd64, and just set lwp_ap 307 * back to the kernel-entry stack copy; 308 * the syscall ml code redoes 309 * move-from-regs to set up for the 310 * syscall handler after we return. For 311 * ia32, save_syscall_args() below makes 312 * an lwp_ap-accessible copy. 313 */ 314 #if defined(_LP64) 315 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) { 316 lwp->lwp_argsaved = 0; 317 lwp->lwp_ap = 318 (long *)&lwptoregs(lwp)->r_rdi; 319 } 320 #endif 321 } 322 mutex_exit(&p->p_lock); 323 } 324 repost = 1; 325 } 326 327 /* 328 * ia32 kernel, or ia32 proc on amd64 kernel: keep args in 329 * lwp_arg for post-syscall processing, regardless of whether 330 * they might have been changed in /proc above. 331 */ 332 #if defined(_LP64) 333 if (lwp_getdatamodel(lwp) != DATAMODEL_NATIVE) 334 #endif 335 (void) save_syscall_args(); 336 337 if (lwp->lwp_sysabort) { 338 /* 339 * lwp_sysabort may have been set via /proc while the process 340 * was stopped on PR_SYSENTRY. If so, abort the system call. 341 * Override any error from the copyin() of the arguments. 342 */ 343 lwp->lwp_sysabort = 0; 344 (void) set_errno(EINTR); /* forces post_sys */ 345 t->t_pre_sys = 1; /* repost anyway */ 346 return (1); /* don't do system call, return EINTR */ 347 } 348 349 /* 350 * begin auditing for this syscall if the c2audit module is loaded 351 * and auditing is enabled 352 */ 353 if (audit_active == C2AUDIT_LOADED) { 354 uint32_t auditing = au_zone_getstate(NULL); 355 356 if (auditing & AU_AUDIT_MASK) { 357 int error; 358 if (error = audit_start(T_SYSCALL, code, auditing, \ 359 0, lwp)) { 360 t->t_pre_sys = 1; /* repost anyway */ 361 (void) set_errno(error); 362 return (1); 363 } 364 repost = 1; 365 } 366 } 367 368 #ifndef NPROBE 369 /* Kernel probe */ 370 if (tnf_tracing_active) { 371 TNF_PROBE_1(syscall_start, "syscall thread", /* CSTYLED */, 372 tnf_sysnum, sysnum, t->t_sysnum); 373 t->t_post_sys = 1; /* make sure post_syscall runs */ 374 repost = 1; 375 } 376 #endif /* NPROBE */ 377 378 #ifdef SYSCALLTRACE 379 if (syscalltrace) { 380 int i; 381 long *ap; 382 char *cp; 383 char *sysname; 384 struct sysent *callp; 385 386 if (code >= NSYSCALL) 387 callp = &nosys_ent; /* nosys has no args */ 388 else 389 callp = LWP_GETSYSENT(lwp) + code; 390 (void) save_syscall_args(); 391 mutex_enter(&systrace_lock); 392 printf("%d: ", p->p_pid); 393 if (code >= NSYSCALL) 394 printf("0x%x", code); 395 else { 396 sysname = mod_getsysname(code); 397 printf("%s[0x%x/0x%p]", sysname == NULL ? "NULL" : 398 sysname, code, callp->sy_callc); 399 } 400 cp = "("; 401 for (i = 0, ap = lwp->lwp_ap; i < callp->sy_narg; i++, ap++) { 402 printf("%s%lx", cp, *ap); 403 cp = ", "; 404 } 405 if (i) 406 printf(")"); 407 printf(" %s id=0x%p\n", PTOU(p)->u_comm, curthread); 408 mutex_exit(&systrace_lock); 409 } 410 #endif /* SYSCALLTRACE */ 411 412 /* 413 * If there was a continuing reason for pre-syscall processing, 414 * set the t_pre_sys flag for the next system call. 415 */ 416 if (repost) 417 t->t_pre_sys = 1; 418 lwp->lwp_error = 0; /* for old drivers */ 419 lwp->lwp_badpriv = PRIV_NONE; 420 return (0); 421 } 422 423 424 /* 425 * Post-syscall processing. Perform abnormal system call completion 426 * actions such as /proc tracing, profiling, signals, preemption, etc. 427 * 428 * This routine is called only if t_post_sys, t_sig_check, or t_astflag is set. 429 * Any condition requiring pre-syscall handling must set one of these. 430 * If the condition is persistent, this routine will repost t_post_sys. 431 */ 432 void 433 post_syscall(long rval1, long rval2) 434 { 435 kthread_t *t = curthread; 436 klwp_t *lwp = ttolwp(t); 437 proc_t *p = ttoproc(t); 438 struct regs *rp = lwptoregs(lwp); 439 uint_t error; 440 uint_t code = t->t_sysnum; 441 int repost = 0; 442 int proc_stop = 0; /* non-zero if stopping */ 443 int sigprof = 0; /* non-zero if sending SIGPROF */ 444 445 t->t_post_sys = 0; 446 447 error = lwp->lwp_errno; 448 449 /* 450 * Code can be zero if this is a new LWP returning after a forkall(), 451 * other than the one which matches the one in the parent which called 452 * forkall(). In these LWPs, skip most of post-syscall activity. 453 */ 454 if (code == 0) 455 goto sig_check; 456 /* 457 * If the trace flag is set, mark the lwp to take a single-step trap 458 * on return to user level (below). The x86 lcall interface and 459 * sysenter has already done this, and turned off the flag, but 460 * amd64 syscall interface has not. 461 */ 462 if (rp->r_ps & PS_T) { 463 lwp->lwp_pcb.pcb_flags |= DEBUG_PENDING; 464 rp->r_ps &= ~PS_T; 465 aston(curthread); 466 } 467 468 /* put out audit record for this syscall */ 469 if (AU_AUDITING()) { 470 rval_t rval; 471 472 /* XX64 -- truncation of 64-bit return values? */ 473 rval.r_val1 = (int)rval1; 474 rval.r_val2 = (int)rval2; 475 audit_finish(T_SYSCALL, code, error, &rval); 476 repost = 1; 477 } 478 479 if (curthread->t_pdmsg != NULL) { 480 char *m = curthread->t_pdmsg; 481 482 uprintf("%s", m); 483 kmem_free(m, strlen(m) + 1); 484 curthread->t_pdmsg = NULL; 485 } 486 487 /* 488 * If we're going to stop for /proc tracing, set the flag and 489 * save the arguments so that the return values don't smash them. 490 */ 491 if (PTOU(p)->u_systrap) { 492 if (prismember(&PTOU(p)->u_exitmask, code)) { 493 if (lwp_getdatamodel(lwp) == DATAMODEL_LP64) 494 (void) save_syscall_args(); 495 proc_stop = 1; 496 } 497 repost = 1; 498 } 499 500 /* 501 * Similarly check to see if SIGPROF might be sent. 502 */ 503 if (curthread->t_rprof != NULL && 504 curthread->t_rprof->rp_anystate != 0) { 505 if (lwp_getdatamodel(lwp) == DATAMODEL_LP64) 506 (void) save_syscall_args(); 507 sigprof = 1; 508 } 509 510 if (lwp->lwp_eosys == NORMALRETURN) { 511 if (error == 0) { 512 #ifdef SYSCALLTRACE 513 if (syscalltrace) { 514 mutex_enter(&systrace_lock); 515 printf( 516 "%d: r_val1=0x%lx, r_val2=0x%lx, id 0x%p\n", 517 p->p_pid, rval1, rval2, curthread); 518 mutex_exit(&systrace_lock); 519 } 520 #endif /* SYSCALLTRACE */ 521 rp->r_ps &= ~PS_C; 522 rp->r_r0 = rval1; 523 rp->r_r1 = rval2; 524 } else { 525 int sig; 526 #ifdef SYSCALLTRACE 527 if (syscalltrace) { 528 mutex_enter(&systrace_lock); 529 printf("%d: error=%d, id 0x%p\n", 530 p->p_pid, error, curthread); 531 mutex_exit(&systrace_lock); 532 } 533 #endif /* SYSCALLTRACE */ 534 if (error == EINTR && t->t_activefd.a_stale) 535 error = EBADF; 536 if (error == EINTR && 537 (sig = lwp->lwp_cursig) != 0 && 538 sigismember(&PTOU(p)->u_sigrestart, sig) && 539 PTOU(p)->u_signal[sig - 1] != SIG_DFL && 540 PTOU(p)->u_signal[sig - 1] != SIG_IGN) 541 error = ERESTART; 542 rp->r_r0 = error; 543 rp->r_ps |= PS_C; 544 } 545 } 546 547 /* 548 * From the proc(4) manual page: 549 * When exit from a system call is being traced, the traced process 550 * stops on completion of the system call just prior to checking for 551 * signals and returning to user level. At this point all return 552 * values have been stored into the traced process's saved registers. 553 */ 554 if (proc_stop) { 555 mutex_enter(&p->p_lock); 556 if (PTOU(p)->u_systrap && 557 prismember(&PTOU(p)->u_exitmask, code)) 558 stop(PR_SYSEXIT, code); 559 mutex_exit(&p->p_lock); 560 } 561 562 /* 563 * If we are the parent returning from a successful 564 * vfork, wait for the child to exec or exit. 565 * This code must be here and not in the bowels of the system 566 * so that /proc can intercept exit from vfork in a timely way. 567 */ 568 if (t->t_flag & T_VFPARENT) { 569 ASSERT(code == SYS_vfork || code == SYS_forksys); 570 ASSERT(rp->r_r1 == 0 && error == 0); 571 vfwait((pid_t)rval1); 572 t->t_flag &= ~T_VFPARENT; 573 } 574 575 /* 576 * If profiling is active, bill the current PC in user-land 577 * and keep reposting until profiling is disabled. 578 */ 579 if (p->p_prof.pr_scale) { 580 if (lwp->lwp_oweupc) 581 profil_tick(rp->r_pc); 582 repost = 1; 583 } 584 585 sig_check: 586 /* 587 * Reset flag for next time. 588 * We must do this after stopping on PR_SYSEXIT 589 * because /proc uses the information in lwp_eosys. 590 */ 591 lwp->lwp_eosys = NORMALRETURN; 592 clear_stale_fd(); 593 t->t_flag &= ~T_FORKALL; 594 595 if (t->t_astflag | t->t_sig_check) { 596 /* 597 * Turn off the AST flag before checking all the conditions that 598 * may have caused an AST. This flag is on whenever a signal or 599 * unusual condition should be handled after the next trap or 600 * syscall. 601 */ 602 astoff(t); 603 /* 604 * If a single-step trap occurred on a syscall (see trap()) 605 * recognize it now. Do this before checking for signals 606 * because deferred_singlestep_trap() may generate a SIGTRAP to 607 * the LWP or may otherwise mark the LWP to call issig(FORREAL). 608 */ 609 if (lwp->lwp_pcb.pcb_flags & DEBUG_PENDING) 610 deferred_singlestep_trap((caddr_t)rp->r_pc); 611 612 t->t_sig_check = 0; 613 614 /* 615 * The following check is legal for the following reasons: 616 * 1) The thread we are checking, is ourselves, so there is 617 * no way the proc can go away. 618 * 2) The only time we need to be protected by the 619 * lock is if the binding is changed. 620 * 621 * Note we will still take the lock and check the binding 622 * if the condition was true without the lock held. This 623 * prevents lock contention among threads owned by the 624 * same proc. 625 */ 626 627 if (curthread->t_proc_flag & TP_CHANGEBIND) { 628 mutex_enter(&p->p_lock); 629 if (curthread->t_proc_flag & TP_CHANGEBIND) { 630 timer_lwpbind(); 631 curthread->t_proc_flag &= ~TP_CHANGEBIND; 632 } 633 mutex_exit(&p->p_lock); 634 } 635 636 /* 637 * for kaio requests on the special kaio poll queue, 638 * copyout their results to user memory. 639 */ 640 if (p->p_aio) 641 aio_cleanup(0); 642 /* 643 * If this LWP was asked to hold, call holdlwp(), which will 644 * stop. holdlwps() sets this up and calls pokelwps() which 645 * sets the AST flag. 646 * 647 * Also check TP_EXITLWP, since this is used by fresh new LWPs 648 * through lwp_rtt(). That flag is set if the lwp_create(2) 649 * syscall failed after creating the LWP. 650 */ 651 if (ISHOLD(p) || (t->t_proc_flag & TP_EXITLWP)) 652 holdlwp(); 653 654 /* 655 * All code that sets signals and makes ISSIG_PENDING 656 * evaluate true must set t_sig_check afterwards. 657 */ 658 if (ISSIG_PENDING(t, lwp, p)) { 659 if (issig(FORREAL)) 660 psig(); 661 t->t_sig_check = 1; /* recheck next time */ 662 } 663 664 if (sigprof) { 665 int nargs = (code > 0 && code < NSYSCALL)? 666 LWP_GETSYSENT(lwp)[code].sy_narg : 0; 667 realsigprof(code, nargs, error); 668 t->t_sig_check = 1; /* recheck next time */ 669 } 670 671 /* 672 * If a performance counter overflow interrupt was 673 * delivered *during* the syscall, then re-enable the 674 * AST so that we take a trip through trap() to cause 675 * the SIGEMT to be delivered. 676 */ 677 if (lwp->lwp_pcb.pcb_flags & CPC_OVERFLOW) 678 aston(t); 679 680 /* 681 * /proc can't enable/disable the trace bit itself 682 * because that could race with the call gate used by 683 * system calls via "lcall". If that happened, an 684 * invalid EFLAGS would result. prstep()/prnostep() 685 * therefore schedule an AST for the purpose. 686 */ 687 if (lwp->lwp_pcb.pcb_flags & REQUEST_STEP) { 688 lwp->lwp_pcb.pcb_flags &= ~REQUEST_STEP; 689 rp->r_ps |= PS_T; 690 } 691 if (lwp->lwp_pcb.pcb_flags & REQUEST_NOSTEP) { 692 lwp->lwp_pcb.pcb_flags &= ~REQUEST_NOSTEP; 693 rp->r_ps &= ~PS_T; 694 } 695 } 696 697 lwp->lwp_errno = 0; /* clear error for next time */ 698 699 #ifndef NPROBE 700 /* Kernel probe */ 701 if (tnf_tracing_active) { 702 TNF_PROBE_3(syscall_end, "syscall thread", /* CSTYLED */, 703 tnf_long, rval1, rval1, 704 tnf_long, rval2, rval2, 705 tnf_long, errno, (long)error); 706 repost = 1; 707 } 708 #endif /* NPROBE */ 709 710 /* 711 * Set state to LWP_USER here so preempt won't give us a kernel 712 * priority if it occurs after this point. Call CL_TRAPRET() to 713 * restore the user-level priority. 714 * 715 * It is important that no locks (other than spinlocks) be entered 716 * after this point before returning to user mode (unless lwp_state 717 * is set back to LWP_SYS). 718 * 719 * XXX Sampled times past this point are charged to the user. 720 */ 721 lwp->lwp_state = LWP_USER; 722 723 if (t->t_trapret) { 724 t->t_trapret = 0; 725 thread_lock(t); 726 CL_TRAPRET(t); 727 thread_unlock(t); 728 } 729 if (CPU->cpu_runrun || t->t_schedflag & TS_ANYWAITQ) 730 preempt(); 731 prunstop(); 732 733 lwp->lwp_errno = 0; /* clear error for next time */ 734 735 /* 736 * The thread lock must be held in order to clear sysnum and reset 737 * lwp_ap atomically with respect to other threads in the system that 738 * may be looking at the args via lwp_ap from get_syscall_args(). 739 */ 740 741 thread_lock(t); 742 t->t_sysnum = 0; /* no longer in a system call */ 743 744 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) { 745 #if defined(_LP64) 746 /* 747 * In case the args were copied to the lwp, reset the 748 * pointer so the next syscall will have the right 749 * lwp_ap pointer. 750 */ 751 lwp->lwp_ap = (long *)&rp->r_rdi; 752 } else { 753 #endif 754 lwp->lwp_ap = NULL; /* reset on every syscall entry */ 755 } 756 thread_unlock(t); 757 758 lwp->lwp_argsaved = 0; 759 760 /* 761 * If there was a continuing reason for post-syscall processing, 762 * set the t_post_sys flag for the next system call. 763 */ 764 if (repost) 765 t->t_post_sys = 1; 766 767 /* 768 * If there is a ustack registered for this lwp, and the stack rlimit 769 * has been altered, read in the ustack. If the saved stack rlimit 770 * matches the bounds of the ustack, update the ustack to reflect 771 * the new rlimit. If the new stack rlimit is RLIM_INFINITY, disable 772 * stack checking by setting the size to 0. 773 */ 774 if (lwp->lwp_ustack != 0 && lwp->lwp_old_stk_ctl != 0) { 775 rlim64_t new_size; 776 caddr_t top; 777 stack_t stk; 778 struct rlimit64 rl; 779 780 mutex_enter(&p->p_lock); 781 new_size = p->p_stk_ctl; 782 top = p->p_usrstack; 783 (void) rctl_rlimit_get(rctlproc_legacy[RLIMIT_STACK], p, &rl); 784 mutex_exit(&p->p_lock); 785 786 if (rl.rlim_cur == RLIM64_INFINITY) 787 new_size = 0; 788 789 if (copyin((stack_t *)lwp->lwp_ustack, &stk, 790 sizeof (stack_t)) == 0 && 791 (stk.ss_size == lwp->lwp_old_stk_ctl || 792 stk.ss_size == 0) && 793 stk.ss_sp == top - stk.ss_size) { 794 stk.ss_sp = (void *)((uintptr_t)stk.ss_sp + 795 stk.ss_size - (uintptr_t)new_size); 796 stk.ss_size = new_size; 797 798 (void) copyout(&stk, (stack_t *)lwp->lwp_ustack, 799 sizeof (stack_t)); 800 } 801 802 lwp->lwp_old_stk_ctl = 0; 803 } 804 } 805 806 /* 807 * Called from post_syscall() when a deferred singlestep is to be taken. 808 */ 809 void 810 deferred_singlestep_trap(caddr_t pc) 811 { 812 proc_t *p = ttoproc(curthread); 813 klwp_t *lwp = ttolwp(curthread); 814 pcb_t *pcb = &lwp->lwp_pcb; 815 uint_t fault = 0; 816 k_siginfo_t siginfo; 817 818 bzero(&siginfo, sizeof (siginfo)); 819 820 /* 821 * If both NORMAL_STEP and WATCH_STEP are in 822 * effect, give precedence to WATCH_STEP. 823 * If neither is set, user must have set the 824 * PS_T bit in %efl; treat this as NORMAL_STEP. 825 */ 826 if ((fault = undo_watch_step(&siginfo)) == 0 && 827 ((pcb->pcb_flags & NORMAL_STEP) || 828 !(pcb->pcb_flags & WATCH_STEP))) { 829 siginfo.si_signo = SIGTRAP; 830 siginfo.si_code = TRAP_TRACE; 831 siginfo.si_addr = pc; 832 fault = FLTTRACE; 833 } 834 pcb->pcb_flags &= ~(DEBUG_PENDING|NORMAL_STEP|WATCH_STEP); 835 836 if (fault) { 837 /* 838 * Remember the fault and fault adddress 839 * for real-time (SIGPROF) profiling. 840 */ 841 lwp->lwp_lastfault = fault; 842 lwp->lwp_lastfaddr = siginfo.si_addr; 843 /* 844 * If a debugger has declared this fault to be an 845 * event of interest, stop the lwp. Otherwise just 846 * deliver the associated signal. 847 */ 848 if (prismember(&p->p_fltmask, fault) && 849 stop_on_fault(fault, &siginfo) == 0) 850 siginfo.si_signo = 0; 851 } 852 853 if (siginfo.si_signo) 854 trapsig(&siginfo, 1); 855 } 856 857 /* 858 * nonexistent system call-- signal lwp (may want to handle it) 859 * flag error if lwp won't see signal immediately 860 */ 861 int64_t 862 nosys() 863 { 864 tsignal(curthread, SIGSYS); 865 return (set_errno(ENOSYS)); 866 } 867 868 /* 869 * Execute a 32-bit system call on behalf of the current thread. 870 */ 871 void 872 dosyscall(void) 873 { 874 /* 875 * Need space on the stack to store syscall arguments. 876 */ 877 long syscall_args[MAXSYSARGS]; 878 struct sysent *se; 879 int64_t ret; 880 881 syscall_mstate(LMS_TRAP, LMS_SYSTEM); 882 883 ASSERT(curproc->p_model == DATAMODEL_ILP32); 884 885 CPU_STATS_ENTER_K(); 886 CPU_STATS_ADDQ(CPU, sys, syscall, 1); 887 CPU_STATS_EXIT_K(); 888 889 se = syscall_entry(curthread, syscall_args); 890 891 /* 892 * syscall_entry() copied all 8 arguments into syscall_args. 893 */ 894 ret = se->sy_callc(syscall_args[0], syscall_args[1], syscall_args[2], 895 syscall_args[3], syscall_args[4], syscall_args[5], syscall_args[6], 896 syscall_args[7]); 897 898 syscall_exit(curthread, (int)ret & 0xffffffffu, (int)(ret >> 32)); 899 syscall_mstate(LMS_SYSTEM, LMS_TRAP); 900 } 901 902 /* 903 * Get the arguments to the current system call. See comment atop 904 * save_syscall_args() regarding lwp_ap usage. 905 */ 906 907 uint_t 908 get_syscall_args(klwp_t *lwp, long *argp, int *nargsp) 909 { 910 kthread_t *t = lwptot(lwp); 911 ulong_t mask = 0xfffffffful; 912 uint_t code; 913 long *ap; 914 int nargs; 915 916 #if defined(_LP64) 917 if (lwp_getdatamodel(lwp) == DATAMODEL_LP64) 918 mask = 0xfffffffffffffffful; 919 #endif 920 921 /* 922 * The thread lock must be held while looking at the arguments to ensure 923 * they don't go away via post_syscall(). 924 * get_syscall_args() is the only routine to read them which is callable 925 * outside the LWP in question and hence the only one that must be 926 * synchronized in this manner. 927 */ 928 thread_lock(t); 929 930 code = t->t_sysnum; 931 ap = lwp->lwp_ap; 932 933 thread_unlock(t); 934 935 if (code != 0 && code < NSYSCALL) { 936 nargs = LWP_GETSYSENT(lwp)[code].sy_narg; 937 938 ASSERT(nargs <= MAXSYSARGS); 939 940 *nargsp = nargs; 941 while (nargs-- > 0) 942 *argp++ = *ap++ & mask; 943 } else { 944 *nargsp = 0; 945 } 946 947 return (code); 948 } 949 950 #ifdef _SYSCALL32_IMPL 951 /* 952 * Get the arguments to the current 32-bit system call. 953 */ 954 uint_t 955 get_syscall32_args(klwp_t *lwp, int *argp, int *nargsp) 956 { 957 long args[MAXSYSARGS]; 958 uint_t i, code; 959 960 code = get_syscall_args(lwp, args, nargsp); 961 962 for (i = 0; i != *nargsp; i++) 963 *argp++ = (int)args[i]; 964 return (code); 965 } 966 #endif 967 968 /* 969 * Save the system call arguments in a safe place. 970 * 971 * On the i386 kernel: 972 * 973 * Copy the users args prior to changing the stack or stack pointer. 974 * This is so /proc will be able to get a valid copy of the 975 * args from the user stack even after the user stack has been changed. 976 * Note that the kernel stack copy of the args may also have been 977 * changed by a system call handler which takes C-style arguments. 978 * 979 * Note that this may be called by stop() from trap(). In that case 980 * t_sysnum will be zero (syscall_exit clears it), so no args will be 981 * copied. 982 * 983 * On the amd64 kernel: 984 * 985 * For 64-bit applications, lwp->lwp_ap normally points to %rdi..%r9 986 * in the reg structure. If the user is going to change the argument 987 * registers, rax, or the stack and might want to get the args (for 988 * /proc tracing), it must copy the args elsewhere via save_syscall_args(). 989 * 990 * For 32-bit applications, lwp->lwp_ap normally points to a copy of 991 * the system call arguments on the kernel stack made from the user 992 * stack. Copy the args prior to change the stack or stack pointer. 993 * This is so /proc will be able to get a valid copy of the args 994 * from the user stack even after that stack has been changed. 995 * 996 * This may be called from stop() even when we're not in a system call. 997 * Since there's no easy way to tell, this must be safe (not panic). 998 * If the copyins get data faults, return non-zero. 999 */ 1000 int 1001 save_syscall_args() 1002 { 1003 kthread_t *t = curthread; 1004 klwp_t *lwp = ttolwp(t); 1005 uint_t code = t->t_sysnum; 1006 uint_t nargs; 1007 1008 if (lwp->lwp_argsaved || code == 0) 1009 return (0); /* args already saved or not needed */ 1010 1011 if (code >= NSYSCALL) { 1012 nargs = 0; /* illegal syscall */ 1013 } else { 1014 struct sysent *se = LWP_GETSYSENT(lwp); 1015 struct sysent *callp = se + code; 1016 1017 nargs = callp->sy_narg; 1018 if (LOADABLE_SYSCALL(callp) && nargs == 0) { 1019 krwlock_t *module_lock; 1020 1021 /* 1022 * Find out how many arguments the system 1023 * call uses. 1024 * 1025 * We have the property that loaded syscalls 1026 * never change the number of arguments they 1027 * use after they've been loaded once. This 1028 * allows us to stop for /proc tracing without 1029 * holding the module lock. 1030 * /proc is assured that sy_narg is valid. 1031 */ 1032 module_lock = lock_syscall(se, code); 1033 nargs = callp->sy_narg; 1034 rw_exit(module_lock); 1035 } 1036 } 1037 1038 /* 1039 * Fetch the system call arguments. 1040 */ 1041 if (nargs == 0) 1042 goto out; 1043 1044 ASSERT(nargs <= MAXSYSARGS); 1045 1046 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) { 1047 #if defined(_LP64) 1048 struct regs *rp = lwptoregs(lwp); 1049 1050 lwp->lwp_arg[0] = rp->r_rdi; 1051 lwp->lwp_arg[1] = rp->r_rsi; 1052 lwp->lwp_arg[2] = rp->r_rdx; 1053 lwp->lwp_arg[3] = rp->r_rcx; 1054 lwp->lwp_arg[4] = rp->r_r8; 1055 lwp->lwp_arg[5] = rp->r_r9; 1056 if (nargs > 6 && copyin_args(rp, &lwp->lwp_arg[6], nargs - 6)) 1057 return (-1); 1058 } else { 1059 #endif 1060 if (COPYIN_ARGS32(lwptoregs(lwp), lwp->lwp_arg, nargs)) 1061 return (-1); 1062 } 1063 out: 1064 lwp->lwp_ap = lwp->lwp_arg; 1065 lwp->lwp_argsaved = 1; 1066 t->t_post_sys = 1; /* so lwp_ap will be reset */ 1067 return (0); 1068 } 1069 1070 void 1071 reset_syscall_args(void) 1072 { 1073 ttolwp(curthread)->lwp_argsaved = 0; 1074 } 1075 1076 /* 1077 * Call a system call which takes a pointer to the user args struct and 1078 * a pointer to the return values. This is a bit slower than the standard 1079 * C arg-passing method in some cases. 1080 */ 1081 int64_t 1082 syscall_ap(void) 1083 { 1084 uint_t error; 1085 struct sysent *callp; 1086 rval_t rval; 1087 kthread_t *t = curthread; 1088 klwp_t *lwp = ttolwp(t); 1089 struct regs *rp = lwptoregs(lwp); 1090 1091 callp = LWP_GETSYSENT(lwp) + t->t_sysnum; 1092 1093 #if defined(__amd64) 1094 /* 1095 * If the arguments don't fit in registers %rdi-%r9, make sure they 1096 * have been copied to the lwp_arg array. 1097 */ 1098 if (callp->sy_narg > 6 && save_syscall_args()) 1099 return ((int64_t)set_errno(EFAULT)); 1100 #endif 1101 1102 rval.r_val1 = 0; 1103 rval.r_val2 = rp->r_r1; 1104 lwp->lwp_error = 0; /* for old drivers */ 1105 error = (*(callp->sy_call))(lwp->lwp_ap, &rval); 1106 if (error) 1107 return ((longlong_t)set_errno(error)); 1108 return (rval.r_vals); 1109 } 1110 1111 /* 1112 * Load system call module. 1113 * Returns with pointer to held read lock for module. 1114 */ 1115 static krwlock_t * 1116 lock_syscall(struct sysent *table, uint_t code) 1117 { 1118 krwlock_t *module_lock; 1119 struct modctl *modp; 1120 int id; 1121 struct sysent *callp; 1122 1123 callp = table + code; 1124 module_lock = callp->sy_lock; 1125 1126 /* 1127 * Optimization to only call modload if we don't have a loaded 1128 * syscall. 1129 */ 1130 rw_enter(module_lock, RW_READER); 1131 if (LOADED_SYSCALL(callp)) 1132 return (module_lock); 1133 rw_exit(module_lock); 1134 1135 for (;;) { 1136 if ((id = modload("sys", syscallnames[code])) == -1) 1137 break; 1138 1139 /* 1140 * If we loaded successfully at least once, the modctl 1141 * will still be valid, so we try to grab it by filename. 1142 * If this call fails, it's because the mod_filename 1143 * was changed after the call to modload() (mod_hold_by_name() 1144 * is the likely culprit). We can safely just take 1145 * another lap if this is the case; the modload() will 1146 * change the mod_filename back to one by which we can 1147 * find the modctl. 1148 */ 1149 modp = mod_find_by_filename("sys", syscallnames[code]); 1150 1151 if (modp == NULL) 1152 continue; 1153 1154 mutex_enter(&mod_lock); 1155 1156 if (!modp->mod_installed) { 1157 mutex_exit(&mod_lock); 1158 continue; 1159 } 1160 break; 1161 } 1162 rw_enter(module_lock, RW_READER); 1163 1164 if (id != -1) 1165 mutex_exit(&mod_lock); 1166 1167 return (module_lock); 1168 } 1169 1170 /* 1171 * Loadable syscall support. 1172 * If needed, load the module, then reserve it by holding a read 1173 * lock for the duration of the call. 1174 * Later, if the syscall is not unloadable, it could patch the vector. 1175 */ 1176 /*ARGSUSED*/ 1177 int64_t 1178 loadable_syscall( 1179 long a0, long a1, long a2, long a3, 1180 long a4, long a5, long a6, long a7) 1181 { 1182 klwp_t *lwp = ttolwp(curthread); 1183 int64_t rval; 1184 struct sysent *callp; 1185 struct sysent *se = LWP_GETSYSENT(lwp); 1186 krwlock_t *module_lock; 1187 int code, error = 0; 1188 int64_t (*sy_call)(); 1189 1190 code = curthread->t_sysnum; 1191 callp = se + code; 1192 1193 /* 1194 * Try to autoload the system call if necessary 1195 */ 1196 module_lock = lock_syscall(se, code); 1197 THREAD_KPRI_RELEASE(); /* drop priority given by rw_enter */ 1198 1199 /* 1200 * we've locked either the loaded syscall or nosys 1201 */ 1202 1203 if (lwp_getdatamodel(lwp) == DATAMODEL_NATIVE) { 1204 #if defined(_LP64) 1205 if (callp->sy_flags & SE_ARGC) { 1206 sy_call = (int64_t (*)())callp->sy_call; 1207 rval = (*sy_call)(a0, a1, a2, a3, a4, a5); 1208 } else 1209 rval = syscall_ap(); 1210 } else { 1211 #endif 1212 /* 1213 * Now that it's loaded, make sure enough args were copied. 1214 */ 1215 if (COPYIN_ARGS32(lwptoregs(lwp), lwp->lwp_ap, callp->sy_narg)) 1216 error = EFAULT; 1217 if (error) { 1218 rval = set_errno(error); 1219 } else if (callp->sy_flags & SE_ARGC) { 1220 sy_call = (int64_t (*)())callp->sy_call; 1221 rval = (*sy_call)(lwp->lwp_ap[0], lwp->lwp_ap[1], 1222 lwp->lwp_ap[2], lwp->lwp_ap[3], lwp->lwp_ap[4], 1223 lwp->lwp_ap[5]); 1224 } else 1225 rval = syscall_ap(); 1226 } 1227 1228 THREAD_KPRI_REQUEST(); /* regain priority from read lock */ 1229 rw_exit(module_lock); 1230 return (rval); 1231 } 1232 1233 /* 1234 * Indirect syscall handled in libc on x86 architectures 1235 */ 1236 int64_t 1237 indir() 1238 { 1239 return (nosys()); 1240 } 1241 1242 /* 1243 * set_errno - set an error return from the current system call. 1244 * This could be a macro. 1245 * This returns the value it is passed, so that the caller can 1246 * use tail-recursion-elimination and do return (set_errno(ERRNO)); 1247 */ 1248 uint_t 1249 set_errno(uint_t error) 1250 { 1251 ASSERT(error != 0); /* must not be used to clear errno */ 1252 1253 curthread->t_post_sys = 1; /* have post_syscall do error return */ 1254 return (ttolwp(curthread)->lwp_errno = error); 1255 } 1256 1257 /* 1258 * set_proc_pre_sys - Set pre-syscall processing for entire process. 1259 */ 1260 void 1261 set_proc_pre_sys(proc_t *p) 1262 { 1263 kthread_t *t; 1264 kthread_t *first; 1265 1266 ASSERT(MUTEX_HELD(&p->p_lock)); 1267 1268 t = first = p->p_tlist; 1269 do { 1270 t->t_pre_sys = 1; 1271 } while ((t = t->t_forw) != first); 1272 } 1273 1274 /* 1275 * set_proc_post_sys - Set post-syscall processing for entire process. 1276 */ 1277 void 1278 set_proc_post_sys(proc_t *p) 1279 { 1280 kthread_t *t; 1281 kthread_t *first; 1282 1283 ASSERT(MUTEX_HELD(&p->p_lock)); 1284 1285 t = first = p->p_tlist; 1286 do { 1287 t->t_post_sys = 1; 1288 } while ((t = t->t_forw) != first); 1289 } 1290 1291 /* 1292 * set_proc_sys - Set pre- and post-syscall processing for entire process. 1293 */ 1294 void 1295 set_proc_sys(proc_t *p) 1296 { 1297 kthread_t *t; 1298 kthread_t *first; 1299 1300 ASSERT(MUTEX_HELD(&p->p_lock)); 1301 1302 t = first = p->p_tlist; 1303 do { 1304 t->t_pre_sys = 1; 1305 t->t_post_sys = 1; 1306 } while ((t = t->t_forw) != first); 1307 } 1308 1309 /* 1310 * set_all_proc_sys - set pre- and post-syscall processing flags for all 1311 * user processes. 1312 * 1313 * This is needed when auditing, tracing, or other facilities which affect 1314 * all processes are turned on. 1315 */ 1316 void 1317 set_all_proc_sys() 1318 { 1319 kthread_t *t; 1320 kthread_t *first; 1321 1322 mutex_enter(&pidlock); 1323 t = first = curthread; 1324 do { 1325 t->t_pre_sys = 1; 1326 t->t_post_sys = 1; 1327 } while ((t = t->t_next) != first); 1328 mutex_exit(&pidlock); 1329 } 1330 1331 /* 1332 * set_all_zone_usr_proc_sys - set pre- and post-syscall processing flags for 1333 * all user processes running in the zone of the current process 1334 * 1335 * This is needed when auditing, tracing, or other facilities which affect 1336 * all processes are turned on. 1337 */ 1338 void 1339 set_all_zone_usr_proc_sys(zoneid_t zoneid) 1340 { 1341 proc_t *p; 1342 kthread_t *t; 1343 1344 mutex_enter(&pidlock); 1345 for (p = practive; p != NULL; p = p->p_next) { 1346 /* skip kernel and incomplete processes */ 1347 if (p->p_exec == NULLVP || p->p_as == &kas || 1348 p->p_stat == SIDL || p->p_stat == SZOMB || 1349 (p->p_flag & (SSYS | SEXITING | SEXITLWPS))) 1350 continue; 1351 /* 1352 * Only processes in the given zone (eventually in 1353 * all zones) are taken into account 1354 */ 1355 if (zoneid == ALL_ZONES || p->p_zone->zone_id == zoneid) { 1356 mutex_enter(&p->p_lock); 1357 if ((t = p->p_tlist) == NULL) { 1358 mutex_exit(&p->p_lock); 1359 continue; 1360 } 1361 /* 1362 * Set pre- and post-syscall processing flags 1363 * for all threads of the process 1364 */ 1365 do { 1366 t->t_pre_sys = 1; 1367 t->t_post_sys = 1; 1368 } while (p->p_tlist != (t = t->t_forw)); 1369 mutex_exit(&p->p_lock); 1370 } 1371 } 1372 mutex_exit(&pidlock); 1373 } 1374 1375 /* 1376 * set_proc_ast - Set asynchronous service trap (AST) flag for all 1377 * threads in process. 1378 */ 1379 void 1380 set_proc_ast(proc_t *p) 1381 { 1382 kthread_t *t; 1383 kthread_t *first; 1384 1385 ASSERT(MUTEX_HELD(&p->p_lock)); 1386 1387 t = first = p->p_tlist; 1388 do { 1389 aston(t); 1390 } while ((t = t->t_forw) != first); 1391 }