1 /*
   2  * CDDL HEADER START
   3  *
   4  * The contents of this file are subject to the terms of the
   5  * Common Development and Distribution License (the "License").
   6  * You may not use this file except in compliance with the License.
   7  *
   8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
   9  * or http://www.opensolaris.org/os/licensing.
  10  * See the License for the specific language governing permissions
  11  * and limitations under the License.
  12  *
  13  * When distributing Covered Code, include this CDDL HEADER in each
  14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
  15  * If applicable, add the following below this CDDL HEADER, with the
  16  * fields enclosed by brackets "[]" replaced with your own identifying
  17  * information: Portions Copyright [yyyy] [name of copyright owner]
  18  *
  19  * CDDL HEADER END
  20  */
  21 /*
  22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
  23  * Use is subject to license terms.
  24  *
  25  * Portions Copyright 2008 Denis Cheng
  26  */
  27 
  28 #include <signal.h>
  29 #include <fcntl.h>
  30 #include <sys/stat.h>
  31 #include <sys/wait.h>
  32 
  33 #include "filebench.h"
  34 #include "procflow.h"
  35 #include "flowop.h"
  36 #include "ipc.h"
  37 
  38 /* pid and procflow pointer for this process */
  39 pid_t my_pid;
  40 procflow_t *my_procflow = NULL;
  41 
  42 static procflow_t *procflow_define_common(procflow_t **list, char *name,
  43     procflow_t *inherit, int instance);
  44 static void procflow_sleep(procflow_t *procflow, int wait_cnt);
  45 
  46 #ifdef USE_PROCESS_MODEL
  47 
  48 static enum create_n_wait {
  49         CNW_DONE,
  50         CNW_ERROR
  51 } cnw_wait;
  52 
  53 #endif  /* USE_PROCESS_MODEL */
  54 
  55 
  56 /*
  57  * Procflows are filebench entities which manage processes. Each
  58  * worker procflow spawns a separate filebench process, with attributes
  59  * inherited from a FLOW_MASTER procflow created during f model language
  60  * parsing. This section contains routines to define, create, control,
  61  * and delete procflows.
  62  *
  63  * Each process defined in the f model creates a FLOW_MASTER
  64  * procflow which encapsulates the defined attributes, and threads of
  65  * the f process, including the number of instances to create. At
  66  * runtime, a worker procflow instance with an associated filebench
  67  * process is created, which runs until told to quite by the original
  68  * filebench process or is specifically deleted.
  69  */
  70 
  71 
  72 /*
  73  * Prints a summary of the syntax for setting procflow parameters.
  74  */
  75 void
  76 procflow_usage(void)
  77 {
  78         (void) fprintf(stderr,
  79             "define process name=<name>[,instances=<count>]\n");
  80         (void) fprintf(stderr, "{\n");
  81         (void) fprintf(stderr, "  thread ...\n");
  82         (void) fprintf(stderr, "  thread ...\n");
  83         (void) fprintf(stderr, "  thread ...\n");
  84         (void) fprintf(stderr, "}\n");
  85         (void) fprintf(stderr, "\n");
  86         (void) fprintf(stderr, "\n");
  87 }
  88 
  89 /*
  90  * If filebench has been compiled to support multiple processes
  91  * (USE_PROCESS_MODEL defined), this routine forks a child
  92  * process and uses either system() or exec() to start up a new
  93  * instance of filebench, passing it the procflow name, instance
  94  * number and shared memory region address.
  95  * If USE_PROCESS_MODEL is NOT defined, then the routine
  96  * just creates a child thread which begins executing
  97  * threadflow_init() for the specified procflow.
  98  */
  99 static int
 100 procflow_createproc(procflow_t *procflow)
 101 {
 102         char instance[128];
 103         char shmaddr[128];
 104         char procname[128];
 105         pid_t pid;
 106 
 107 #ifdef USE_PROCESS_MODEL
 108 
 109         (void) snprintf(instance, sizeof (instance), "%d",
 110             procflow->pf_instance);
 111         (void) snprintf(procname, sizeof (procname), "%s", procflow->pf_name);
 112 #if defined(_LP64) || (__WORDSIZE == 64)
 113         (void) snprintf(shmaddr, sizeof (shmaddr), "%llx", filebench_shm);
 114 #else
 115         (void) snprintf(shmaddr, sizeof (shmaddr), "%x", filebench_shm);
 116 #endif
 117         filebench_log(LOG_DEBUG_IMPL, "creating process %s",
 118             procflow->pf_name);
 119 
 120         procflow->pf_running = 0;
 121 
 122 #ifdef HAVE_FORK1
 123         if ((pid = fork1()) < 0) {
 124                 filebench_log(LOG_ERROR,
 125                     "procflow_createproc fork failed: %s",
 126                     strerror(errno));
 127                 return (-1);
 128         }
 129 #else
 130         if ((pid = fork()) < 0) {
 131                 filebench_log(LOG_ERROR,
 132                     "procflow_createproc fork failed: %s",
 133                     strerror(errno));
 134                 return (-1);
 135         }
 136 #endif /* HAVE_FORK1 */
 137 
 138         /* if child, start up new copy of filebench */
 139         if (pid == 0) {
 140 #ifdef USE_SYSTEM
 141                 char syscmd[1024];
 142 #endif
 143 
 144                 (void) sigignore(SIGINT);
 145                 filebench_log(LOG_DEBUG_SCRIPT,
 146                     "Starting %s-%d", procflow->pf_name,
 147                     procflow->pf_instance);
 148                 /* Child */
 149 
 150 #ifdef USE_SYSTEM
 151                 (void) snprintf(syscmd, sizeof (syscmd), "%s -a %s -i %s -s %s",
 152                     execname,
 153                     procname,
 154                     instance,
 155                     shmaddr);
 156                 if (system(syscmd) < 0) {
 157                         filebench_log(LOG_ERROR,
 158                             "procflow exec proc failed: %s",
 159                             strerror(errno));
 160                         filebench_shutdown(1);
 161                 }
 162 
 163 #else
 164                 if (execlp(execname, procname, "-a", procname, "-i",
 165                     instance, "-s", shmaddr, "-m", shmpath, NULL) < 0) {
 166                         filebench_log(LOG_ERROR,
 167                             "procflow exec proc failed: %s",
 168                             strerror(errno));
 169                         filebench_shutdown(1);
 170                 }
 171 #endif
 172                 exit(1);
 173         } else {
 174                 /* if parent, save pid and return */
 175                 procflow->pf_pid = pid;
 176         }
 177 #else
 178         procflow->pf_running = 1;
 179         if (pthread_create(&procflow->pf_tid, NULL,
 180             (void *(*)(void*))threadflow_init, procflow) != 0) {
 181                 filebench_log(LOG_ERROR, "proc-thread create failed");
 182                 procflow->pf_running = 0;
 183         }
 184 #endif
 185         filebench_log(LOG_DEBUG_IMPL, "procflow_createproc created pid %d",
 186             pid);
 187 
 188         return (0);
 189 }
 190 
 191 /*
 192  * Find a procflow of name "name" and instance "instance" on the
 193  * master procflow list, filebench_shm->shm_proclist. Locks the list
 194  * and scans through it searching for a procflow with matching
 195  * name and instance number. If found returns a pointer to the
 196  * procflow, otherwise returns NULL.
 197  */
 198 static procflow_t *
 199 procflow_find(char *name, int instance)
 200 {
 201         procflow_t *procflow = filebench_shm->shm_proclist;
 202 
 203         filebench_log(LOG_DEBUG_IMPL, "Find: (%s-%d) proclist = %zx",
 204             name, instance, procflow);
 205 
 206         (void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
 207 
 208         while (procflow) {
 209                 filebench_log(LOG_DEBUG_IMPL, "Find: (%s-%d) == (%s-%d)",
 210                     name, instance,
 211                     procflow->pf_name,
 212                     procflow->pf_instance);
 213                 if ((strcmp(name, procflow->pf_name) == 0) &&
 214                     (instance == procflow->pf_instance)) {
 215 
 216                         (void) ipc_mutex_unlock(
 217                             &filebench_shm->shm_procflow_lock);
 218 
 219                         return (procflow);
 220                 }
 221                 procflow = procflow->pf_next;
 222         }
 223 
 224         (void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
 225 
 226         return (NULL);
 227 }
 228 
 229 static int
 230 procflow_create_all_procs(void)
 231 {
 232         procflow_t *procflow = filebench_shm->shm_proclist;
 233         int     ret = 0;
 234 
 235         while (procflow) {
 236                 int i, instances;
 237 
 238                 instances = (int)avd_get_int(procflow->pf_instances);
 239                 filebench_log(LOG_INFO, "Starting %d %s instances",
 240                     instances, procflow->pf_name);
 241 
 242                 /* Create instances of procflow */
 243                 for (i = 0; (i < instances) && (ret == 0); i++) {
 244                         procflow_t *newproc;
 245 
 246                         /* Create processes */
 247                         newproc =
 248                             procflow_define_common(&filebench_shm->shm_proclist,
 249                             procflow->pf_name, procflow, i + 1);
 250                         if (newproc == NULL)
 251                                 ret = -1;
 252                         else
 253                                 ret = procflow_createproc(newproc);
 254                 }
 255 
 256                 if (ret != 0)
 257                         break;
 258 
 259                 procflow = procflow->pf_next;
 260         }
 261 
 262         return (ret);
 263 }
 264 
 265 #ifdef USE_PROCESS_MODEL
 266 /*
 267  * Used to start up threads on a child process, when filebench is
 268  * compiled to support multiple processes. Uses the name string
 269  * and instance number passed to the child to find the previously
 270  * created procflow entity. Then uses nice() to reduce the
 271  * process' priority by at least 10. A call is then made to
 272  * threadflow_init() which creates and runs the process' threads
 273  * and flowops to completion. When threadflow_init() returns,
 274  * a call to exit() terminates the child process.
 275  */
 276 int
 277 procflow_exec(char *name, int instance)
 278 {
 279         procflow_t *procflow;
 280         int proc_nice;
 281 #ifdef HAVE_SETRLIMIT
 282         struct rlimit rlp;
 283 #endif
 284         int ret;
 285 
 286         filebench_log(LOG_DEBUG_IMPL,
 287             "procflow_execproc %s-%d",
 288             name, instance);
 289 
 290         if ((procflow = procflow_find(name, instance)) == NULL) {
 291                 filebench_log(LOG_ERROR,
 292                     "procflow_exec could not find %s-%d",
 293                     name, instance);
 294                 return (-1);
 295         }
 296 
 297         /* set the slave process' procflow pointer */
 298         my_procflow = procflow;
 299 
 300         /* set its pid from value stored by main() */
 301         procflow->pf_pid = my_pid;
 302 
 303         filebench_log(LOG_DEBUG_IMPL,
 304             "Started up %s pid %d", procflow->pf_name, my_pid);
 305 
 306         filebench_log(LOG_DEBUG_IMPL,
 307             "nice = %llx", procflow->pf_nice);
 308 
 309         proc_nice = avd_get_int(procflow->pf_nice);
 310         filebench_log(LOG_DEBUG_IMPL, "Setting pri of %s-%d to %d",
 311             name, instance, nice(proc_nice + 10));
 312 
 313         procflow->pf_running = 1;
 314 
 315 #ifdef HAVE_SETRLIMIT
 316         /* Get resource limits */
 317         (void) getrlimit(RLIMIT_NOFILE, &rlp);
 318         filebench_log(LOG_DEBUG_SCRIPT, "%d file descriptors", rlp.rlim_cur);
 319 #endif
 320 
 321         if ((ret = threadflow_init(procflow)) != FILEBENCH_OK) {
 322                 if (ret < 0) {
 323                         filebench_log(LOG_ERROR,
 324                             "Failed to start threads for %s pid %d",
 325                             procflow->pf_name, my_pid);
 326                 }
 327         } else {
 328                 filebench_log(LOG_DEBUG_IMPL,
 329                     "procflow_createproc exiting...");
 330         }
 331 
 332         (void) ipc_mutex_lock(&filebench_shm->shm_procs_running_lock);
 333         filebench_shm->shm_procs_running --;
 334         (void) ipc_mutex_unlock(&filebench_shm->shm_procs_running_lock);
 335         procflow->pf_running = 0;
 336 
 337         return (ret);
 338 }
 339 
 340 
 341 /*
 342  * A special thread from which worker (child) processes are created, and
 343  * which then waits for worker processes to die. If they die unexpectedly,
 344  * that is not a simple exit(0), then report an error and terminate the
 345  * run.
 346  */
 347 /* ARGSUSED */
 348 static void *
 349 procflow_createnwait(void *nothing)
 350 {
 351         (void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
 352 
 353         if (procflow_create_all_procs() == 0)
 354                 cnw_wait = CNW_DONE;
 355         else
 356                 cnw_wait = CNW_ERROR;
 357 
 358         if (pthread_cond_signal(&filebench_shm->shm_procflow_procs_cv) != 0)
 359                 exit(1);
 360 
 361         (void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
 362 
 363         /* CONSTCOND */
 364         while (1) {
 365                 siginfo_t status;
 366 
 367                 /* wait for any child process to exit */
 368                 if (waitid(P_ALL, 0, &status, WEXITED) != 0)
 369                         pthread_exit(0);
 370 
 371                 (void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
 372                 /* if normal shutdown in progress, just quit */
 373                 if (filebench_shm->shm_f_abort) {
 374                         (void) ipc_mutex_unlock(
 375                             &filebench_shm->shm_procflow_lock);
 376                         pthread_exit(0);
 377                 }
 378 
 379                 /* if nothing running, exit */
 380                 if (filebench_shm->shm_procs_running == 0) {
 381                         filebench_shm->shm_f_abort = FILEBENCH_ABORT_RSRC;
 382                         (void) ipc_mutex_unlock(
 383                             &filebench_shm->shm_procflow_lock);
 384                         pthread_exit(0);
 385                 }
 386                 (void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
 387 
 388                 if (status.si_code == CLD_EXITED) {
 389                         /* A process called exit(); check returned status */
 390                         if (status.si_status != 0) {
 391                                 filebench_log(LOG_ERROR,
 392                                     "Unexpected Process termination; exiting",
 393                                     status.si_status);
 394                                 filebench_shutdown(1);
 395                         }
 396                 } else {
 397                         /* A process quit because of some fatal error */
 398                         filebench_log(LOG_ERROR,
 399                             "Unexpected Process termination Code %d, Errno %d",
 400                             status.si_code, status.si_errno);
 401                         filebench_shutdown(1);
 402                 }
 403 
 404         }
 405         /* NOTREACHED */
 406         return (NULL);
 407 }
 408 
 409 /*
 410  * Cancel all threads within a processes, as well as the process itself.
 411  * Called by ^c or by sig_kill
 412  */
 413 /* ARGSUSED */
 414 static void
 415 procflow_cancel(int arg1)
 416 {
 417         filebench_log(LOG_DEBUG_IMPL, "Process signal handler on pid %",
 418             my_procflow->pf_pid);
 419 
 420         procflow_sleep(my_procflow, SHUTDOWN_WAIT_SECONDS);
 421 
 422         threadflow_delete_all(&my_procflow->pf_threads);
 423 
 424         /* quit the main procflow thread and hence the process */
 425         exit(0);
 426 }
 427 
 428 #endif  /* USE_PROCESS_MODEL */
 429 
 430 /*
 431  * Iterates through proclist, the master list of procflows,
 432  * creating the number of instances of each procflow specified
 433  * by its pf_instance attribute. Returns 0 on success, or -1
 434  * times the number of procflow instances that were not
 435  * successfully created.
 436  */
 437 int
 438 procflow_init(void)
 439 {
 440         procflow_t *procflow = filebench_shm->shm_proclist;
 441         pthread_t tid;
 442         int ret = 0;
 443 
 444         if (procflow == NULL) {
 445                 filebench_log(LOG_ERROR, "Workload has no processes");
 446                 return (FILEBENCH_ERROR);
 447         }
 448 
 449         filebench_log(LOG_DEBUG_IMPL,
 450             "procflow_init %s, %llu",
 451             procflow->pf_name,
 452             (u_longlong_t)avd_get_int(procflow->pf_instances));
 453 
 454 #ifdef USE_PROCESS_MODEL
 455         if ((pthread_create(&tid, NULL, procflow_createnwait, NULL)) != 0)
 456                 return (ret);
 457 
 458         (void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
 459 
 460         (void) signal(SIGUSR1, procflow_cancel);
 461 
 462         if ((ret = pthread_cond_wait(&filebench_shm->shm_procflow_procs_cv,
 463             &filebench_shm->shm_procflow_lock)) != 0)
 464                 return (ret);
 465 
 466         if (cnw_wait == CNW_ERROR)
 467                 ret = -1;
 468 
 469 #else /* USE_PROCESS_MODEL */
 470         (void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
 471 
 472         ret = procflow_create_all_procs();
 473 #endif /* USE_PROCESS_MODEL */
 474 
 475         (void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
 476 
 477         return (ret);
 478 }
 479 
 480 #ifdef USE_PROCESS_MODEL
 481 /*
 482  * Waits for child processes to finish and returns their exit
 483  * status. Used by procflow_delete() when the process model is
 484  * enabled to wait for a deleted process to exit.
 485  */
 486 static void
 487 procflow_wait(pid_t pid)
 488 {
 489         pid_t wpid;
 490         int stat;
 491 
 492         (void) waitpid(pid, &stat, 0);
 493         while ((wpid = waitpid(getpid() * -1, &stat, WNOHANG)) > 0)
 494                 filebench_log(LOG_DEBUG_IMPL, "Waited for pid %d", (int)wpid);
 495 }
 496 #endif
 497 
 498 /*
 499  * Common routine to sleep for wait_cnt seconds or for pf_running to
 500  * go false. Checks once a second to see if pf_running has gone false.
 501  */
 502 static void
 503 procflow_sleep(procflow_t *procflow, int wait_cnt)
 504 {
 505         while (procflow->pf_running & wait_cnt) {
 506                 (void) sleep(1);
 507                 wait_cnt--;
 508         }
 509 }
 510 
 511 /*
 512  * Deletes the designated procflow. Finally it frees the
 513  * procflow entity. filebench_shm->shm_procflow_lock must be held on entry.
 514  *
 515  * If the designated procflow is not found on the list it returns -1 and
 516  * the procflow is not deleted. Otherwise it returns 0.
 517  */
 518 static int
 519 procflow_cleanup(procflow_t *procflow)
 520 {
 521         procflow_t *entry;
 522 
 523         filebench_log(LOG_DEBUG_SCRIPT,
 524             "Deleted proc: (%s-%d) pid %d",
 525             procflow->pf_name,
 526             procflow->pf_instance,
 527             procflow->pf_pid);
 528 
 529         procflow->pf_running = 0;
 530 
 531         /* remove entry from proclist */
 532         entry = filebench_shm->shm_proclist;
 533 
 534         /* unlink procflow entity from proclist */
 535         if (entry == procflow) {
 536                 /* at head of list */
 537                 filebench_shm->shm_proclist = procflow->pf_next;
 538         } else {
 539                 /* search list for procflow */
 540                 while (entry && entry->pf_next != procflow)
 541                         entry = entry->pf_next;
 542 
 543                 /* if entity found, unlink it */
 544                 if (entry == NULL)
 545                         return (-1);
 546                 else
 547                         entry->pf_next = procflow->pf_next;
 548         }
 549 
 550         /* free up the procflow entity */
 551         ipc_free(FILEBENCH_PROCFLOW, (char *)procflow);
 552         return (0);
 553 }
 554 
 555 
 556 /*
 557  * Waits till all threadflows are started, or a timeout occurs.
 558  * Checks through the list of procflows, waiting up to 30
 559  * seconds for each one to set its pf_running flag to 1. If not
 560  * set after 30 seconds, continues on to the next procflow
 561  * anyway after logging the fact. Once pf_running is set
 562  * to 1 for a given procflow or the timeout is reached,
 563  * threadflow_allstarted() is called to start the threads.
 564  * Returns 0 (OK), unless filebench_shm->shm_f_abort is signaled,
 565  * in which case it returns -1.
 566  */
 567 int
 568 procflow_allstarted()
 569 {
 570         procflow_t *procflow = filebench_shm->shm_proclist;
 571         int running_procs = 0;
 572         int ret = 0;
 573 
 574         (void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
 575 
 576         (void) sleep(1);
 577 
 578         while (procflow) {
 579                 int waits;
 580 
 581                 if (procflow->pf_instance &&
 582                     (procflow->pf_instance == FLOW_MASTER)) {
 583                         procflow = procflow->pf_next;
 584                         continue;
 585                 }
 586 
 587                 waits = 10;
 588                 while (waits && procflow->pf_running == 0) {
 589                         (void) ipc_mutex_unlock(
 590                             &filebench_shm->shm_procflow_lock);
 591                         if (filebench_shm->shm_f_abort == 1)
 592                                 return (-1);
 593 
 594                         if (waits < 3)
 595                                 filebench_log(LOG_INFO,
 596                                     "Waiting for process %s-%d %d",
 597                                     procflow->pf_name,
 598                                     procflow->pf_instance,
 599                                     procflow->pf_pid);
 600 
 601                         (void) sleep(3);
 602                         waits--;
 603                         (void) ipc_mutex_lock(
 604                             &filebench_shm->shm_procflow_lock);
 605                 }
 606 
 607                 if (waits == 0)
 608                         filebench_log(LOG_INFO,
 609                             "Failed to start process %s-%d",
 610                             procflow->pf_name,
 611                             procflow->pf_instance);
 612 
 613                 running_procs++;
 614                 threadflow_allstarted(procflow->pf_pid, procflow->pf_threads);
 615 
 616                 procflow = procflow->pf_next;
 617         }
 618         (void) ipc_mutex_lock(&filebench_shm->shm_procs_running_lock);
 619         filebench_shm->shm_procs_running = running_procs;
 620         (void) ipc_mutex_unlock(&filebench_shm->shm_procs_running_lock);
 621 
 622         (void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
 623 
 624 
 625         return (ret);
 626 }
 627 
 628 
 629 /*
 630  * Sets the f_abort flag and clears the running count to stop
 631  * all the flowop execution threads from running. Iterates
 632  * through the procflow list and deletes all procflows except
 633  * for the FLOW_MASTER procflow. Resets the f_abort flag when
 634  * finished.
 635  *
 636  */
 637 void
 638 procflow_shutdown(void)
 639 {
 640         procflow_t *procflow, *next_procflow;
 641         int wait_cnt = SHUTDOWN_WAIT_SECONDS;
 642 
 643         (void) ipc_mutex_lock(&filebench_shm->shm_procs_running_lock);
 644         if (filebench_shm->shm_procs_running <= 0) {
 645                 /* No processes running, so no need to do anything */
 646                 (void) ipc_mutex_unlock(&filebench_shm->shm_procs_running_lock);
 647                 return;
 648         }
 649         (void) ipc_mutex_unlock(&filebench_shm->shm_procs_running_lock);
 650 
 651         (void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
 652         if (filebench_shm->shm_f_abort == FILEBENCH_ABORT_FINI) {
 653                 (void) ipc_mutex_unlock(
 654                     &filebench_shm->shm_procflow_lock);
 655                 return;
 656         }
 657 
 658         procflow = filebench_shm->shm_proclist;
 659         if (filebench_shm->shm_f_abort == FILEBENCH_OK)
 660                 filebench_shm->shm_f_abort = FILEBENCH_ABORT_DONE;
 661 
 662         while (procflow) {
 663                 if (procflow->pf_instance &&
 664                     (procflow->pf_instance == FLOW_MASTER)) {
 665                         procflow = procflow->pf_next;
 666                         continue;
 667                 }
 668                 filebench_log(LOG_DEBUG_IMPL, "Deleting process %s-%d %d",
 669                     procflow->pf_name,
 670                     procflow->pf_instance,
 671                     procflow->pf_pid);
 672 
 673                 next_procflow = procflow->pf_next;
 674 
 675                 /*
 676                  * Signalling the process with SIGUSR1 will result in it
 677                  * gracefully shutting down and exiting
 678                  */
 679                 procflow_sleep(procflow, wait_cnt);
 680                 if (procflow->pf_running) {
 681 #ifdef USE_PROCESS_MODEL
 682                         pid_t pid;
 683 
 684                         pid = procflow->pf_pid;
 685 #ifdef HAVE_SIGSEND
 686                         (void) sigsend(P_PID, pid, SIGUSR1);
 687 #else
 688                         (void) kill(pid, SIGUSR1);
 689 #endif
 690                         procflow_wait(pid);
 691 
 692 #else /* USE_PROCESS_MODEL */
 693                         threadflow_delete_all(&procflow->pf_threads);
 694 #endif /* USE_PROCESS_MODEL */
 695                 }
 696                 (void) procflow_cleanup(procflow);
 697                 procflow = next_procflow;
 698                 if (wait_cnt > 0)
 699                         wait_cnt--;
 700         }
 701 
 702         filebench_shm->shm_f_abort = FILEBENCH_ABORT_FINI;
 703         (void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
 704 
 705         /* indicate all processes are stopped, even if some are "stuck" */
 706         (void) ipc_mutex_lock(&filebench_shm->shm_procs_running_lock);
 707         filebench_shm->shm_procs_running = 0;
 708         (void) ipc_mutex_unlock(&filebench_shm->shm_procs_running_lock);
 709 }
 710 
 711 
 712 /*
 713  * Create an in-memory process object. Allocates a procflow
 714  * entity, initialized from the "inherit" procflow if supplied.
 715  * The name and instance number are set from the supplied name
 716  * and instance number and the procflow is added to the head of
 717  * the master procflow list. Returns pointer to the allocated
 718  * procflow, or NULL if a name isn't supplied or the procflow
 719  * entity cannot be allocated.
 720  *
 721  * The calling routine must hold the filebench_shm->shm_procflow_lock.
 722  */
 723 static procflow_t *
 724 procflow_define_common(procflow_t **list, char *name,
 725     procflow_t *inherit, int instance)
 726 {
 727         procflow_t *procflow;
 728 
 729         if (name == NULL)
 730                 return (NULL);
 731 
 732         procflow = (procflow_t *)ipc_malloc(FILEBENCH_PROCFLOW);
 733 
 734         if (procflow == NULL)
 735                 return (NULL);
 736 
 737         if (inherit)
 738                 (void) memcpy(procflow, inherit, sizeof (procflow_t));
 739         else
 740                 (void) memset(procflow, 0, sizeof (procflow_t));
 741 
 742         procflow->pf_instance = instance;
 743         (void) strcpy(procflow->pf_name, name);
 744 
 745         filebench_log(LOG_DEBUG_IMPL, "defining process %s-%d", name, instance);
 746 
 747         /* Add procflow to list, lock is being held already */
 748         if (*list == NULL) {
 749                 *list = procflow;
 750                 procflow->pf_next = NULL;
 751         } else {
 752                 procflow->pf_next = *list;
 753                 *list = procflow;
 754         }
 755         filebench_log(LOG_DEBUG_IMPL, "process %s-%d proclist %zx",
 756             name, instance, filebench_shm->shm_proclist);
 757 
 758         return (procflow);
 759 }
 760 
 761 /*
 762  * Create an in-memory process object as described by the syntax.
 763  * Acquires the filebench_shm->shm_procflow_lock and calls
 764  * procflow_define_common() to create and initialize a
 765  * FLOW_MASTER procflow entity from the optional "inherit"
 766  * procflow with the given name and configured for "instances"
 767  * number of worker procflows. Currently only called from
 768  * parser_proc_define().
 769  */
 770 procflow_t *
 771 procflow_define(char *name, procflow_t *inherit, avd_t instances)
 772 {
 773         procflow_t *procflow;
 774 
 775         (void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
 776 
 777         procflow = procflow_define_common(&filebench_shm->shm_proclist,
 778             name, inherit, FLOW_MASTER);
 779         procflow->pf_instances = instances;
 780 
 781         (void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
 782 
 783         return (procflow);
 784 }