xref: /onnv-gate/usr/src/cmd/filebench/common/procflow.c (revision 9326:475779da8c08)
1 /*
2  * CDDL HEADER START
3  *
4  * The contents of this file are subject to the terms of the
5  * Common Development and Distribution License (the "License").
6  * You may not use this file except in compliance with the License.
7  *
8  * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9  * or http://www.opensolaris.org/os/licensing.
10  * See the License for the specific language governing permissions
11  * and limitations under the License.
12  *
13  * When distributing Covered Code, include this CDDL HEADER in each
14  * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15  * If applicable, add the following below this CDDL HEADER, with the
16  * fields enclosed by brackets "[]" replaced with your own identifying
17  * information: Portions Copyright [yyyy] [name of copyright owner]
18  *
19  * CDDL HEADER END
20  */
21 /*
22  * Copyright 2009 Sun Microsystems, Inc.  All rights reserved.
23  * Use is subject to license terms.
24  *
25  * Portions Copyright 2008 Denis Cheng
26  */
27 
28 #include <signal.h>
29 #include <fcntl.h>
30 #include <sys/stat.h>
31 #include <sys/wait.h>
32 
33 #include "filebench.h"
34 #include "procflow.h"
35 #include "flowop.h"
36 #include "ipc.h"
37 
38 /* pid and procflow pointer for this process */
39 pid_t my_pid;
40 procflow_t *my_procflow = NULL;
41 
42 static procflow_t *procflow_define_common(procflow_t **list, char *name,
43     procflow_t *inherit, int instance);
44 static void procflow_sleep(procflow_t *procflow, int wait_cnt);
45 
46 #ifdef USE_PROCESS_MODEL
47 
48 static enum create_n_wait {
49 	CNW_DONE,
50 	CNW_ERROR
51 } cnw_wait;
52 
53 #endif	/* USE_PROCESS_MODEL */
54 
55 
56 /*
57  * Procflows are filebench entities which manage processes. Each
58  * worker procflow spawns a separate filebench process, with attributes
59  * inherited from a FLOW_MASTER procflow created during f model language
60  * parsing. This section contains routines to define, create, control,
61  * and delete procflows.
62  *
63  * Each process defined in the f model creates a FLOW_MASTER
64  * procflow which encapsulates the defined attributes, and threads of
65  * the f process, including the number of instances to create. At
66  * runtime, a worker procflow instance with an associated filebench
67  * process is created, which runs until told to quite by the original
68  * filebench process or is specifically deleted.
69  */
70 
71 
72 /*
73  * Prints a summary of the syntax for setting procflow parameters.
74  */
75 void
procflow_usage(void)76 procflow_usage(void)
77 {
78 	(void) fprintf(stderr,
79 	    "define process name=<name>[,instances=<count>]\n");
80 	(void) fprintf(stderr, "{\n");
81 	(void) fprintf(stderr, "  thread ...\n");
82 	(void) fprintf(stderr, "  thread ...\n");
83 	(void) fprintf(stderr, "  thread ...\n");
84 	(void) fprintf(stderr, "}\n");
85 	(void) fprintf(stderr, "\n");
86 	(void) fprintf(stderr, "\n");
87 }
88 
89 /*
90  * If filebench has been compiled to support multiple processes
91  * (USE_PROCESS_MODEL defined), this routine forks a child
92  * process and uses either system() or exec() to start up a new
93  * instance of filebench, passing it the procflow name, instance
94  * number and shared memory region address.
95  * If USE_PROCESS_MODEL is NOT defined, then the routine
96  * just creates a child thread which begins executing
97  * threadflow_init() for the specified procflow.
98  */
99 static int
procflow_createproc(procflow_t * procflow)100 procflow_createproc(procflow_t *procflow)
101 {
102 	char instance[128];
103 	char shmaddr[128];
104 	char procname[128];
105 	pid_t pid;
106 
107 #ifdef USE_PROCESS_MODEL
108 
109 	(void) snprintf(instance, sizeof (instance), "%d",
110 	    procflow->pf_instance);
111 	(void) snprintf(procname, sizeof (procname), "%s", procflow->pf_name);
112 #if defined(_LP64) || (__WORDSIZE == 64)
113 	(void) snprintf(shmaddr, sizeof (shmaddr), "%llx", filebench_shm);
114 #else
115 	(void) snprintf(shmaddr, sizeof (shmaddr), "%x", filebench_shm);
116 #endif
117 	filebench_log(LOG_DEBUG_IMPL, "creating process %s",
118 	    procflow->pf_name);
119 
120 	procflow->pf_running = 0;
121 
122 #ifdef HAVE_FORK1
123 	if ((pid = fork1()) < 0) {
124 		filebench_log(LOG_ERROR,
125 		    "procflow_createproc fork failed: %s",
126 		    strerror(errno));
127 		return (-1);
128 	}
129 #else
130 	if ((pid = fork()) < 0) {
131 		filebench_log(LOG_ERROR,
132 		    "procflow_createproc fork failed: %s",
133 		    strerror(errno));
134 		return (-1);
135 	}
136 #endif /* HAVE_FORK1 */
137 
138 	/* if child, start up new copy of filebench */
139 	if (pid == 0) {
140 #ifdef USE_SYSTEM
141 		char syscmd[1024];
142 #endif
143 
144 		(void) sigignore(SIGINT);
145 		filebench_log(LOG_DEBUG_SCRIPT,
146 		    "Starting %s-%d", procflow->pf_name,
147 		    procflow->pf_instance);
148 		/* Child */
149 
150 #ifdef USE_SYSTEM
151 		(void) snprintf(syscmd, sizeof (syscmd), "%s -a %s -i %s -s %s",
152 		    execname,
153 		    procname,
154 		    instance,
155 		    shmaddr);
156 		if (system(syscmd) < 0) {
157 			filebench_log(LOG_ERROR,
158 			    "procflow exec proc failed: %s",
159 			    strerror(errno));
160 			filebench_shutdown(1);
161 		}
162 
163 #else
164 		if (execlp(execname, procname, "-a", procname, "-i",
165 		    instance, "-s", shmaddr, "-m", shmpath, NULL) < 0) {
166 			filebench_log(LOG_ERROR,
167 			    "procflow exec proc failed: %s",
168 			    strerror(errno));
169 			filebench_shutdown(1);
170 		}
171 #endif
172 		exit(1);
173 	} else {
174 		/* if parent, save pid and return */
175 		procflow->pf_pid = pid;
176 	}
177 #else
178 	procflow->pf_running = 1;
179 	if (pthread_create(&procflow->pf_tid, NULL,
180 	    (void *(*)(void*))threadflow_init, procflow) != 0) {
181 		filebench_log(LOG_ERROR, "proc-thread create failed");
182 		procflow->pf_running = 0;
183 	}
184 #endif
185 	filebench_log(LOG_DEBUG_IMPL, "procflow_createproc created pid %d",
186 	    pid);
187 
188 	return (0);
189 }
190 
191 /*
192  * Find a procflow of name "name" and instance "instance" on the
193  * master procflow list, filebench_shm->shm_proclist. Locks the list
194  * and scans through it searching for a procflow with matching
195  * name and instance number. If found returns a pointer to the
196  * procflow, otherwise returns NULL.
197  */
198 static procflow_t *
procflow_find(char * name,int instance)199 procflow_find(char *name, int instance)
200 {
201 	procflow_t *procflow = filebench_shm->shm_proclist;
202 
203 	filebench_log(LOG_DEBUG_IMPL, "Find: (%s-%d) proclist = %zx",
204 	    name, instance, procflow);
205 
206 	(void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
207 
208 	while (procflow) {
209 		filebench_log(LOG_DEBUG_IMPL, "Find: (%s-%d) == (%s-%d)",
210 		    name, instance,
211 		    procflow->pf_name,
212 		    procflow->pf_instance);
213 		if ((strcmp(name, procflow->pf_name) == 0) &&
214 		    (instance == procflow->pf_instance)) {
215 
216 			(void) ipc_mutex_unlock(
217 			    &filebench_shm->shm_procflow_lock);
218 
219 			return (procflow);
220 		}
221 		procflow = procflow->pf_next;
222 	}
223 
224 	(void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
225 
226 	return (NULL);
227 }
228 
229 static int
procflow_create_all_procs(void)230 procflow_create_all_procs(void)
231 {
232 	procflow_t *procflow = filebench_shm->shm_proclist;
233 	int	ret = 0;
234 
235 	while (procflow) {
236 		int i, instances;
237 
238 		instances = (int)avd_get_int(procflow->pf_instances);
239 		filebench_log(LOG_INFO, "Starting %d %s instances",
240 		    instances, procflow->pf_name);
241 
242 		/* Create instances of procflow */
243 		for (i = 0; (i < instances) && (ret == 0); i++) {
244 			procflow_t *newproc;
245 
246 			/* Create processes */
247 			newproc =
248 			    procflow_define_common(&filebench_shm->shm_proclist,
249 			    procflow->pf_name, procflow, i + 1);
250 			if (newproc == NULL)
251 				ret = -1;
252 			else
253 				ret = procflow_createproc(newproc);
254 		}
255 
256 		if (ret != 0)
257 			break;
258 
259 		procflow = procflow->pf_next;
260 	}
261 
262 	return (ret);
263 }
264 
265 #ifdef USE_PROCESS_MODEL
266 /*
267  * Used to start up threads on a child process, when filebench is
268  * compiled to support multiple processes. Uses the name string
269  * and instance number passed to the child to find the previously
270  * created procflow entity. Then uses nice() to reduce the
271  * process' priority by at least 10. A call is then made to
272  * threadflow_init() which creates and runs the process' threads
273  * and flowops to completion. When threadflow_init() returns,
274  * a call to exit() terminates the child process.
275  */
276 int
procflow_exec(char * name,int instance)277 procflow_exec(char *name, int instance)
278 {
279 	procflow_t *procflow;
280 	int proc_nice;
281 #ifdef HAVE_SETRLIMIT
282 	struct rlimit rlp;
283 #endif
284 	int ret;
285 
286 	filebench_log(LOG_DEBUG_IMPL,
287 	    "procflow_execproc %s-%d",
288 	    name, instance);
289 
290 	if ((procflow = procflow_find(name, instance)) == NULL) {
291 		filebench_log(LOG_ERROR,
292 		    "procflow_exec could not find %s-%d",
293 		    name, instance);
294 		return (-1);
295 	}
296 
297 	/* set the slave process' procflow pointer */
298 	my_procflow = procflow;
299 
300 	/* set its pid from value stored by main() */
301 	procflow->pf_pid = my_pid;
302 
303 	filebench_log(LOG_DEBUG_IMPL,
304 	    "Started up %s pid %d", procflow->pf_name, my_pid);
305 
306 	filebench_log(LOG_DEBUG_IMPL,
307 	    "nice = %llx", procflow->pf_nice);
308 
309 	proc_nice = avd_get_int(procflow->pf_nice);
310 	filebench_log(LOG_DEBUG_IMPL, "Setting pri of %s-%d to %d",
311 	    name, instance, nice(proc_nice + 10));
312 
313 	procflow->pf_running = 1;
314 
315 #ifdef HAVE_SETRLIMIT
316 	/* Get resource limits */
317 	(void) getrlimit(RLIMIT_NOFILE, &rlp);
318 	filebench_log(LOG_DEBUG_SCRIPT, "%d file descriptors", rlp.rlim_cur);
319 #endif
320 
321 	if ((ret = threadflow_init(procflow)) != FILEBENCH_OK) {
322 		if (ret < 0) {
323 			filebench_log(LOG_ERROR,
324 			    "Failed to start threads for %s pid %d",
325 			    procflow->pf_name, my_pid);
326 		}
327 	} else {
328 		filebench_log(LOG_DEBUG_IMPL,
329 		    "procflow_createproc exiting...");
330 	}
331 
332 	(void) ipc_mutex_lock(&filebench_shm->shm_procs_running_lock);
333 	filebench_shm->shm_procs_running --;
334 	(void) ipc_mutex_unlock(&filebench_shm->shm_procs_running_lock);
335 	procflow->pf_running = 0;
336 
337 	return (ret);
338 }
339 
340 
341 /*
342  * A special thread from which worker (child) processes are created, and
343  * which then waits for worker processes to die. If they die unexpectedly,
344  * that is not a simple exit(0), then report an error and terminate the
345  * run.
346  */
347 /* ARGSUSED */
348 static void *
procflow_createnwait(void * nothing)349 procflow_createnwait(void *nothing)
350 {
351 	(void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
352 
353 	if (procflow_create_all_procs() == 0)
354 		cnw_wait = CNW_DONE;
355 	else
356 		cnw_wait = CNW_ERROR;
357 
358 	if (pthread_cond_signal(&filebench_shm->shm_procflow_procs_cv) != 0)
359 		exit(1);
360 
361 	(void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
362 
363 	/* CONSTCOND */
364 	while (1) {
365 		siginfo_t status;
366 
367 		/* wait for any child process to exit */
368 		if (waitid(P_ALL, 0, &status, WEXITED) != 0)
369 			pthread_exit(0);
370 
371 		(void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
372 		/* if normal shutdown in progress, just quit */
373 		if (filebench_shm->shm_f_abort) {
374 			(void) ipc_mutex_unlock(
375 			    &filebench_shm->shm_procflow_lock);
376 			pthread_exit(0);
377 		}
378 
379 		/* if nothing running, exit */
380 		if (filebench_shm->shm_procs_running == 0) {
381 			filebench_shm->shm_f_abort = FILEBENCH_ABORT_RSRC;
382 			(void) ipc_mutex_unlock(
383 			    &filebench_shm->shm_procflow_lock);
384 			pthread_exit(0);
385 		}
386 		(void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
387 
388 		if (status.si_code == CLD_EXITED) {
389 			/* A process called exit(); check returned status */
390 			if (status.si_status != 0) {
391 				filebench_log(LOG_ERROR,
392 				    "Unexpected Process termination; exiting",
393 				    status.si_status);
394 				filebench_shutdown(1);
395 			}
396 		} else {
397 			/* A process quit because of some fatal error */
398 			filebench_log(LOG_ERROR,
399 			    "Unexpected Process termination Code %d, Errno %d",
400 			    status.si_code, status.si_errno);
401 			filebench_shutdown(1);
402 		}
403 
404 	}
405 	/* NOTREACHED */
406 	return (NULL);
407 }
408 
409 /*
410  * Cancel all threads within a processes, as well as the process itself.
411  * Called by ^c or by sig_kill
412  */
413 /* ARGSUSED */
414 static void
procflow_cancel(int arg1)415 procflow_cancel(int arg1)
416 {
417 	filebench_log(LOG_DEBUG_IMPL, "Process signal handler on pid %",
418 	    my_procflow->pf_pid);
419 
420 	procflow_sleep(my_procflow, SHUTDOWN_WAIT_SECONDS);
421 
422 	threadflow_delete_all(&my_procflow->pf_threads);
423 
424 	/* quit the main procflow thread and hence the process */
425 	exit(0);
426 }
427 
428 #endif	/* USE_PROCESS_MODEL */
429 
430 /*
431  * Iterates through proclist, the master list of procflows,
432  * creating the number of instances of each procflow specified
433  * by its pf_instance attribute. Returns 0 on success, or -1
434  * times the number of procflow instances that were not
435  * successfully created.
436  */
437 int
procflow_init(void)438 procflow_init(void)
439 {
440 	procflow_t *procflow = filebench_shm->shm_proclist;
441 	pthread_t tid;
442 	int ret = 0;
443 
444 	if (procflow == NULL) {
445 		filebench_log(LOG_ERROR, "Workload has no processes");
446 		return (FILEBENCH_ERROR);
447 	}
448 
449 	filebench_log(LOG_DEBUG_IMPL,
450 	    "procflow_init %s, %llu",
451 	    procflow->pf_name,
452 	    (u_longlong_t)avd_get_int(procflow->pf_instances));
453 
454 #ifdef USE_PROCESS_MODEL
455 	if ((pthread_create(&tid, NULL, procflow_createnwait, NULL)) != 0)
456 		return (ret);
457 
458 	(void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
459 
460 	(void) signal(SIGUSR1, procflow_cancel);
461 
462 	if ((ret = pthread_cond_wait(&filebench_shm->shm_procflow_procs_cv,
463 	    &filebench_shm->shm_procflow_lock)) != 0)
464 		return (ret);
465 
466 	if (cnw_wait == CNW_ERROR)
467 		ret = -1;
468 
469 #else /* USE_PROCESS_MODEL */
470 	(void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
471 
472 	ret = procflow_create_all_procs();
473 #endif /* USE_PROCESS_MODEL */
474 
475 	(void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
476 
477 	return (ret);
478 }
479 
480 #ifdef USE_PROCESS_MODEL
481 /*
482  * Waits for child processes to finish and returns their exit
483  * status. Used by procflow_delete() when the process model is
484  * enabled to wait for a deleted process to exit.
485  */
486 static void
procflow_wait(pid_t pid)487 procflow_wait(pid_t pid)
488 {
489 	pid_t wpid;
490 	int stat;
491 
492 	(void) waitpid(pid, &stat, 0);
493 	while ((wpid = waitpid(getpid() * -1, &stat, WNOHANG)) > 0)
494 		filebench_log(LOG_DEBUG_IMPL, "Waited for pid %d", (int)wpid);
495 }
496 #endif
497 
498 /*
499  * Common routine to sleep for wait_cnt seconds or for pf_running to
500  * go false. Checks once a second to see if pf_running has gone false.
501  */
502 static void
procflow_sleep(procflow_t * procflow,int wait_cnt)503 procflow_sleep(procflow_t *procflow, int wait_cnt)
504 {
505 	while (procflow->pf_running & wait_cnt) {
506 		(void) sleep(1);
507 		wait_cnt--;
508 	}
509 }
510 
511 /*
512  * Deletes the designated procflow. Finally it frees the
513  * procflow entity. filebench_shm->shm_procflow_lock must be held on entry.
514  *
515  * If the designated procflow is not found on the list it returns -1 and
516  * the procflow is not deleted. Otherwise it returns 0.
517  */
518 static int
procflow_cleanup(procflow_t * procflow)519 procflow_cleanup(procflow_t *procflow)
520 {
521 	procflow_t *entry;
522 
523 	filebench_log(LOG_DEBUG_SCRIPT,
524 	    "Deleted proc: (%s-%d) pid %d",
525 	    procflow->pf_name,
526 	    procflow->pf_instance,
527 	    procflow->pf_pid);
528 
529 	procflow->pf_running = 0;
530 
531 	/* remove entry from proclist */
532 	entry = filebench_shm->shm_proclist;
533 
534 	/* unlink procflow entity from proclist */
535 	if (entry == procflow) {
536 		/* at head of list */
537 		filebench_shm->shm_proclist = procflow->pf_next;
538 	} else {
539 		/* search list for procflow */
540 		while (entry && entry->pf_next != procflow)
541 			entry = entry->pf_next;
542 
543 		/* if entity found, unlink it */
544 		if (entry == NULL)
545 			return (-1);
546 		else
547 			entry->pf_next = procflow->pf_next;
548 	}
549 
550 	/* free up the procflow entity */
551 	ipc_free(FILEBENCH_PROCFLOW, (char *)procflow);
552 	return (0);
553 }
554 
555 
556 /*
557  * Waits till all threadflows are started, or a timeout occurs.
558  * Checks through the list of procflows, waiting up to 30
559  * seconds for each one to set its pf_running flag to 1. If not
560  * set after 30 seconds, continues on to the next procflow
561  * anyway after logging the fact. Once pf_running is set
562  * to 1 for a given procflow or the timeout is reached,
563  * threadflow_allstarted() is called to start the threads.
564  * Returns 0 (OK), unless filebench_shm->shm_f_abort is signaled,
565  * in which case it returns -1.
566  */
567 int
procflow_allstarted()568 procflow_allstarted()
569 {
570 	procflow_t *procflow = filebench_shm->shm_proclist;
571 	int running_procs = 0;
572 	int ret = 0;
573 
574 	(void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
575 
576 	(void) sleep(1);
577 
578 	while (procflow) {
579 		int waits;
580 
581 		if (procflow->pf_instance &&
582 		    (procflow->pf_instance == FLOW_MASTER)) {
583 			procflow = procflow->pf_next;
584 			continue;
585 		}
586 
587 		waits = 10;
588 		while (waits && procflow->pf_running == 0) {
589 			(void) ipc_mutex_unlock(
590 			    &filebench_shm->shm_procflow_lock);
591 			if (filebench_shm->shm_f_abort == 1)
592 				return (-1);
593 
594 			if (waits < 3)
595 				filebench_log(LOG_INFO,
596 				    "Waiting for process %s-%d %d",
597 				    procflow->pf_name,
598 				    procflow->pf_instance,
599 				    procflow->pf_pid);
600 
601 			(void) sleep(3);
602 			waits--;
603 			(void) ipc_mutex_lock(
604 			    &filebench_shm->shm_procflow_lock);
605 		}
606 
607 		if (waits == 0)
608 			filebench_log(LOG_INFO,
609 			    "Failed to start process %s-%d",
610 			    procflow->pf_name,
611 			    procflow->pf_instance);
612 
613 		running_procs++;
614 		threadflow_allstarted(procflow->pf_pid, procflow->pf_threads);
615 
616 		procflow = procflow->pf_next;
617 	}
618 	(void) ipc_mutex_lock(&filebench_shm->shm_procs_running_lock);
619 	filebench_shm->shm_procs_running = running_procs;
620 	(void) ipc_mutex_unlock(&filebench_shm->shm_procs_running_lock);
621 
622 	(void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
623 
624 
625 	return (ret);
626 }
627 
628 
629 /*
630  * Sets the f_abort flag and clears the running count to stop
631  * all the flowop execution threads from running. Iterates
632  * through the procflow list and deletes all procflows except
633  * for the FLOW_MASTER procflow. Resets the f_abort flag when
634  * finished.
635  *
636  */
637 void
procflow_shutdown(void)638 procflow_shutdown(void)
639 {
640 	procflow_t *procflow, *next_procflow;
641 	int wait_cnt = SHUTDOWN_WAIT_SECONDS;
642 
643 	(void) ipc_mutex_lock(&filebench_shm->shm_procs_running_lock);
644 	if (filebench_shm->shm_procs_running <= 0) {
645 		/* No processes running, so no need to do anything */
646 		(void) ipc_mutex_unlock(&filebench_shm->shm_procs_running_lock);
647 		return;
648 	}
649 	(void) ipc_mutex_unlock(&filebench_shm->shm_procs_running_lock);
650 
651 	(void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
652 	if (filebench_shm->shm_f_abort == FILEBENCH_ABORT_FINI) {
653 		(void) ipc_mutex_unlock(
654 		    &filebench_shm->shm_procflow_lock);
655 		return;
656 	}
657 
658 	procflow = filebench_shm->shm_proclist;
659 	if (filebench_shm->shm_f_abort == FILEBENCH_OK)
660 		filebench_shm->shm_f_abort = FILEBENCH_ABORT_DONE;
661 
662 	while (procflow) {
663 		if (procflow->pf_instance &&
664 		    (procflow->pf_instance == FLOW_MASTER)) {
665 			procflow = procflow->pf_next;
666 			continue;
667 		}
668 		filebench_log(LOG_DEBUG_IMPL, "Deleting process %s-%d %d",
669 		    procflow->pf_name,
670 		    procflow->pf_instance,
671 		    procflow->pf_pid);
672 
673 		next_procflow = procflow->pf_next;
674 
675 		/*
676 		 * Signalling the process with SIGUSR1 will result in it
677 		 * gracefully shutting down and exiting
678 		 */
679 		procflow_sleep(procflow, wait_cnt);
680 		if (procflow->pf_running) {
681 #ifdef USE_PROCESS_MODEL
682 			pid_t pid;
683 
684 			pid = procflow->pf_pid;
685 #ifdef HAVE_SIGSEND
686 			(void) sigsend(P_PID, pid, SIGUSR1);
687 #else
688 			(void) kill(pid, SIGUSR1);
689 #endif
690 			procflow_wait(pid);
691 
692 #else /* USE_PROCESS_MODEL */
693 			threadflow_delete_all(&procflow->pf_threads);
694 #endif /* USE_PROCESS_MODEL */
695 		}
696 		(void) procflow_cleanup(procflow);
697 		procflow = next_procflow;
698 		if (wait_cnt > 0)
699 			wait_cnt--;
700 	}
701 
702 	filebench_shm->shm_f_abort = FILEBENCH_ABORT_FINI;
703 	(void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
704 
705 	/* indicate all processes are stopped, even if some are "stuck" */
706 	(void) ipc_mutex_lock(&filebench_shm->shm_procs_running_lock);
707 	filebench_shm->shm_procs_running = 0;
708 	(void) ipc_mutex_unlock(&filebench_shm->shm_procs_running_lock);
709 }
710 
711 
712 /*
713  * Create an in-memory process object. Allocates a procflow
714  * entity, initialized from the "inherit" procflow if supplied.
715  * The name and instance number are set from the supplied name
716  * and instance number and the procflow is added to the head of
717  * the master procflow list. Returns pointer to the allocated
718  * procflow, or NULL if a name isn't supplied or the procflow
719  * entity cannot be allocated.
720  *
721  * The calling routine must hold the filebench_shm->shm_procflow_lock.
722  */
723 static procflow_t *
procflow_define_common(procflow_t ** list,char * name,procflow_t * inherit,int instance)724 procflow_define_common(procflow_t **list, char *name,
725     procflow_t *inherit, int instance)
726 {
727 	procflow_t *procflow;
728 
729 	if (name == NULL)
730 		return (NULL);
731 
732 	procflow = (procflow_t *)ipc_malloc(FILEBENCH_PROCFLOW);
733 
734 	if (procflow == NULL)
735 		return (NULL);
736 
737 	if (inherit)
738 		(void) memcpy(procflow, inherit, sizeof (procflow_t));
739 	else
740 		(void) memset(procflow, 0, sizeof (procflow_t));
741 
742 	procflow->pf_instance = instance;
743 	(void) strcpy(procflow->pf_name, name);
744 
745 	filebench_log(LOG_DEBUG_IMPL, "defining process %s-%d", name, instance);
746 
747 	/* Add procflow to list, lock is being held already */
748 	if (*list == NULL) {
749 		*list = procflow;
750 		procflow->pf_next = NULL;
751 	} else {
752 		procflow->pf_next = *list;
753 		*list = procflow;
754 	}
755 	filebench_log(LOG_DEBUG_IMPL, "process %s-%d proclist %zx",
756 	    name, instance, filebench_shm->shm_proclist);
757 
758 	return (procflow);
759 }
760 
761 /*
762  * Create an in-memory process object as described by the syntax.
763  * Acquires the filebench_shm->shm_procflow_lock and calls
764  * procflow_define_common() to create and initialize a
765  * FLOW_MASTER procflow entity from the optional "inherit"
766  * procflow with the given name and configured for "instances"
767  * number of worker procflows. Currently only called from
768  * parser_proc_define().
769  */
770 procflow_t *
procflow_define(char * name,procflow_t * inherit,avd_t instances)771 procflow_define(char *name, procflow_t *inherit, avd_t instances)
772 {
773 	procflow_t *procflow;
774 
775 	(void) ipc_mutex_lock(&filebench_shm->shm_procflow_lock);
776 
777 	procflow = procflow_define_common(&filebench_shm->shm_proclist,
778 	    name, inherit, FLOW_MASTER);
779 	procflow->pf_instances = instances;
780 
781 	(void) ipc_mutex_unlock(&filebench_shm->shm_procflow_lock);
782 
783 	return (procflow);
784 }
785