xref: /openbsd-src/sys/kern/kern_exit.c (revision a554f8d9c26f17ff529f06b95d7012ef0acf0fc6)
1 /*	$OpenBSD: kern_exit.c,v 1.128 2013/10/08 03:50:07 guenther Exp $	*/
2 /*	$NetBSD: kern_exit.c,v 1.39 1996/04/22 01:38:25 christos Exp $	*/
3 
4 /*
5  * Copyright (c) 1982, 1986, 1989, 1991, 1993
6  *	The Regents of the University of California.  All rights reserved.
7  * (c) UNIX System Laboratories, Inc.
8  * All or some portions of this file are derived from material licensed
9  * to the University of California by American Telephone and Telegraph
10  * Co. or Unix System Laboratories, Inc. and are reproduced herein with
11  * the permission of UNIX System Laboratories, Inc.
12  *
13  * Redistribution and use in source and binary forms, with or without
14  * modification, are permitted provided that the following conditions
15  * are met:
16  * 1. Redistributions of source code must retain the above copyright
17  *    notice, this list of conditions and the following disclaimer.
18  * 2. Redistributions in binary form must reproduce the above copyright
19  *    notice, this list of conditions and the following disclaimer in the
20  *    documentation and/or other materials provided with the distribution.
21  * 3. Neither the name of the University nor the names of its contributors
22  *    may be used to endorse or promote products derived from this software
23  *    without specific prior written permission.
24  *
25  * THIS SOFTWARE IS PROVIDED BY THE REGENTS AND CONTRIBUTORS ``AS IS'' AND
26  * ANY EXPRESS OR IMPLIED WARRANTIES, INCLUDING, BUT NOT LIMITED TO, THE
27  * IMPLIED WARRANTIES OF MERCHANTABILITY AND FITNESS FOR A PARTICULAR PURPOSE
28  * ARE DISCLAIMED.  IN NO EVENT SHALL THE REGENTS OR CONTRIBUTORS BE LIABLE
29  * FOR ANY DIRECT, INDIRECT, INCIDENTAL, SPECIAL, EXEMPLARY, OR CONSEQUENTIAL
30  * DAMAGES (INCLUDING, BUT NOT LIMITED TO, PROCUREMENT OF SUBSTITUTE GOODS
31  * OR SERVICES; LOSS OF USE, DATA, OR PROFITS; OR BUSINESS INTERRUPTION)
32  * HOWEVER CAUSED AND ON ANY THEORY OF LIABILITY, WHETHER IN CONTRACT, STRICT
33  * LIABILITY, OR TORT (INCLUDING NEGLIGENCE OR OTHERWISE) ARISING IN ANY WAY
34  * OUT OF THE USE OF THIS SOFTWARE, EVEN IF ADVISED OF THE POSSIBILITY OF
35  * SUCH DAMAGE.
36  *
37  *	@(#)kern_exit.c	8.7 (Berkeley) 2/12/94
38  */
39 
40 #include <sys/param.h>
41 #include <sys/systm.h>
42 #include <sys/ioctl.h>
43 #include <sys/proc.h>
44 #include <sys/tty.h>
45 #include <sys/time.h>
46 #include <sys/resource.h>
47 #include <sys/kernel.h>
48 #include <sys/buf.h>
49 #include <sys/wait.h>
50 #include <sys/file.h>
51 #include <sys/vnode.h>
52 #include <sys/syslog.h>
53 #include <sys/malloc.h>
54 #include <sys/resourcevar.h>
55 #include <sys/ptrace.h>
56 #include <sys/acct.h>
57 #include <sys/filedesc.h>
58 #include <sys/signalvar.h>
59 #include <sys/sched.h>
60 #include <sys/ktrace.h>
61 #include <sys/pool.h>
62 #include <sys/mutex.h>
63 #ifdef SYSVSEM
64 #include <sys/sem.h>
65 #endif
66 
67 #include "systrace.h"
68 #include <dev/systrace.h>
69 
70 #include <sys/mount.h>
71 #include <sys/syscallargs.h>
72 
73 
74 #include <uvm/uvm_extern.h>
75 
76 /*
77  * exit --
78  *	Death of process.
79  */
80 int
81 sys_exit(struct proc *p, void *v, register_t *retval)
82 {
83 	struct sys_exit_args /* {
84 		syscallarg(int) rval;
85 	} */ *uap = v;
86 
87 	exit1(p, W_EXITCODE(SCARG(uap, rval), 0), EXIT_NORMAL);
88 	/* NOTREACHED */
89 	return (0);
90 }
91 
92 int
93 sys___threxit(struct proc *p, void *v, register_t *retval)
94 {
95 	struct sys___threxit_args /* {
96 		syscallarg(pid_t *) notdead;
97 	} */ *uap = v;
98 
99 	if (SCARG(uap, notdead) != NULL) {
100 		pid_t zero = 0;
101 		if (copyout(&zero, SCARG(uap, notdead), sizeof(zero)))
102 			psignal(p, SIGSEGV);
103 	}
104 	exit1(p, 0, EXIT_THREAD);
105 
106 	return (0);
107 }
108 
109 /*
110  * Exit: deallocate address space and other resources, change proc state
111  * to zombie, and unlink proc from allproc and parent's lists.  Save exit
112  * status and rusage for wait().  Check for child processes and orphan them.
113  */
114 void
115 exit1(struct proc *p, int rv, int flags)
116 {
117 	struct process *pr, *qr, *nqr;
118 	struct rusage *rup;
119 	struct vnode *ovp;
120 
121 	if (p->p_pid == 1)
122 		panic("init died (signal %d, exit %d)",
123 		    WTERMSIG(rv), WEXITSTATUS(rv));
124 
125 	atomic_setbits_int(&p->p_flag, P_WEXIT);
126 
127 	pr = p->p_p;
128 
129 	/* single-threaded? */
130 	if (TAILQ_FIRST(&pr->ps_threads) == p &&
131 	    TAILQ_NEXT(p, p_thr_link) == NULL) {
132 		flags = EXIT_NORMAL;
133 	} else {
134 		/* nope, multi-threaded */
135 		if (flags == EXIT_NORMAL)
136 			single_thread_set(p, SINGLE_EXIT, 0);
137 		else if (flags == EXIT_THREAD)
138 			single_thread_check(p, 0);
139 	}
140 
141 	if (flags == EXIT_NORMAL) {
142 		atomic_setbits_int(&pr->ps_flags, PS_EXITING);
143 		pr->ps_mainproc->p_xstat = rv;
144 
145 		/*
146 		 * If parent is waiting for us to exit or exec, PS_PPWAIT
147 		 * is set; we wake up the parent early to avoid deadlock.
148 		 */
149 		if (pr->ps_flags & PS_PPWAIT) {
150 			atomic_clearbits_int(&pr->ps_flags, PS_PPWAIT);
151 			atomic_clearbits_int(&pr->ps_pptr->ps_flags,
152 			    PS_ISPWAIT);
153 			wakeup(pr->ps_pptr);
154 		}
155 	}
156 
157 	/* unlink ourselves from the active threads */
158 	TAILQ_REMOVE(&pr->ps_threads, p, p_thr_link);
159 	if ((p->p_flag & P_THREAD) == 0) {
160 		/* main thread gotta wait because it has the pid, et al */
161 		while (!TAILQ_EMPTY(&pr->ps_threads))
162 			tsleep(&pr->ps_threads, PUSER, "thrdeath", 0);
163 		if (pr->ps_flags & PS_PROFIL)
164 			stopprofclock(pr);
165 	} else if (TAILQ_EMPTY(&pr->ps_threads)) {
166 		wakeup(&pr->ps_threads);
167 	}
168 
169 	rup = pr->ps_ru;
170 	if (rup == NULL) {
171 		rup = pool_get(&rusage_pool, PR_WAITOK | PR_ZERO);
172 		if (pr->ps_ru == NULL) {
173 			pr->ps_ru = rup;
174 		} else {
175 			pool_put(&rusage_pool, rup);
176 			rup = pr->ps_ru;
177 		}
178 	}
179 	p->p_siglist = 0;
180 
181 	/*
182 	 * Close open files and release open-file table.
183 	 */
184 	fdfree(p);
185 
186 	if ((p->p_flag & P_THREAD) == 0) {
187 		timeout_del(&pr->ps_realit_to);
188 #ifdef SYSVSEM
189 		semexit(pr);
190 #endif
191 		if (SESS_LEADER(pr)) {
192 			struct session *sp = pr->ps_session;
193 
194 			if (sp->s_ttyvp) {
195 				/*
196 				 * Controlling process.
197 				 * Signal foreground pgrp,
198 				 * drain controlling terminal
199 				 * and revoke access to controlling terminal.
200 				 */
201 				if (sp->s_ttyp->t_session == sp) {
202 					if (sp->s_ttyp->t_pgrp)
203 						pgsignal(sp->s_ttyp->t_pgrp,
204 						    SIGHUP, 1);
205 					ttywait(sp->s_ttyp);
206 					/*
207 					 * The tty could have been revoked
208 					 * if we blocked.
209 					 */
210 					if (sp->s_ttyvp)
211 						VOP_REVOKE(sp->s_ttyvp,
212 						    REVOKEALL);
213 				}
214 				ovp = sp->s_ttyvp;
215 				sp->s_ttyvp = NULL;
216 				if (ovp)
217 					vrele(ovp);
218 				/*
219 				 * s_ttyp is not zero'd; we use this to
220 				 * indicate that the session once had a
221 				 * controlling terminal.  (for logging and
222 				 * informational purposes)
223 				 */
224 			}
225 			sp->s_leader = NULL;
226 		}
227 		fixjobc(pr, pr->ps_pgrp, 0);
228 
229 #ifdef ACCOUNTING
230 		acct_process(p);
231 #endif
232 
233 #ifdef KTRACE
234 		/* release trace file */
235 		if (pr->ps_tracevp)
236 			ktrcleartrace(pr);
237 #endif
238 	}
239 
240 #if NSYSTRACE > 0
241 	if (ISSET(p->p_flag, P_SYSTRACE))
242 		systrace_exit(p);
243 #endif
244 
245 	/*
246 	 * If emulation has process exit hook, call it now.
247 	 */
248 	if (p->p_emul->e_proc_exit)
249 		(*p->p_emul->e_proc_exit)(p);
250 
251         /*
252          * Remove proc from pidhash chain so looking it up won't
253          * work.  Move it from allproc to zombproc, but do not yet
254          * wake up the reaper.  We will put the proc on the
255          * deadproc list later (using the p_hash member), and
256          * wake up the reaper when we do.
257          */
258 	/*
259 	 * NOTE: WE ARE NO LONGER ALLOWED TO SLEEP!
260 	 */
261 	p->p_stat = SDEAD;
262 
263 	LIST_REMOVE(p, p_hash);
264 	LIST_REMOVE(p, p_list);
265 	LIST_INSERT_HEAD(&zombproc, p, p_list);
266 
267 	/*
268 	 * Give orphaned children to init(8).
269 	 */
270 	if ((p->p_flag & P_THREAD) == 0) {
271 		qr = LIST_FIRST(&pr->ps_children);
272 		if (qr)		/* only need this if any child is S_ZOMB */
273 			wakeup(initproc->p_p);
274 		for (; qr != 0; qr = nqr) {
275 			nqr = LIST_NEXT(qr, ps_sibling);
276 			proc_reparent(qr, initproc->p_p);
277 			/*
278 			 * Traced processes are killed since their
279 			 * existence means someone is screwing up.
280 			 */
281 			if (qr->ps_flags & PS_TRACED &&
282 			    !(qr->ps_flags & PS_EXITING)) {
283 				atomic_clearbits_int(&qr->ps_flags, PS_TRACED);
284 				/*
285 				 * If single threading is active,
286 				 * direct the signal to the active
287 				 * thread to avoid deadlock.
288 				 */
289 				if (qr->ps_single)
290 					ptsignal(qr->ps_single, SIGKILL,
291 					    STHREAD);
292 				else
293 					prsignal(qr, SIGKILL);
294 			}
295 		}
296 	}
297 
298 
299 	/* add thread's accumulated rusage into the process's total */
300 	ruadd(rup, &p->p_ru);
301 
302 	/*
303 	 * clear %cpu usage during swap
304 	 */
305 	p->p_pctcpu = 0;
306 
307 	if ((p->p_flag & P_THREAD) == 0) {
308 		/*
309 		 * Final thread has died, so add on our children's rusage
310 		 * and calculate the total times
311 		 */
312 		calcru(&pr->ps_tu, &rup->ru_utime, &rup->ru_stime, NULL);
313 		ruadd(rup, &pr->ps_cru);
314 
315 		/* notify interested parties of our demise and clean up */
316 		knote_processexit(pr);
317 
318 		/*
319 		 * Notify parent that we're gone.  If we have P_NOZOMBIE
320 		 * or parent has the SAS_NOCLDWAIT flag set, notify process 1
321 		 * instead (and hope it will handle this situation).
322 		 */
323 		if ((p->p_flag & P_NOZOMBIE) ||
324 		    (pr->ps_pptr->ps_mainproc->p_sigacts->ps_flags &
325 		    SAS_NOCLDWAIT)) {
326 			struct process *ppr = pr->ps_pptr;
327 			proc_reparent(pr, initproc->p_p);
328 
329 			/*
330 			 * Notify parent, so in case he was wait(2)ing or
331 			 * executing waitpid(2) with our pid, he will
332 			 * continue.
333 			 */
334 			wakeup(ppr);
335 		}
336 	}
337 
338 	/*
339 	 * Release the process's signal state.
340 	 */
341 	sigactsfree(p);
342 
343 	/*
344 	 * Other substructures are freed from reaper and wait().
345 	 */
346 
347 	/*
348 	 * Finally, call machine-dependent code to switch to a new
349 	 * context (possibly the idle context).  Once we are no longer
350 	 * using the dead process's vmspace and stack, exit2() will be
351 	 * called to schedule those resources to be released by the
352 	 * reaper thread.
353 	 *
354 	 * Note that cpu_exit() will end with a call equivalent to
355 	 * cpu_switch(), finishing our execution (pun intended).
356 	 */
357 	uvmexp.swtch++;
358 	cpu_exit(p);
359 	panic("cpu_exit returned");
360 }
361 
362 /*
363  * Locking of this proclist is special; it's accessed in a
364  * critical section of process exit, and thus locking it can't
365  * modify interrupt state.  We use a simple spin lock for this
366  * proclist.  Processes on this proclist are also on zombproc;
367  * we use the p_hash member to linkup to deadproc.
368  */
369 struct mutex deadproc_mutex = MUTEX_INITIALIZER(IPL_NONE);
370 struct proclist deadproc = LIST_HEAD_INITIALIZER(deadproc);
371 
372 /*
373  * We are called from cpu_exit() once it is safe to schedule the
374  * dead process's resources to be freed.
375  *
376  * NOTE: One must be careful with locking in this routine.  It's
377  * called from a critical section in machine-dependent code, so
378  * we should refrain from changing any interrupt state.
379  *
380  * We lock the deadproc list, place the proc on that list (using
381  * the p_hash member), and wake up the reaper.
382  */
383 void
384 exit2(struct proc *p)
385 {
386 	mtx_enter(&deadproc_mutex);
387 	LIST_INSERT_HEAD(&deadproc, p, p_hash);
388 	mtx_leave(&deadproc_mutex);
389 
390 	wakeup(&deadproc);
391 }
392 
393 /*
394  * Process reaper.  This is run by a kernel thread to free the resources
395  * of a dead process.  Once the resources are free, the process becomes
396  * a zombie, and the parent is allowed to read the undead's status.
397  */
398 void
399 reaper(void)
400 {
401 	struct proc *p;
402 
403 	KERNEL_UNLOCK();
404 
405 	SCHED_ASSERT_UNLOCKED();
406 
407 	for (;;) {
408 		mtx_enter(&deadproc_mutex);
409 		while ((p = LIST_FIRST(&deadproc)) == NULL)
410 			msleep(&deadproc, &deadproc_mutex, PVM, "reaper", 0);
411 
412 		/* Remove us from the deadproc list. */
413 		LIST_REMOVE(p, p_hash);
414 		mtx_leave(&deadproc_mutex);
415 
416 		KERNEL_LOCK();
417 
418 		/*
419 		 * Free the VM resources we're still holding on to.
420 		 * We must do this from a valid thread because doing
421 		 * so may block.
422 		 */
423 		uvm_exit(p);
424 
425 		/* Process is now a true zombie. */
426 		if ((p->p_flag & P_NOZOMBIE) == 0) {
427 			p->p_stat = SZOMB;
428 
429 			if (P_EXITSIG(p) != 0)
430 				prsignal(p->p_p->ps_pptr, P_EXITSIG(p));
431 			/* Wake up the parent so it can get exit status. */
432 			wakeup(p->p_p->ps_pptr);
433 		} else {
434 			/* Noone will wait for us. Just zap the process now */
435 			proc_zap(p);
436 		}
437 
438 		KERNEL_UNLOCK();
439 	}
440 }
441 
442 int	dowait4(struct proc *, pid_t, int *, int, struct rusage *,
443 	    register_t *);
444 
445 int
446 sys_wait4(struct proc *q, void *v, register_t *retval)
447 {
448 	struct sys_wait4_args /* {
449 		syscallarg(pid_t) pid;
450 		syscallarg(int *) status;
451 		syscallarg(int) options;
452 		syscallarg(struct rusage *) rusage;
453 	} */ *uap = v;
454 	struct rusage ru;
455 	int error;
456 
457 	error = dowait4(q, SCARG(uap, pid), SCARG(uap, status),
458 	    SCARG(uap, options), SCARG(uap, rusage) ? &ru : NULL, retval);
459 	if (error == 0 && SCARG(uap, rusage)) {
460 		error = copyout(&ru, SCARG(uap, rusage), sizeof(ru));
461 	}
462 	return (error);
463 }
464 
465 int
466 dowait4(struct proc *q, pid_t pid, int *statusp, int options,
467     struct rusage *rusage, register_t *retval)
468 {
469 	int nfound;
470 	struct process *pr;
471 	struct proc *p;
472 	int status, error;
473 
474 	if (pid == 0)
475 		pid = -q->p_p->ps_pgid;
476 	if (options &~ (WUNTRACED|WNOHANG|WALTSIG|WCONTINUED))
477 		return (EINVAL);
478 
479 loop:
480 	nfound = 0;
481 	LIST_FOREACH(pr, &q->p_p->ps_children, ps_sibling) {
482 		p = pr->ps_mainproc;
483 		if ((p->p_flag & P_NOZOMBIE) ||
484 		    (pid != WAIT_ANY &&
485 		    p->p_pid != pid &&
486 		    pr->ps_pgid != -pid))
487 			continue;
488 
489 		/*
490 		 * Wait for processes with p_exitsig != SIGCHLD processes only
491 		 * if WALTSIG is set; wait for processes with pexitsig ==
492 		 * SIGCHLD only if WALTSIG is clear.
493 		 */
494 		if ((options & WALTSIG) ?
495 		    (p->p_exitsig == SIGCHLD) : (P_EXITSIG(p) != SIGCHLD))
496 			continue;
497 
498 		nfound++;
499 		if (p->p_stat == SZOMB) {
500 			retval[0] = p->p_pid;
501 
502 			if (statusp != NULL) {
503 				status = p->p_xstat;	/* convert to int */
504 				error = copyout(&status,
505 				    statusp, sizeof(status));
506 				if (error)
507 					return (error);
508 			}
509 			if (rusage != NULL)
510 				memcpy(rusage, pr->ps_ru, sizeof(*rusage));
511 			proc_finish_wait(q, p);
512 			return (0);
513 		}
514 		if (pr->ps_flags & PS_TRACED &&
515 		    (pr->ps_flags & PS_WAITED) == 0 && pr->ps_single &&
516 		    pr->ps_single->p_stat == SSTOP &&
517 		    (pr->ps_single->p_flag & P_SUSPSINGLE) == 0) {
518 			atomic_setbits_int(&pr->ps_flags, PS_WAITED);
519 			retval[0] = p->p_pid;
520 
521 			if (statusp != NULL) {
522 				status = W_STOPCODE(pr->ps_single->p_xstat);
523 				error = copyout(&status, statusp,
524 				    sizeof(status));
525 			} else
526 				error = 0;
527 			return (error);
528 		}
529 		if (p->p_stat == SSTOP &&
530 		    (pr->ps_flags & PS_WAITED) == 0 &&
531 		    (p->p_flag & P_SUSPSINGLE) == 0 &&
532 		    (pr->ps_flags & PS_TRACED ||
533 		    options & WUNTRACED)) {
534 			atomic_setbits_int(&pr->ps_flags, PS_WAITED);
535 			retval[0] = p->p_pid;
536 
537 			if (statusp != NULL) {
538 				status = W_STOPCODE(p->p_xstat);
539 				error = copyout(&status, statusp,
540 				    sizeof(status));
541 			} else
542 				error = 0;
543 			return (error);
544 		}
545 		if ((options & WCONTINUED) && (p->p_flag & P_CONTINUED)) {
546 			atomic_clearbits_int(&p->p_flag, P_CONTINUED);
547 			retval[0] = p->p_pid;
548 
549 			if (statusp != NULL) {
550 				status = _WCONTINUED;
551 				error = copyout(&status, statusp,
552 				    sizeof(status));
553 			} else
554 				error = 0;
555 			return (error);
556 		}
557 	}
558 	if (nfound == 0)
559 		return (ECHILD);
560 	if (options & WNOHANG) {
561 		retval[0] = 0;
562 		return (0);
563 	}
564 	if ((error = tsleep(q->p_p, PWAIT | PCATCH, "wait", 0)) != 0)
565 		return (error);
566 	goto loop;
567 }
568 
569 void
570 proc_finish_wait(struct proc *waiter, struct proc *p)
571 {
572 	struct process *pr, *tr;
573 	struct rusage *rup;
574 
575 	/*
576 	 * If we got the child via a ptrace 'attach',
577 	 * we need to give it back to the old parent.
578 	 */
579 	pr = p->p_p;
580 	if ((p->p_flag & P_THREAD) == 0 && pr->ps_oppid &&
581 	    (tr = prfind(pr->ps_oppid))) {
582 		atomic_clearbits_int(&pr->ps_flags, PS_TRACED);
583 		pr->ps_oppid = 0;
584 		proc_reparent(pr, tr);
585 		if (p->p_exitsig != 0)
586 			prsignal(tr, p->p_exitsig);
587 		wakeup(tr);
588 	} else {
589 		scheduler_wait_hook(waiter, p);
590 		p->p_xstat = 0;
591 		rup = &waiter->p_p->ps_cru;
592 		ruadd(rup, pr->ps_ru);
593 		proc_zap(p);
594 	}
595 }
596 
597 /*
598  * make process 'parent' the new parent of process 'child'.
599  */
600 void
601 proc_reparent(struct process *child, struct process *parent)
602 {
603 
604 	if (child->ps_pptr == parent)
605 		return;
606 
607 	if (parent == initproc->p_p)
608 		child->ps_mainproc->p_exitsig = SIGCHLD;
609 
610 	LIST_REMOVE(child, ps_sibling);
611 	LIST_INSERT_HEAD(&parent->ps_children, child, ps_sibling);
612 	child->ps_pptr = parent;
613 }
614 
615 void
616 proc_zap(struct proc *p)
617 {
618 	struct process *pr = p->p_p;
619 	struct vnode *otvp;
620 
621 	/*
622 	 * Finally finished with old proc entry.
623 	 * Unlink it from its process group and free it.
624 	 */
625 	if ((p->p_flag & P_THREAD) == 0)
626 		leavepgrp(pr);
627 	LIST_REMOVE(p, p_list);	/* off zombproc */
628 	if ((p->p_flag & P_THREAD) == 0) {
629 		LIST_REMOVE(pr, ps_sibling);
630 
631 		/*
632 		 * Decrement the count of procs running with this uid.
633 		 */
634 		(void)chgproccnt(p->p_cred->p_ruid, -1);
635 	}
636 
637 	/*
638 	 * Release reference to text vnode
639 	 */
640 	otvp = p->p_textvp;
641 	p->p_textvp = NULL;
642 	if (otvp)
643 		vrele(otvp);
644 
645 	/*
646 	 * Remove us from our process list, possibly killing the process
647 	 * in the process (pun intended).
648 	 */
649 	if (--pr->ps_refcnt == 0) {
650 		if (pr->ps_ptstat != NULL)
651 			free(pr->ps_ptstat, M_SUBPROC);
652 		pool_put(&rusage_pool, pr->ps_ru);
653 		KASSERT(TAILQ_EMPTY(&pr->ps_threads));
654 		limfree(pr->ps_limit);
655 		crfree(pr->ps_cred->pc_ucred);
656 		pool_put(&pcred_pool, pr->ps_cred);
657 		pool_put(&process_pool, pr);
658 		nprocesses--;
659 	}
660 
661 	freepid(p->p_pid);
662 	pool_put(&proc_pool, p);
663 	nthreads--;
664 }
665