1 /*
2 * CDDL HEADER START
3 *
4 * The contents of this file are subject to the terms of the
5 * Common Development and Distribution License (the "License").
6 * You may not use this file except in compliance with the License.
7 *
8 * You can obtain a copy of the license at usr/src/OPENSOLARIS.LICENSE
9 * or http://www.opensolaris.org/os/licensing.
10 * See the License for the specific language governing permissions
11 * and limitations under the License.
12 *
13 * When distributing Covered Code, include this CDDL HEADER in each
14 * file and include the License file at usr/src/OPENSOLARIS.LICENSE.
15 * If applicable, add the following below this CDDL HEADER, with the
16 * fields enclosed by brackets "[]" replaced with your own identifying
17 * information: Portions Copyright [yyyy] [name of copyright owner]
18 *
19 * CDDL HEADER END
20 */
21
22 /*
23 * Copyright (c) 2004, 2010, Oracle and/or its affiliates. All rights reserved.
24 */
25
26 /*
27 * method.c - method execution functions
28 *
29 * This file contains the routines needed to run a method: a fork(2)-exec(2)
30 * invocation monitored using either the contract filesystem or waitpid(2).
31 * (Plain fork1(2) support is provided in fork.c.)
32 *
33 * Contract Transfer
34 * When we restart a service, we want to transfer any contracts that the old
35 * service's contract inherited. This means that (a) we must not abandon the
36 * old contract when the service dies and (b) we must write the id of the old
37 * contract into the terms of the new contract. There should be limits to
38 * (a), though, since we don't want to keep the contract around forever. To
39 * this end we'll say that services in the offline state may have a contract
40 * to be transfered and services in the disabled or maintenance states cannot.
41 * This means that when a service transitions from online (or degraded) to
42 * offline, the contract should be preserved, and when the service transitions
43 * from offline to online (i.e., the start method), we'll transfer inherited
44 * contracts.
45 */
46
47 #include <sys/contract/process.h>
48 #include <sys/ctfs.h>
49 #include <sys/stat.h>
50 #include <sys/time.h>
51 #include <sys/types.h>
52 #include <sys/uio.h>
53 #include <sys/wait.h>
54 #include <alloca.h>
55 #include <assert.h>
56 #include <errno.h>
57 #include <fcntl.h>
58 #include <libcontract.h>
59 #include <libcontract_priv.h>
60 #include <libgen.h>
61 #include <librestart.h>
62 #include <libscf.h>
63 #include <limits.h>
64 #include <port.h>
65 #include <sac.h>
66 #include <signal.h>
67 #include <stdlib.h>
68 #include <string.h>
69 #include <strings.h>
70 #include <unistd.h>
71 #include <atomic.h>
72 #include <poll.h>
73
74 #include "startd.h"
75
76 #define SBIN_SH "/sbin/sh"
77
78 /*
79 * Used to tell if contracts are in the process of being
80 * stored into the svc.startd internal hash table.
81 */
82 volatile uint16_t storing_contract = 0;
83
84 /*
85 * Mapping from restart_on method-type to contract events. Must correspond to
86 * enum method_restart_t.
87 */
88 static uint_t method_events[] = {
89 /* METHOD_RESTART_ALL */
90 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE | CT_PR_EV_EMPTY,
91 /* METHOD_RESTART_EXTERNAL_FAULT */
92 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL,
93 /* METHOD_RESTART_ANY_FAULT */
94 CT_PR_EV_HWERR | CT_PR_EV_SIGNAL | CT_PR_EV_CORE
95 };
96
97 /*
98 * method_record_start(restarter_inst_t *)
99 * Record a service start for rate limiting. Place the current time
100 * in the circular array of instance starts.
101 */
102 static void
method_record_start(restarter_inst_t * inst)103 method_record_start(restarter_inst_t *inst)
104 {
105 int index = inst->ri_start_index++ % RINST_START_TIMES;
106
107 inst->ri_start_time[index] = gethrtime();
108 }
109
110 /*
111 * method_rate_critical(restarter_inst_t *)
112 * Return true if the average start interval is less than the permitted
113 * interval. Implicit success if insufficient measurements for an
114 * average exist.
115 */
116 static int
method_rate_critical(restarter_inst_t * inst)117 method_rate_critical(restarter_inst_t *inst)
118 {
119 uint_t n = inst->ri_start_index;
120 hrtime_t avg_ns = 0;
121
122 if (inst->ri_start_index < RINST_START_TIMES)
123 return (0);
124
125 avg_ns =
126 (inst->ri_start_time[(n - 1) % RINST_START_TIMES] -
127 inst->ri_start_time[n % RINST_START_TIMES]) /
128 (RINST_START_TIMES - 1);
129
130 return (avg_ns < RINST_FAILURE_RATE_NS);
131 }
132
133 /*
134 * int method_is_transient()
135 * Determine if the method for the given instance is transient,
136 * from a contract perspective. Return 1 if it is, and 0 if it isn't.
137 */
138 static int
method_is_transient(restarter_inst_t * inst,int type)139 method_is_transient(restarter_inst_t *inst, int type)
140 {
141 if (instance_is_transient_style(inst) || type != METHOD_START)
142 return (1);
143 else
144 return (0);
145 }
146
147 /*
148 * void method_store_contract()
149 * Store the newly created contract id into local structures and
150 * the repository. If the repository connection is broken it is rebound.
151 */
152 static void
method_store_contract(restarter_inst_t * inst,int type,ctid_t * cid)153 method_store_contract(restarter_inst_t *inst, int type, ctid_t *cid)
154 {
155 int r;
156 boolean_t primary;
157
158 if (errno = contract_latest(cid))
159 uu_die("%s: Couldn't get new contract's id", inst->ri_i.i_fmri);
160
161 primary = !method_is_transient(inst, type);
162
163 if (!primary) {
164 if (inst->ri_i.i_transient_ctid != 0) {
165 log_framework(LOG_INFO,
166 "%s: transient ctid expected to be 0 but "
167 "was set to %ld\n", inst->ri_i.i_fmri,
168 inst->ri_i.i_transient_ctid);
169 }
170
171 inst->ri_i.i_transient_ctid = *cid;
172 } else {
173 if (inst->ri_i.i_primary_ctid != 0) {
174 /*
175 * There was an old contract that we transferred.
176 * Remove it.
177 */
178 method_remove_contract(inst, B_TRUE, B_FALSE);
179 }
180
181 if (inst->ri_i.i_primary_ctid != 0) {
182 log_framework(LOG_INFO,
183 "%s: primary ctid expected to be 0 but "
184 "was set to %ld\n", inst->ri_i.i_fmri,
185 inst->ri_i.i_primary_ctid);
186 }
187
188 inst->ri_i.i_primary_ctid = *cid;
189 inst->ri_i.i_primary_ctid_stopped = 0;
190
191 log_framework(LOG_DEBUG, "Storing primary contract %ld for "
192 "%s.\n", *cid, inst->ri_i.i_fmri);
193
194 contract_hash_store(*cid, inst->ri_id);
195 }
196
197 again:
198 if (inst->ri_mi_deleted)
199 return;
200
201 r = restarter_store_contract(inst->ri_m_inst, *cid, primary ?
202 RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
203 switch (r) {
204 case 0:
205 break;
206
207 case ECANCELED:
208 inst->ri_mi_deleted = B_TRUE;
209 break;
210
211 case ECONNABORTED:
212 libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
213 /* FALLTHROUGH */
214
215 case EBADF:
216 libscf_reget_instance(inst);
217 goto again;
218
219 case ENOMEM:
220 case EPERM:
221 case EACCES:
222 case EROFS:
223 uu_die("%s: Couldn't store contract id %ld",
224 inst->ri_i.i_fmri, *cid);
225 /* NOTREACHED */
226
227 case EINVAL:
228 default:
229 bad_error("restarter_store_contract", r);
230 }
231 }
232
233 /*
234 * void method_remove_contract()
235 * Remove any non-permanent contracts from internal structures and
236 * the repository, then abandon them.
237 * Returns
238 * 0 - success
239 * ECANCELED - inst was deleted from the repository
240 *
241 * If the repository connection was broken, it is rebound.
242 */
243 void
method_remove_contract(restarter_inst_t * inst,boolean_t primary,boolean_t abandon)244 method_remove_contract(restarter_inst_t *inst, boolean_t primary,
245 boolean_t abandon)
246 {
247 ctid_t * const ctidp = primary ? &inst->ri_i.i_primary_ctid :
248 &inst->ri_i.i_transient_ctid;
249
250 int r;
251
252 assert(*ctidp != 0);
253
254 log_framework(LOG_DEBUG, "Removing %s contract %lu for %s.\n",
255 primary ? "primary" : "transient", *ctidp, inst->ri_i.i_fmri);
256
257 if (abandon)
258 contract_abandon(*ctidp);
259
260 again:
261 if (inst->ri_mi_deleted) {
262 r = ECANCELED;
263 goto out;
264 }
265
266 r = restarter_remove_contract(inst->ri_m_inst, *ctidp, primary ?
267 RESTARTER_CONTRACT_PRIMARY : RESTARTER_CONTRACT_TRANSIENT);
268 switch (r) {
269 case 0:
270 break;
271
272 case ECANCELED:
273 inst->ri_mi_deleted = B_TRUE;
274 break;
275
276 case ECONNABORTED:
277 libscf_handle_rebind(scf_instance_handle(inst->ri_m_inst));
278 /* FALLTHROUGH */
279
280 case EBADF:
281 libscf_reget_instance(inst);
282 goto again;
283
284 case ENOMEM:
285 case EPERM:
286 case EACCES:
287 case EROFS:
288 log_error(LOG_INFO, "%s: Couldn't remove contract id %ld: "
289 "%s.\n", inst->ri_i.i_fmri, *ctidp, strerror(r));
290 break;
291
292 case EINVAL:
293 default:
294 bad_error("restarter_remove_contract", r);
295 }
296
297 out:
298 if (primary)
299 contract_hash_remove(*ctidp);
300
301 *ctidp = 0;
302 }
303
304 static const char *method_names[] = { "start", "stop", "refresh" };
305
306 /*
307 * int method_ready_contract(restarter_inst_t *, int, method_restart_t, int)
308 *
309 * Activate a contract template for the type method of inst. type,
310 * restart_on, and cte_mask dictate the critical events term of the contract.
311 * Returns
312 * 0 - success
313 * ECANCELED - inst has been deleted from the repository
314 */
315 static int
method_ready_contract(restarter_inst_t * inst,int type,method_restart_t restart_on,uint_t cte_mask)316 method_ready_contract(restarter_inst_t *inst, int type,
317 method_restart_t restart_on, uint_t cte_mask)
318 {
319 int tmpl, err, istrans, iswait, ret;
320 uint_t cevents, fevents;
321
322 /*
323 * Correctly supporting wait-style services is tricky without
324 * rearchitecting startd to cope with multiple event sources
325 * simultaneously trying to stop an instance. Until a better
326 * solution is implemented, we avoid this problem for
327 * wait-style services by making contract events fatal and
328 * letting the wait code alone handle stopping the service.
329 */
330 iswait = instance_is_wait_style(inst);
331 istrans = method_is_transient(inst, type);
332
333 tmpl = open64(CTFS_ROOT "/process/template", O_RDWR);
334 if (tmpl == -1)
335 uu_die("Could not create contract template");
336
337 /*
338 * We assume non-login processes are unlikely to create
339 * multiple process groups, and set CT_PR_PGRPONLY for all
340 * wait-style services' contracts.
341 */
342 err = ct_pr_tmpl_set_param(tmpl, CT_PR_INHERIT | CT_PR_REGENT |
343 (iswait ? CT_PR_PGRPONLY : 0));
344 assert(err == 0);
345
346 if (istrans) {
347 cevents = 0;
348 fevents = 0;
349 } else {
350 assert(restart_on >= 0);
351 assert(restart_on <= METHOD_RESTART_ANY_FAULT);
352 cevents = method_events[restart_on] & ~cte_mask;
353 fevents = iswait ?
354 (method_events[restart_on] & ~cte_mask & CT_PR_ALLFATAL) :
355 0;
356 }
357
358 err = ct_tmpl_set_critical(tmpl, cevents);
359 assert(err == 0);
360
361 err = ct_tmpl_set_informative(tmpl, 0);
362 assert(err == 0);
363 err = ct_pr_tmpl_set_fatal(tmpl, fevents);
364 assert(err == 0);
365
366 err = ct_tmpl_set_cookie(tmpl, istrans ? METHOD_OTHER_COOKIE :
367 METHOD_START_COOKIE);
368 assert(err == 0);
369
370 if (type == METHOD_START && inst->ri_i.i_primary_ctid != 0) {
371 ret = ct_pr_tmpl_set_transfer(tmpl, inst->ri_i.i_primary_ctid);
372 switch (ret) {
373 case 0:
374 break;
375
376 case ENOTEMPTY:
377 /* No contracts for you! */
378 method_remove_contract(inst, B_TRUE, B_TRUE);
379 if (inst->ri_mi_deleted) {
380 ret = ECANCELED;
381 goto out;
382 }
383 break;
384
385 case EINVAL:
386 case ESRCH:
387 case EACCES:
388 default:
389 bad_error("ct_pr_tmpl_set_transfer", ret);
390 }
391 }
392
393 err = ct_pr_tmpl_set_svc_fmri(tmpl, inst->ri_i.i_fmri);
394 assert(err == 0);
395 err = ct_pr_tmpl_set_svc_aux(tmpl, method_names[type]);
396 assert(err == 0);
397
398 err = ct_tmpl_activate(tmpl);
399 assert(err == 0);
400
401 ret = 0;
402
403 out:
404 err = close(tmpl);
405 assert(err == 0);
406
407 return (ret);
408 }
409
410 static void
exec_method(const restarter_inst_t * inst,int type,const char * method,struct method_context * mcp,uint8_t need_session)411 exec_method(const restarter_inst_t *inst, int type, const char *method,
412 struct method_context *mcp, uint8_t need_session)
413 {
414 char *cmd;
415 const char *errf;
416 char **nenv;
417 int rsmc_errno = 0;
418
419 cmd = uu_msprintf("exec %s", method);
420
421 if (inst->ri_utmpx_prefix[0] != '\0' && inst->ri_utmpx_prefix != NULL)
422 (void) utmpx_mark_init(getpid(), inst->ri_utmpx_prefix);
423
424 setlog(inst->ri_logstem);
425 log_instance(inst, B_FALSE, "Executing %s method (\"%s\").",
426 method_names[type], method);
427
428 if (need_session)
429 (void) setpgrp();
430
431 /* Set credentials. */
432 rsmc_errno = restarter_set_method_context(mcp, &errf);
433 if (rsmc_errno != 0) {
434 log_instance(inst, B_FALSE,
435 "svc.startd could not set context for method: ");
436
437 if (rsmc_errno == -1) {
438 if (strcmp(errf, "core_set_process_path") == 0) {
439 log_instance(inst, B_FALSE,
440 "Could not set corefile path.");
441 } else if (strcmp(errf, "setproject") == 0) {
442 log_instance(inst, B_FALSE, "%s: a resource "
443 "control assignment failed", errf);
444 } else if (strcmp(errf, "pool_set_binding") == 0) {
445 log_instance(inst, B_FALSE, "%s: a system "
446 "error occurred", errf);
447 } else {
448 #ifndef NDEBUG
449 uu_warn("%s:%d: Bad function name \"%s\" for "
450 "error %d from "
451 "restarter_set_method_context().\n",
452 __FILE__, __LINE__, errf, rsmc_errno);
453 #endif
454 abort();
455 }
456
457 exit(1);
458 }
459
460 if (errf != NULL && strcmp(errf, "pool_set_binding") == 0) {
461 switch (rsmc_errno) {
462 case ENOENT:
463 log_instance(inst, B_FALSE, "%s: the pool "
464 "could not be found", errf);
465 break;
466
467 case EBADF:
468 log_instance(inst, B_FALSE, "%s: the "
469 "configuration is invalid", errf);
470 break;
471
472 case EINVAL:
473 log_instance(inst, B_FALSE, "%s: pool name "
474 "\"%s\" is invalid", errf,
475 mcp->resource_pool);
476 break;
477
478 default:
479 #ifndef NDEBUG
480 uu_warn("%s:%d: Bad error %d for function %s "
481 "in restarter_set_method_context().\n",
482 __FILE__, __LINE__, rsmc_errno, errf);
483 #endif
484 abort();
485 }
486
487 exit(SMF_EXIT_ERR_CONFIG);
488 }
489
490 if (errf != NULL) {
491 errno = rsmc_errno;
492 perror(errf);
493
494 switch (rsmc_errno) {
495 case EINVAL:
496 case EPERM:
497 case ENOENT:
498 case ENAMETOOLONG:
499 case ERANGE:
500 case ESRCH:
501 exit(SMF_EXIT_ERR_CONFIG);
502 /* NOTREACHED */
503
504 default:
505 exit(1);
506 }
507 }
508
509 switch (rsmc_errno) {
510 case ENOMEM:
511 log_instance(inst, B_FALSE, "Out of memory.");
512 exit(1);
513 /* NOTREACHED */
514
515 case ENOENT:
516 log_instance(inst, B_FALSE, "Missing passwd entry for "
517 "user.");
518 exit(SMF_EXIT_ERR_CONFIG);
519 /* NOTREACHED */
520
521 default:
522 #ifndef NDEBUG
523 uu_warn("%s:%d: Bad miscellaneous error %d from "
524 "restarter_set_method_context().\n", __FILE__,
525 __LINE__, rsmc_errno);
526 #endif
527 abort();
528 }
529 }
530
531 nenv = set_smf_env(mcp->env, mcp->env_sz, NULL, inst,
532 method_names[type]);
533
534 log_preexec();
535
536 (void) execle(SBIN_SH, SBIN_SH, "-c", cmd, NULL, nenv);
537
538 exit(10);
539 }
540
541 static void
write_status(restarter_inst_t * inst,const char * mname,int stat)542 write_status(restarter_inst_t *inst, const char *mname, int stat)
543 {
544 int r;
545
546 again:
547 if (inst->ri_mi_deleted)
548 return;
549
550 r = libscf_write_method_status(inst->ri_m_inst, mname, stat);
551 switch (r) {
552 case 0:
553 break;
554
555 case ECONNABORTED:
556 libscf_reget_instance(inst);
557 goto again;
558
559 case ECANCELED:
560 inst->ri_mi_deleted = 1;
561 break;
562
563 case EPERM:
564 case EACCES:
565 case EROFS:
566 log_framework(LOG_INFO, "Could not write exit status "
567 "for %s method of %s: %s.\n", mname,
568 inst->ri_i.i_fmri, strerror(r));
569 break;
570
571 case ENAMETOOLONG:
572 default:
573 bad_error("libscf_write_method_status", r);
574 }
575 }
576
577 /*
578 * int method_run()
579 * Execute the type method of instp. If it requires a fork(), wait for it
580 * to return and return its exit code in *exit_code. Otherwise set
581 * *exit_code to 0 if the method succeeds & -1 if it fails. If the
582 * repository connection is broken, it is rebound, but inst may not be
583 * reset.
584 * Returns
585 * 0 - success
586 * EINVAL - A correct method or method context couldn't be retrieved.
587 * EIO - Contract kill failed.
588 * EFAULT - Method couldn't be executed successfully.
589 * ELOOP - Retry threshold exceeded.
590 * ECANCELED - inst was deleted from the repository before method was run
591 * ERANGE - Timeout retry threshold exceeded.
592 * EAGAIN - Failed due to external cause, retry.
593 */
594 int
method_run(restarter_inst_t ** instp,int type,int * exit_code)595 method_run(restarter_inst_t **instp, int type, int *exit_code)
596 {
597 char *method;
598 int ret_status;
599 pid_t pid;
600 method_restart_t restart_on;
601 uint_t cte_mask;
602 uint8_t need_session;
603 scf_handle_t *h;
604 scf_snapshot_t *snap;
605 const char *mname;
606 mc_error_t *m_error;
607 struct method_context *mcp;
608 int result = 0, timeout_fired = 0;
609 int sig, r;
610 boolean_t transient;
611 uint64_t timeout;
612 uint8_t timeout_retry;
613 ctid_t ctid;
614 int ctfd = -1;
615 restarter_inst_t *inst = *instp;
616 int id = inst->ri_id;
617 int forkerr;
618
619 assert(MUTEX_HELD(&inst->ri_lock));
620 assert(instance_in_transition(inst));
621
622 if (inst->ri_mi_deleted)
623 return (ECANCELED);
624
625 *exit_code = 0;
626
627 assert(0 <= type && type <= 2);
628 mname = method_names[type];
629
630 if (type == METHOD_START)
631 inst->ri_pre_online_hook();
632
633 h = scf_instance_handle(inst->ri_m_inst);
634
635 snap = scf_snapshot_create(h);
636 if (snap == NULL ||
637 scf_instance_get_snapshot(inst->ri_m_inst, "running", snap) != 0) {
638 log_framework(LOG_DEBUG,
639 "Could not get running snapshot for %s. "
640 "Using editing version to run method %s.\n",
641 inst->ri_i.i_fmri, mname);
642 scf_snapshot_destroy(snap);
643 snap = NULL;
644 }
645
646 /*
647 * After this point, we may be logging to the instance log.
648 * Make sure we've noted where that log is as a property of
649 * the instance.
650 */
651 r = libscf_note_method_log(inst->ri_m_inst, st->st_log_prefix,
652 inst->ri_logstem);
653 if (r != 0) {
654 log_framework(LOG_WARNING,
655 "%s: couldn't note log location: %s\n",
656 inst->ri_i.i_fmri, strerror(r));
657 }
658
659 if ((method = libscf_get_method(h, type, inst, snap, &restart_on,
660 &cte_mask, &need_session, &timeout, &timeout_retry)) == NULL) {
661 if (errno == LIBSCF_PGROUP_ABSENT) {
662 log_framework(LOG_DEBUG,
663 "%s: instance has no method property group '%s'.\n",
664 inst->ri_i.i_fmri, mname);
665 if (type == METHOD_REFRESH)
666 log_instance(inst, B_TRUE, "No '%s' method "
667 "defined. Treating as :true.", mname);
668 else
669 log_instance(inst, B_TRUE, "Method property "
670 "group '%s' is not present.", mname);
671 scf_snapshot_destroy(snap);
672 return (0);
673 } else if (errno == LIBSCF_PROPERTY_ABSENT) {
674 log_framework(LOG_DEBUG,
675 "%s: instance has no '%s/exec' method property.\n",
676 inst->ri_i.i_fmri, mname);
677 log_instance(inst, B_TRUE, "Method property '%s/exec "
678 "is not present.", mname);
679 scf_snapshot_destroy(snap);
680 return (0);
681 } else {
682 log_error(LOG_WARNING,
683 "%s: instance libscf_get_method failed\n",
684 inst->ri_i.i_fmri);
685 scf_snapshot_destroy(snap);
686 return (EINVAL);
687 }
688 }
689
690 /* open service contract if stopping a non-transient service */
691 if (type == METHOD_STOP && (!instance_is_transient_style(inst))) {
692 if (inst->ri_i.i_primary_ctid == 0) {
693 /* service is not running, nothing to stop */
694 log_framework(LOG_DEBUG, "%s: instance has no primary "
695 "contract, no service to stop.\n",
696 inst->ri_i.i_fmri);
697 scf_snapshot_destroy(snap);
698 return (0);
699 }
700 if ((ctfd = contract_open(inst->ri_i.i_primary_ctid, "process",
701 "events", O_RDONLY)) < 0) {
702 result = EFAULT;
703 log_instance(inst, B_TRUE, "Could not open service "
704 "contract %ld. Stop method not run.",
705 inst->ri_i.i_primary_ctid);
706 goto out;
707 }
708 }
709
710 if (restarter_is_null_method(method)) {
711 log_framework(LOG_DEBUG, "%s: null method succeeds\n",
712 inst->ri_i.i_fmri);
713
714 log_instance(inst, B_TRUE, "Executing %s method (null).",
715 mname);
716
717 if (type == METHOD_START)
718 write_status(inst, mname, 0);
719 goto out;
720 }
721
722 sig = restarter_is_kill_method(method);
723 if (sig >= 0) {
724
725 if (inst->ri_i.i_primary_ctid == 0) {
726 log_error(LOG_ERR, "%s: :kill with no contract\n",
727 inst->ri_i.i_fmri);
728 log_instance(inst, B_TRUE, "Invalid use of \":kill\" "
729 "as stop method for transient service.");
730 result = EINVAL;
731 goto out;
732 }
733
734 log_framework(LOG_DEBUG,
735 "%s: :killing contract with signal %d\n",
736 inst->ri_i.i_fmri, sig);
737
738 log_instance(inst, B_TRUE, "Executing %s method (:kill).",
739 mname);
740
741 if (contract_kill(inst->ri_i.i_primary_ctid, sig,
742 inst->ri_i.i_fmri) != 0) {
743 result = EIO;
744 goto out;
745 } else
746 goto assured_kill;
747 }
748
749 log_framework(LOG_DEBUG, "%s: forking to run method %s\n",
750 inst->ri_i.i_fmri, method);
751
752 m_error = restarter_get_method_context(RESTARTER_METHOD_CONTEXT_VERSION,
753 inst->ri_m_inst, snap, mname, method, &mcp);
754
755 if (m_error != NULL) {
756 log_instance(inst, B_TRUE, "%s", m_error->msg);
757 restarter_mc_error_destroy(m_error);
758 result = EINVAL;
759 goto out;
760 }
761
762 r = method_ready_contract(inst, type, restart_on, cte_mask);
763 if (r != 0) {
764 assert(r == ECANCELED);
765 assert(inst->ri_mi_deleted);
766 restarter_free_method_context(mcp);
767 result = ECANCELED;
768 goto out;
769 }
770
771 /*
772 * Validate safety of method contexts, to save children work.
773 */
774 if (!restarter_rm_libs_loadable())
775 log_framework(LOG_DEBUG, "%s: method contexts limited "
776 "to root-accessible libraries\n", inst->ri_i.i_fmri);
777
778 /*
779 * If the service is restarting too quickly, send it to
780 * maintenance.
781 */
782 if (type == METHOD_START) {
783 method_record_start(inst);
784 if (method_rate_critical(inst)) {
785 log_instance(inst, B_TRUE, "Restarting too quickly, "
786 "changing state to maintenance.");
787 result = ELOOP;
788 restarter_free_method_context(mcp);
789 goto out;
790 }
791 }
792
793 atomic_add_16(&storing_contract, 1);
794 pid = startd_fork1(&forkerr);
795 if (pid == 0)
796 exec_method(inst, type, method, mcp, need_session);
797
798 if (pid == -1) {
799 atomic_add_16(&storing_contract, -1);
800 if (forkerr == EAGAIN)
801 result = EAGAIN;
802 else
803 result = EFAULT;
804
805 log_error(LOG_WARNING,
806 "%s: Couldn't fork to execute method %s: %s\n",
807 inst->ri_i.i_fmri, method, strerror(forkerr));
808
809 restarter_free_method_context(mcp);
810 goto out;
811 }
812
813
814 /*
815 * Get the contract id, decide whether it is primary or transient, and
816 * stash it in inst & the repository.
817 */
818 method_store_contract(inst, type, &ctid);
819 atomic_add_16(&storing_contract, -1);
820
821 restarter_free_method_context(mcp);
822
823 /*
824 * Similarly for the start method PID.
825 */
826 if (type == METHOD_START && !inst->ri_mi_deleted)
827 (void) libscf_write_start_pid(inst->ri_m_inst, pid);
828
829 if (instance_is_wait_style(inst) && type == METHOD_START) {
830 /* Wait style instances don't get timeouts on start methods. */
831 if (wait_register(pid, inst->ri_i.i_fmri, 1, 0)) {
832 log_error(LOG_WARNING,
833 "%s: couldn't register %ld for wait\n",
834 inst->ri_i.i_fmri, pid);
835 result = EFAULT;
836 goto contract_out;
837 }
838 write_status(inst, mname, 0);
839
840 } else {
841 int r, err;
842 time_t start_time;
843 time_t end_time;
844
845 /*
846 * Because on upgrade/live-upgrade we may have no chance
847 * to override faulty timeout values on the way to
848 * manifest import, all services on the path to manifest
849 * import are treated the same as INFINITE timeout services.
850 */
851
852 start_time = time(NULL);
853 if (timeout != METHOD_TIMEOUT_INFINITE && !is_timeout_ovr(inst))
854 timeout_insert(inst, ctid, timeout);
855 else
856 timeout = METHOD_TIMEOUT_INFINITE;
857
858 /* Unlock the instance while waiting for the method. */
859 MUTEX_UNLOCK(&inst->ri_lock);
860
861 do {
862 r = waitpid(pid, &ret_status, NULL);
863 } while (r == -1 && errno == EINTR);
864 if (r == -1)
865 err = errno;
866
867 /* Re-grab the lock. */
868 inst = inst_lookup_by_id(id);
869
870 /*
871 * inst can't be removed, as the removal thread waits
872 * for completion of this one.
873 */
874 assert(inst != NULL);
875 *instp = inst;
876
877 if (inst->ri_timeout != NULL && inst->ri_timeout->te_fired)
878 timeout_fired = 1;
879
880 timeout_remove(inst, ctid);
881
882 log_framework(LOG_DEBUG,
883 "%s method for %s exited with status %d.\n", mname,
884 inst->ri_i.i_fmri, WEXITSTATUS(ret_status));
885
886 if (r == -1) {
887 log_error(LOG_WARNING,
888 "Couldn't waitpid() for %s method of %s (%s).\n",
889 mname, inst->ri_i.i_fmri, strerror(err));
890 result = EFAULT;
891 goto contract_out;
892 }
893
894 if (type == METHOD_START)
895 write_status(inst, mname, ret_status);
896
897 /* return ERANGE if this service doesn't retry on timeout */
898 if (timeout_fired == 1 && timeout_retry == 0) {
899 result = ERANGE;
900 goto contract_out;
901 }
902
903 if (!WIFEXITED(ret_status)) {
904 /*
905 * If method didn't exit itself (it was killed by an
906 * external entity, etc.), consider the entire
907 * method_run as failed.
908 */
909 if (WIFSIGNALED(ret_status)) {
910 char buf[SIG2STR_MAX];
911 (void) sig2str(WTERMSIG(ret_status), buf);
912
913 log_error(LOG_WARNING, "%s: Method \"%s\" "
914 "failed due to signal %s.\n",
915 inst->ri_i.i_fmri, method, buf);
916 log_instance(inst, B_TRUE, "Method \"%s\" "
917 "failed due to signal %s.", mname, buf);
918 } else {
919 log_error(LOG_WARNING, "%s: Method \"%s\" "
920 "failed with exit status %d.\n",
921 inst->ri_i.i_fmri, method,
922 WEXITSTATUS(ret_status));
923 log_instance(inst, B_TRUE, "Method \"%s\" "
924 "failed with exit status %d.", mname,
925 WEXITSTATUS(ret_status));
926 }
927 result = EAGAIN;
928 goto contract_out;
929 }
930
931 *exit_code = WEXITSTATUS(ret_status);
932 if (*exit_code != 0) {
933 log_error(LOG_WARNING,
934 "%s: Method \"%s\" failed with exit status %d.\n",
935 inst->ri_i.i_fmri, method, WEXITSTATUS(ret_status));
936 }
937
938 log_instance(inst, B_TRUE, "Method \"%s\" exited with status "
939 "%d.", mname, *exit_code);
940
941 if (*exit_code != 0)
942 goto contract_out;
943
944 end_time = time(NULL);
945
946 /* Give service contract remaining seconds to empty */
947 if (timeout != METHOD_TIMEOUT_INFINITE)
948 timeout -= (end_time - start_time);
949 }
950
951 assured_kill:
952 /*
953 * For stop methods, assure that the service contract has emptied
954 * before returning.
955 */
956 if (type == METHOD_STOP && (!instance_is_transient_style(inst)) &&
957 !(contract_is_empty(inst->ri_i.i_primary_ctid))) {
958 int times = 0;
959
960 if (timeout != METHOD_TIMEOUT_INFINITE)
961 timeout_insert(inst, inst->ri_i.i_primary_ctid,
962 timeout);
963
964 for (;;) {
965 /*
966 * Check frequently at first, then back off. This
967 * keeps startd from idling while shutting down.
968 */
969 if (times < 20) {
970 (void) poll(NULL, 0, 5);
971 times++;
972 } else {
973 (void) poll(NULL, 0, 100);
974 }
975 if (contract_is_empty(inst->ri_i.i_primary_ctid))
976 break;
977 }
978
979 if (timeout != METHOD_TIMEOUT_INFINITE)
980 if (inst->ri_timeout->te_fired)
981 result = EFAULT;
982
983 timeout_remove(inst, inst->ri_i.i_primary_ctid);
984 }
985
986 contract_out:
987 /* Abandon contracts for transient methods & methods that fail. */
988 transient = method_is_transient(inst, type);
989 if ((transient || *exit_code != 0 || result != 0) &&
990 (restarter_is_kill_method(method) < 0))
991 method_remove_contract(inst, !transient, B_TRUE);
992
993 out:
994 if (ctfd >= 0)
995 (void) close(ctfd);
996 scf_snapshot_destroy(snap);
997 free(method);
998 return (result);
999 }
1000
1001 /*
1002 * The method thread executes a service method to effect a state transition.
1003 * The next_state of info->sf_id should be non-_NONE on entrance, and it will
1004 * be _NONE on exit (state will either be what next_state was (on success), or
1005 * it will be _MAINT (on error)).
1006 *
1007 * There are six classes of methods to consider: start & other (stop, refresh)
1008 * for each of "normal" services, wait services, and transient services. For
1009 * each, the method must be fetched from the repository & executed. fork()ed
1010 * methods must be waited on, except for the start method of wait services
1011 * (which must be registered with the wait subsystem via wait_register()). If
1012 * the method succeeded (returned 0), then for start methods its contract
1013 * should be recorded as the primary contract for the service. For other
1014 * methods, it should be abandoned. If the method fails, then depending on
1015 * the failure, either the method should be reexecuted or the service should
1016 * be put into maintenance. Either way the contract should be abandoned.
1017 */
1018 void *
method_thread(void * arg)1019 method_thread(void *arg)
1020 {
1021 fork_info_t *info = arg;
1022 restarter_inst_t *inst;
1023 scf_handle_t *local_handle;
1024 scf_instance_t *s_inst = NULL;
1025 int r, exit_code;
1026 boolean_t retryable;
1027 restarter_str_t reason;
1028
1029 assert(0 <= info->sf_method_type && info->sf_method_type <= 2);
1030
1031 /* Get (and lock) the restarter_inst_t. */
1032 inst = inst_lookup_by_id(info->sf_id);
1033
1034 assert(inst->ri_method_thread != 0);
1035 assert(instance_in_transition(inst) == 1);
1036
1037 /*
1038 * We cannot leave this function with inst in transition, because
1039 * protocol.c withholds messages for inst otherwise.
1040 */
1041
1042 log_framework(LOG_DEBUG, "method_thread() running %s method for %s.\n",
1043 method_names[info->sf_method_type], inst->ri_i.i_fmri);
1044
1045 local_handle = libscf_handle_create_bound_loop();
1046
1047 rebind_retry:
1048 /* get scf_instance_t */
1049 switch (r = libscf_fmri_get_instance(local_handle, inst->ri_i.i_fmri,
1050 &s_inst)) {
1051 case 0:
1052 break;
1053
1054 case ECONNABORTED:
1055 libscf_handle_rebind(local_handle);
1056 goto rebind_retry;
1057
1058 case ENOENT:
1059 /*
1060 * It's not there, but we need to call this so protocol.c
1061 * doesn't think it's in transition anymore.
1062 */
1063 (void) restarter_instance_update_states(local_handle, inst,
1064 inst->ri_i.i_state, RESTARTER_STATE_NONE, RERR_NONE,
1065 restarter_str_none);
1066 goto out;
1067
1068 case EINVAL:
1069 case ENOTSUP:
1070 default:
1071 bad_error("libscf_fmri_get_instance", r);
1072 }
1073
1074 inst->ri_m_inst = s_inst;
1075 inst->ri_mi_deleted = B_FALSE;
1076
1077 retry:
1078 if (info->sf_method_type == METHOD_START)
1079 log_transition(inst, START_REQUESTED);
1080
1081 r = method_run(&inst, info->sf_method_type, &exit_code);
1082
1083 if (r == 0 && exit_code == 0) {
1084 /* Success! */
1085 assert(inst->ri_i.i_next_state != RESTARTER_STATE_NONE);
1086
1087 /*
1088 * When a stop method succeeds, remove the primary contract of
1089 * the service, unless we're going to offline, in which case
1090 * retain the contract so we can transfer inherited contracts to
1091 * the replacement service.
1092 */
1093
1094 if (info->sf_method_type == METHOD_STOP &&
1095 inst->ri_i.i_primary_ctid != 0) {
1096 if (inst->ri_i.i_next_state == RESTARTER_STATE_OFFLINE)
1097 inst->ri_i.i_primary_ctid_stopped = 1;
1098 else
1099 method_remove_contract(inst, B_TRUE, B_TRUE);
1100 }
1101 /*
1102 * We don't care whether the handle was rebound because this is
1103 * the last thing we do with it.
1104 */
1105 (void) restarter_instance_update_states(local_handle, inst,
1106 inst->ri_i.i_next_state, RESTARTER_STATE_NONE,
1107 info->sf_event_type, info->sf_reason);
1108
1109 (void) update_fault_count(inst, FAULT_COUNT_RESET);
1110
1111 goto out;
1112 }
1113
1114 /* Failure. Retry or go to maintenance. */
1115
1116 if (r != 0 && r != EAGAIN) {
1117 retryable = B_FALSE;
1118 } else {
1119 switch (exit_code) {
1120 case SMF_EXIT_ERR_CONFIG:
1121 case SMF_EXIT_ERR_NOSMF:
1122 case SMF_EXIT_ERR_PERM:
1123 case SMF_EXIT_ERR_FATAL:
1124 retryable = B_FALSE;
1125 break;
1126
1127 default:
1128 retryable = B_TRUE;
1129 }
1130 }
1131
1132 if (retryable && update_fault_count(inst, FAULT_COUNT_INCR) != 1)
1133 goto retry;
1134
1135 /* maintenance */
1136 if (r == ELOOP)
1137 log_transition(inst, START_FAILED_REPEATEDLY);
1138 else if (r == ERANGE)
1139 log_transition(inst, START_FAILED_TIMEOUT_FATAL);
1140 else if (exit_code == SMF_EXIT_ERR_CONFIG)
1141 log_transition(inst, START_FAILED_CONFIGURATION);
1142 else if (exit_code == SMF_EXIT_ERR_FATAL)
1143 log_transition(inst, START_FAILED_FATAL);
1144 else
1145 log_transition(inst, START_FAILED_OTHER);
1146
1147 if (r == ELOOP) {
1148 reason = restarter_str_restarting_too_quickly;
1149 } else if (retryable) {
1150 reason = restarter_str_fault_threshold_reached;
1151 } else {
1152 reason = restarter_str_method_failed;
1153 }
1154
1155 (void) restarter_instance_update_states(local_handle, inst,
1156 RESTARTER_STATE_MAINT, RESTARTER_STATE_NONE, RERR_FAULT,
1157 reason);
1158
1159 if (!method_is_transient(inst, info->sf_method_type) &&
1160 inst->ri_i.i_primary_ctid != 0)
1161 method_remove_contract(inst, B_TRUE, B_TRUE);
1162
1163 out:
1164 inst->ri_method_thread = 0;
1165
1166 /*
1167 * Unlock the mutex after broadcasting to avoid a race condition
1168 * with restarter_delete_inst() when the 'inst' structure is freed.
1169 */
1170 (void) pthread_cond_broadcast(&inst->ri_method_cv);
1171 MUTEX_UNLOCK(&inst->ri_lock);
1172
1173 scf_instance_destroy(s_inst);
1174 scf_handle_destroy(local_handle);
1175 startd_free(info, sizeof (fork_info_t));
1176 return (NULL);
1177 }
1178