1 /* $OpenBSD: mta.c,v 1.228 2019/06/14 19:55:25 eric Exp $ */ 2 3 /* 4 * Copyright (c) 2008 Pierre-Yves Ritschard <pyr@openbsd.org> 5 * Copyright (c) 2008 Gilles Chehade <gilles@poolp.org> 6 * Copyright (c) 2009 Jacek Masiulaniec <jacekm@dobremiasto.net> 7 * Copyright (c) 2012 Eric Faurot <eric@openbsd.org> 8 * 9 * Permission to use, copy, modify, and distribute this software for any 10 * purpose with or without fee is hereby granted, provided that the above 11 * copyright notice and this permission notice appear in all copies. 12 * 13 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 14 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 15 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 16 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 17 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 18 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 19 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 20 */ 21 22 #include <sys/types.h> 23 #include <sys/queue.h> 24 #include <sys/tree.h> 25 #include <sys/socket.h> 26 27 #include <ctype.h> 28 #include <err.h> 29 #include <errno.h> 30 #include <event.h> 31 #include <imsg.h> 32 #include <inttypes.h> 33 #include <netdb.h> 34 #include <limits.h> 35 #include <pwd.h> 36 #include <signal.h> 37 #include <stdio.h> 38 #include <stdlib.h> 39 #include <string.h> 40 #include <time.h> 41 #include <unistd.h> 42 43 #include "smtpd.h" 44 #include "log.h" 45 46 #define MAXERROR_PER_ROUTE 4 47 48 #define DELAY_CHECK_SOURCE 1 49 #define DELAY_CHECK_SOURCE_SLOW 10 50 #define DELAY_CHECK_SOURCE_FAST 0 51 #define DELAY_CHECK_LIMIT 5 52 53 #define DELAY_QUADRATIC 1 54 #define DELAY_ROUTE_BASE 15 55 #define DELAY_ROUTE_MAX 3600 56 57 #define RELAY_ONHOLD 0x01 58 #define RELAY_HOLDQ 0x02 59 60 static void mta_handle_envelope(struct envelope *, const char *); 61 static void mta_query_smarthost(struct envelope *); 62 static void mta_on_smarthost(struct envelope *, const char *); 63 static void mta_query_mx(struct mta_relay *); 64 static void mta_query_secret(struct mta_relay *); 65 static void mta_query_preference(struct mta_relay *); 66 static void mta_query_source(struct mta_relay *); 67 static void mta_on_mx(void *, void *, void *); 68 static void mta_on_secret(struct mta_relay *, const char *); 69 static void mta_on_preference(struct mta_relay *, int); 70 static void mta_on_source(struct mta_relay *, struct mta_source *); 71 static void mta_on_timeout(struct runq *, void *); 72 static void mta_connect(struct mta_connector *); 73 static void mta_route_enable(struct mta_route *); 74 static void mta_route_disable(struct mta_route *, int, int); 75 static void mta_drain(struct mta_relay *); 76 static void mta_delivery_flush_event(int, short, void *); 77 static void mta_flush(struct mta_relay *, int, const char *); 78 static struct mta_route *mta_find_route(struct mta_connector *, time_t, int*, 79 time_t*); 80 static void mta_log(const struct mta_envelope *, const char *, const char *, 81 const char *, const char *); 82 83 SPLAY_HEAD(mta_relay_tree, mta_relay); 84 static struct mta_relay *mta_relay(struct envelope *, struct relayhost *); 85 static void mta_relay_ref(struct mta_relay *); 86 static void mta_relay_unref(struct mta_relay *); 87 static void mta_relay_show(struct mta_relay *, struct mproc *, uint32_t, time_t); 88 static int mta_relay_cmp(const struct mta_relay *, const struct mta_relay *); 89 SPLAY_PROTOTYPE(mta_relay_tree, mta_relay, entry, mta_relay_cmp); 90 91 SPLAY_HEAD(mta_host_tree, mta_host); 92 static struct mta_host *mta_host(const struct sockaddr *); 93 static void mta_host_ref(struct mta_host *); 94 static void mta_host_unref(struct mta_host *); 95 static int mta_host_cmp(const struct mta_host *, const struct mta_host *); 96 SPLAY_PROTOTYPE(mta_host_tree, mta_host, entry, mta_host_cmp); 97 98 SPLAY_HEAD(mta_domain_tree, mta_domain); 99 static struct mta_domain *mta_domain(char *, int); 100 #if 0 101 static void mta_domain_ref(struct mta_domain *); 102 #endif 103 static void mta_domain_unref(struct mta_domain *); 104 static int mta_domain_cmp(const struct mta_domain *, const struct mta_domain *); 105 SPLAY_PROTOTYPE(mta_domain_tree, mta_domain, entry, mta_domain_cmp); 106 107 SPLAY_HEAD(mta_source_tree, mta_source); 108 static struct mta_source *mta_source(const struct sockaddr *); 109 static void mta_source_ref(struct mta_source *); 110 static void mta_source_unref(struct mta_source *); 111 static const char *mta_source_to_text(struct mta_source *); 112 static int mta_source_cmp(const struct mta_source *, const struct mta_source *); 113 SPLAY_PROTOTYPE(mta_source_tree, mta_source, entry, mta_source_cmp); 114 115 static struct mta_connector *mta_connector(struct mta_relay *, 116 struct mta_source *); 117 static void mta_connector_free(struct mta_connector *); 118 static const char *mta_connector_to_text(struct mta_connector *); 119 120 SPLAY_HEAD(mta_route_tree, mta_route); 121 static struct mta_route *mta_route(struct mta_source *, struct mta_host *); 122 static void mta_route_ref(struct mta_route *); 123 static void mta_route_unref(struct mta_route *); 124 static const char *mta_route_to_text(struct mta_route *); 125 static int mta_route_cmp(const struct mta_route *, const struct mta_route *); 126 SPLAY_PROTOTYPE(mta_route_tree, mta_route, entry, mta_route_cmp); 127 128 struct mta_block { 129 SPLAY_ENTRY(mta_block) entry; 130 struct mta_source *source; 131 char *domain; 132 }; 133 134 SPLAY_HEAD(mta_block_tree, mta_block); 135 void mta_block(struct mta_source *, char *); 136 void mta_unblock(struct mta_source *, char *); 137 int mta_is_blocked(struct mta_source *, char *); 138 static int mta_block_cmp(const struct mta_block *, const struct mta_block *); 139 SPLAY_PROTOTYPE(mta_block_tree, mta_block, entry, mta_block_cmp); 140 141 static struct mta_relay_tree relays; 142 static struct mta_domain_tree domains; 143 static struct mta_host_tree hosts; 144 static struct mta_source_tree sources; 145 static struct mta_route_tree routes; 146 static struct mta_block_tree blocks; 147 148 static struct tree wait_mx; 149 static struct tree wait_preference; 150 static struct tree wait_secret; 151 static struct tree wait_smarthost; 152 static struct tree wait_source; 153 static struct tree flush_evp; 154 static struct event ev_flush_evp; 155 156 static struct runq *runq_relay; 157 static struct runq *runq_connector; 158 static struct runq *runq_route; 159 static struct runq *runq_hoststat; 160 161 static time_t max_seen_conndelay_route; 162 static time_t max_seen_discdelay_route; 163 164 #define HOSTSTAT_EXPIRE_DELAY (4 * 3600) 165 struct hoststat { 166 char name[HOST_NAME_MAX+1]; 167 time_t tm; 168 char error[LINE_MAX]; 169 struct tree deferred; 170 }; 171 static struct dict hoststat; 172 173 void mta_hoststat_update(const char *, const char *); 174 void mta_hoststat_cache(const char *, uint64_t); 175 void mta_hoststat_uncache(const char *, uint64_t); 176 void mta_hoststat_reschedule(const char *); 177 static void mta_hoststat_remove_entry(struct hoststat *); 178 179 void 180 mta_imsg(struct mproc *p, struct imsg *imsg) 181 { 182 struct mta_relay *relay; 183 struct mta_domain *domain; 184 struct mta_host *host; 185 struct mta_route *route; 186 struct mta_block *block; 187 struct mta_mx *mx, *imx; 188 struct mta_source *source; 189 struct hoststat *hs; 190 struct sockaddr_storage ss; 191 struct envelope evp, *e; 192 struct msg m; 193 const char *secret; 194 const char *hostname; 195 const char *dom; 196 const char *smarthost; 197 uint64_t reqid; 198 time_t t; 199 char buf[LINE_MAX]; 200 int dnserror, preference, v, status; 201 void *iter; 202 uint64_t u64; 203 204 switch (imsg->hdr.type) { 205 case IMSG_QUEUE_TRANSFER: 206 m_msg(&m, imsg); 207 m_get_envelope(&m, &evp); 208 m_end(&m); 209 mta_handle_envelope(&evp, NULL); 210 return; 211 212 case IMSG_MTA_OPEN_MESSAGE: 213 mta_session_imsg(p, imsg); 214 return; 215 216 case IMSG_MTA_LOOKUP_CREDENTIALS: 217 m_msg(&m, imsg); 218 m_get_id(&m, &reqid); 219 m_get_string(&m, &secret); 220 m_end(&m); 221 relay = tree_xpop(&wait_secret, reqid); 222 mta_on_secret(relay, secret[0] ? secret : NULL); 223 return; 224 225 case IMSG_MTA_LOOKUP_SOURCE: 226 m_msg(&m, imsg); 227 m_get_id(&m, &reqid); 228 m_get_int(&m, &status); 229 if (status == LKA_OK) 230 m_get_sockaddr(&m, (struct sockaddr*)&ss); 231 m_end(&m); 232 233 relay = tree_xpop(&wait_source, reqid); 234 mta_on_source(relay, (status == LKA_OK) ? 235 mta_source((struct sockaddr *)&ss) : NULL); 236 return; 237 238 case IMSG_MTA_LOOKUP_SMARTHOST: 239 m_msg(&m, imsg); 240 m_get_id(&m, &reqid); 241 m_get_int(&m, &status); 242 smarthost = NULL; 243 if (status == LKA_OK) 244 m_get_string(&m, &smarthost); 245 m_end(&m); 246 247 e = tree_xpop(&wait_smarthost, reqid); 248 mta_on_smarthost(e, smarthost); 249 return; 250 251 case IMSG_MTA_LOOKUP_HELO: 252 mta_session_imsg(p, imsg); 253 return; 254 255 case IMSG_MTA_DNS_HOST: 256 m_msg(&m, imsg); 257 m_get_id(&m, &reqid); 258 m_get_sockaddr(&m, (struct sockaddr*)&ss); 259 m_get_int(&m, &preference); 260 m_end(&m); 261 domain = tree_xget(&wait_mx, reqid); 262 mx = xcalloc(1, sizeof *mx); 263 mx->host = mta_host((struct sockaddr*)&ss); 264 mx->preference = preference; 265 TAILQ_FOREACH(imx, &domain->mxs, entry) { 266 if (imx->preference > mx->preference) { 267 TAILQ_INSERT_BEFORE(imx, mx, entry); 268 return; 269 } 270 } 271 TAILQ_INSERT_TAIL(&domain->mxs, mx, entry); 272 return; 273 274 case IMSG_MTA_DNS_HOST_END: 275 m_msg(&m, imsg); 276 m_get_id(&m, &reqid); 277 m_get_int(&m, &dnserror); 278 m_end(&m); 279 domain = tree_xpop(&wait_mx, reqid); 280 domain->mxstatus = dnserror; 281 if (domain->mxstatus == DNS_OK) { 282 log_debug("debug: MXs for domain %s:", 283 domain->name); 284 TAILQ_FOREACH(mx, &domain->mxs, entry) 285 log_debug(" %s preference %d", 286 sa_to_text(mx->host->sa), 287 mx->preference); 288 } 289 else { 290 log_debug("debug: Failed MX query for %s:", 291 domain->name); 292 } 293 domain->lastmxquery = time(NULL); 294 waitq_run(&domain->mxs, domain); 295 return; 296 297 case IMSG_MTA_DNS_MX_PREFERENCE: 298 m_msg(&m, imsg); 299 m_get_id(&m, &reqid); 300 m_get_int(&m, &dnserror); 301 if (dnserror == 0) 302 m_get_int(&m, &preference); 303 m_end(&m); 304 305 relay = tree_xpop(&wait_preference, reqid); 306 if (dnserror) { 307 log_warnx("warn: Couldn't find backup " 308 "preference for %s: error %d", 309 mta_relay_to_text(relay), dnserror); 310 preference = INT_MAX; 311 } 312 mta_on_preference(relay, preference); 313 return; 314 315 case IMSG_CTL_RESUME_ROUTE: 316 u64 = *((uint64_t *)imsg->data); 317 if (u64) 318 log_debug("resuming route: %llu", 319 (unsigned long long)u64); 320 else 321 log_debug("resuming all routes"); 322 SPLAY_FOREACH(route, mta_route_tree, &routes) { 323 if (u64 && route->id != u64) 324 continue; 325 326 if (route->flags & ROUTE_DISABLED) { 327 log_info("smtp-out: Enabling route %s per admin request", 328 mta_route_to_text(route)); 329 if (!runq_cancel(runq_route, route)) { 330 log_warnx("warn: route not on runq"); 331 fatalx("exiting"); 332 } 333 route->flags &= ~ROUTE_DISABLED; 334 route->flags |= ROUTE_NEW; 335 route->nerror = 0; 336 route->penalty = 0; 337 mta_route_unref(route); /* from mta_route_disable */ 338 } 339 340 if (u64) 341 break; 342 } 343 return; 344 345 case IMSG_CTL_MTA_SHOW_HOSTS: 346 t = time(NULL); 347 SPLAY_FOREACH(host, mta_host_tree, &hosts) { 348 (void)snprintf(buf, sizeof(buf), 349 "%s %s refcount=%d nconn=%zu lastconn=%s", 350 sockaddr_to_text(host->sa), 351 host->ptrname, 352 host->refcount, 353 host->nconn, 354 host->lastconn ? duration_to_text(t - host->lastconn) : "-"); 355 m_compose(p, IMSG_CTL_MTA_SHOW_HOSTS, 356 imsg->hdr.peerid, 0, -1, 357 buf, strlen(buf) + 1); 358 } 359 m_compose(p, IMSG_CTL_MTA_SHOW_HOSTS, imsg->hdr.peerid, 360 0, -1, NULL, 0); 361 return; 362 363 case IMSG_CTL_MTA_SHOW_RELAYS: 364 t = time(NULL); 365 SPLAY_FOREACH(relay, mta_relay_tree, &relays) 366 mta_relay_show(relay, p, imsg->hdr.peerid, t); 367 m_compose(p, IMSG_CTL_MTA_SHOW_RELAYS, imsg->hdr.peerid, 368 0, -1, NULL, 0); 369 return; 370 371 case IMSG_CTL_MTA_SHOW_ROUTES: 372 SPLAY_FOREACH(route, mta_route_tree, &routes) { 373 v = runq_pending(runq_route, route, &t); 374 (void)snprintf(buf, sizeof(buf), 375 "%llu. %s %c%c%c%c nconn=%zu nerror=%d penalty=%d timeout=%s", 376 (unsigned long long)route->id, 377 mta_route_to_text(route), 378 route->flags & ROUTE_NEW ? 'N' : '-', 379 route->flags & ROUTE_DISABLED ? 'D' : '-', 380 route->flags & ROUTE_RUNQ ? 'Q' : '-', 381 route->flags & ROUTE_KEEPALIVE ? 'K' : '-', 382 route->nconn, 383 route->nerror, 384 route->penalty, 385 v ? duration_to_text(t - time(NULL)) : "-"); 386 m_compose(p, IMSG_CTL_MTA_SHOW_ROUTES, 387 imsg->hdr.peerid, 0, -1, 388 buf, strlen(buf) + 1); 389 } 390 m_compose(p, IMSG_CTL_MTA_SHOW_ROUTES, imsg->hdr.peerid, 391 0, -1, NULL, 0); 392 return; 393 394 case IMSG_CTL_MTA_SHOW_HOSTSTATS: 395 iter = NULL; 396 while (dict_iter(&hoststat, &iter, &hostname, 397 (void **)&hs)) { 398 (void)snprintf(buf, sizeof(buf), 399 "%s|%llu|%s", 400 hostname, (unsigned long long) hs->tm, 401 hs->error); 402 m_compose(p, IMSG_CTL_MTA_SHOW_HOSTSTATS, 403 imsg->hdr.peerid, 0, -1, 404 buf, strlen(buf) + 1); 405 } 406 m_compose(p, IMSG_CTL_MTA_SHOW_HOSTSTATS, 407 imsg->hdr.peerid, 408 0, -1, NULL, 0); 409 return; 410 411 case IMSG_CTL_MTA_BLOCK: 412 m_msg(&m, imsg); 413 m_get_sockaddr(&m, (struct sockaddr*)&ss); 414 m_get_string(&m, &dom); 415 m_end(&m); 416 source = mta_source((struct sockaddr*)&ss); 417 if (*dom != '\0') { 418 if (!(strlcpy(buf, dom, sizeof(buf)) 419 >= sizeof(buf))) 420 mta_block(source, buf); 421 } 422 else 423 mta_block(source, NULL); 424 mta_source_unref(source); 425 m_compose(p, IMSG_CTL_OK, imsg->hdr.peerid, 0, -1, NULL, 0); 426 return; 427 428 case IMSG_CTL_MTA_UNBLOCK: 429 m_msg(&m, imsg); 430 m_get_sockaddr(&m, (struct sockaddr*)&ss); 431 m_get_string(&m, &dom); 432 m_end(&m); 433 source = mta_source((struct sockaddr*)&ss); 434 if (*dom != '\0') { 435 if (!(strlcpy(buf, dom, sizeof(buf)) 436 >= sizeof(buf))) 437 mta_unblock(source, buf); 438 } 439 else 440 mta_unblock(source, NULL); 441 mta_source_unref(source); 442 m_compose(p, IMSG_CTL_OK, imsg->hdr.peerid, 0, -1, NULL, 0); 443 return; 444 445 case IMSG_CTL_MTA_SHOW_BLOCK: 446 SPLAY_FOREACH(block, mta_block_tree, &blocks) { 447 (void)snprintf(buf, sizeof(buf), "%s -> %s", 448 mta_source_to_text(block->source), 449 block->domain ? block->domain : "*"); 450 m_compose(p, IMSG_CTL_MTA_SHOW_BLOCK, 451 imsg->hdr.peerid, 0, -1, buf, strlen(buf) + 1); 452 } 453 m_compose(p, IMSG_CTL_MTA_SHOW_BLOCK, imsg->hdr.peerid, 454 0, -1, NULL, 0); 455 return; 456 } 457 458 errx(1, "mta_imsg: unexpected %s imsg", imsg_to_str(imsg->hdr.type)); 459 } 460 461 void 462 mta_postfork(void) 463 { 464 } 465 466 void 467 mta_postprivdrop(void) 468 { 469 SPLAY_INIT(&relays); 470 SPLAY_INIT(&domains); 471 SPLAY_INIT(&hosts); 472 SPLAY_INIT(&sources); 473 SPLAY_INIT(&routes); 474 SPLAY_INIT(&blocks); 475 476 tree_init(&wait_secret); 477 tree_init(&wait_smarthost); 478 tree_init(&wait_mx); 479 tree_init(&wait_preference); 480 tree_init(&wait_source); 481 tree_init(&flush_evp); 482 dict_init(&hoststat); 483 484 evtimer_set(&ev_flush_evp, mta_delivery_flush_event, NULL); 485 486 runq_init(&runq_relay, mta_on_timeout); 487 runq_init(&runq_connector, mta_on_timeout); 488 runq_init(&runq_route, mta_on_timeout); 489 runq_init(&runq_hoststat, mta_on_timeout); 490 } 491 492 493 /* 494 * Local error on the given source. 495 */ 496 void 497 mta_source_error(struct mta_relay *relay, struct mta_route *route, const char *e) 498 { 499 struct mta_connector *c; 500 501 /* 502 * Remember the source as broken for this connector. 503 */ 504 c = mta_connector(relay, route->src); 505 if (!(c->flags & CONNECTOR_ERROR_SOURCE)) 506 log_info("smtp-out: Error on %s: %s", 507 mta_route_to_text(route), e); 508 c->flags |= CONNECTOR_ERROR_SOURCE; 509 } 510 511 void 512 mta_route_error(struct mta_relay *relay, struct mta_route *route) 513 { 514 #if 0 515 route->nerror += 1; 516 517 if (route->nerror > MAXERROR_PER_ROUTE) { 518 log_info("smtp-out: Too many errors on %s: " 519 "disabling for a while", mta_route_to_text(route)); 520 mta_route_disable(route, 2, ROUTE_DISABLED_SMTP); 521 } 522 #endif 523 } 524 525 void 526 mta_route_ok(struct mta_relay *relay, struct mta_route *route) 527 { 528 struct mta_connector *c; 529 530 if (!(route->flags & ROUTE_NEW)) 531 return; 532 533 log_debug("debug: mta-routing: route %s is now valid.", 534 mta_route_to_text(route)); 535 536 route->nerror = 0; 537 route->flags &= ~ROUTE_NEW; 538 539 c = mta_connector(relay, route->src); 540 mta_connect(c); 541 } 542 543 void 544 mta_route_down(struct mta_relay *relay, struct mta_route *route) 545 { 546 #if 0 547 mta_route_disable(route, 2, ROUTE_DISABLED_SMTP); 548 #endif 549 } 550 551 void 552 mta_route_collect(struct mta_relay *relay, struct mta_route *route) 553 { 554 struct mta_connector *c; 555 556 log_debug("debug: mta_route_collect(%s)", 557 mta_route_to_text(route)); 558 559 relay->nconn -= 1; 560 relay->domain->nconn -= 1; 561 route->nconn -= 1; 562 route->src->nconn -= 1; 563 route->dst->nconn -= 1; 564 route->lastdisc = time(NULL); 565 566 /* First connection failed */ 567 if (route->flags & ROUTE_NEW) 568 mta_route_disable(route, 1, ROUTE_DISABLED_NET); 569 570 c = mta_connector(relay, route->src); 571 c->nconn -= 1; 572 mta_connect(c); 573 mta_route_unref(route); /* from mta_find_route() */ 574 mta_relay_unref(relay); /* from mta_connect() */ 575 } 576 577 struct mta_task * 578 mta_route_next_task(struct mta_relay *relay, struct mta_route *route) 579 { 580 struct mta_task *task; 581 582 if ((task = TAILQ_FIRST(&relay->tasks))) { 583 TAILQ_REMOVE(&relay->tasks, task, entry); 584 relay->ntask -= 1; 585 task->relay = NULL; 586 587 /* When the number of tasks is down to lowat, query some evp */ 588 if (relay->ntask == (size_t)relay->limits->task_lowat) { 589 if (relay->state & RELAY_ONHOLD) { 590 log_info("smtp-out: back to lowat on %s: releasing", 591 mta_relay_to_text(relay)); 592 relay->state &= ~RELAY_ONHOLD; 593 } 594 if (relay->state & RELAY_HOLDQ) { 595 m_create(p_queue, IMSG_MTA_HOLDQ_RELEASE, 0, 0, -1); 596 m_add_id(p_queue, relay->id); 597 m_add_int(p_queue, relay->limits->task_release); 598 m_close(p_queue); 599 } 600 } 601 else if (relay->ntask == 0 && relay->state & RELAY_HOLDQ) { 602 m_create(p_queue, IMSG_MTA_HOLDQ_RELEASE, 0, 0, -1); 603 m_add_id(p_queue, relay->id); 604 m_add_int(p_queue, 0); 605 m_close(p_queue); 606 } 607 } 608 609 return (task); 610 } 611 612 static void 613 mta_handle_envelope(struct envelope *evp, const char *smarthost) 614 { 615 struct mta_relay *relay; 616 struct mta_task *task; 617 struct mta_envelope *e; 618 struct dispatcher *dispatcher; 619 struct mailaddr maddr; 620 struct relayhost relayh; 621 char buf[LINE_MAX]; 622 623 dispatcher = dict_xget(env->sc_dispatchers, evp->dispatcher); 624 if (dispatcher->u.remote.smarthost && smarthost == NULL) { 625 mta_query_smarthost(evp); 626 return; 627 } 628 629 memset(&relayh, 0, sizeof(relayh)); 630 relayh.tls = RELAY_TLS_OPPORTUNISTIC; 631 if (smarthost && !text_to_relayhost(&relayh, smarthost)) { 632 log_warnx("warn: Failed to parse smarthost %s", smarthost); 633 m_create(p_queue, IMSG_MTA_DELIVERY_TEMPFAIL, 0, 0, -1); 634 m_add_evpid(p_queue, evp->id); 635 m_add_string(p_queue, "Cannot parse smarthost"); 636 m_add_int(p_queue, ESC_OTHER_STATUS); 637 m_close(p_queue); 638 return; 639 } 640 641 if (relayh.flags & RELAY_AUTH && dispatcher->u.remote.auth == NULL) { 642 log_warnx("warn: No auth table on action \"%s\" for relay %s", 643 evp->dispatcher, smarthost); 644 m_create(p_queue, IMSG_MTA_DELIVERY_TEMPFAIL, 0, 0, -1); 645 m_add_evpid(p_queue, evp->id); 646 m_add_string(p_queue, "No auth table for relaying"); 647 m_add_int(p_queue, ESC_OTHER_STATUS); 648 m_close(p_queue); 649 return; 650 } 651 652 if (dispatcher->u.remote.tls_required) { 653 /* Reject relay if smtp+notls:// is requested */ 654 if (relayh.tls == RELAY_TLS_NO) { 655 log_warnx("warn: TLS required for action \"%s\"", 656 evp->dispatcher); 657 m_create(p_queue, IMSG_MTA_DELIVERY_TEMPFAIL, 0, 0, -1); 658 m_add_evpid(p_queue, evp->id); 659 m_add_string(p_queue, "TLS required for relaying"); 660 m_add_int(p_queue, ESC_OTHER_STATUS); 661 m_close(p_queue); 662 return; 663 } 664 /* Update smtp:// to smtp+tls:// */ 665 if (relayh.tls == RELAY_TLS_OPPORTUNISTIC) 666 relayh.tls = RELAY_TLS_STARTTLS; 667 } 668 669 relay = mta_relay(evp, &relayh); 670 /* ignore if we don't know the limits yet */ 671 if (relay->limits && 672 relay->ntask >= (size_t)relay->limits->task_hiwat) { 673 if (!(relay->state & RELAY_ONHOLD)) { 674 log_info("smtp-out: hiwat reached on %s: holding envelopes", 675 mta_relay_to_text(relay)); 676 relay->state |= RELAY_ONHOLD; 677 } 678 } 679 680 /* 681 * If the relay has too many pending tasks, tell the 682 * scheduler to hold it until further notice 683 */ 684 if (relay->state & RELAY_ONHOLD) { 685 relay->state |= RELAY_HOLDQ; 686 m_create(p_queue, IMSG_MTA_DELIVERY_HOLD, 0, 0, -1); 687 m_add_evpid(p_queue, evp->id); 688 m_add_id(p_queue, relay->id); 689 m_close(p_queue); 690 mta_relay_unref(relay); /* from here */ 691 return; 692 } 693 694 task = NULL; 695 TAILQ_FOREACH(task, &relay->tasks, entry) 696 if (task->msgid == evpid_to_msgid(evp->id)) 697 break; 698 699 if (task == NULL) { 700 task = xmalloc(sizeof *task); 701 TAILQ_INIT(&task->envelopes); 702 task->relay = relay; 703 relay->ntask += 1; 704 TAILQ_INSERT_TAIL(&relay->tasks, task, entry); 705 task->msgid = evpid_to_msgid(evp->id); 706 if (evp->sender.user[0] || evp->sender.domain[0]) 707 (void)snprintf(buf, sizeof buf, "%s@%s", 708 evp->sender.user, evp->sender.domain); 709 else 710 buf[0] = '\0'; 711 712 if (dispatcher->u.remote.mail_from && evp->sender.user[0]) { 713 memset(&maddr, 0, sizeof (maddr)); 714 if (text_to_mailaddr(&maddr, 715 dispatcher->u.remote.mail_from)) { 716 (void)snprintf(buf, sizeof buf, "%s@%s", 717 maddr.user[0] ? maddr.user : evp->sender.user, 718 maddr.domain[0] ? maddr.domain : evp->sender.domain); 719 } 720 } 721 722 task->sender = xstrdup(buf); 723 stat_increment("mta.task", 1); 724 } 725 726 e = xcalloc(1, sizeof *e); 727 e->id = evp->id; 728 e->creation = evp->creation; 729 e->smtpname = xstrdup(evp->smtpname); 730 (void)snprintf(buf, sizeof buf, "%s@%s", 731 evp->dest.user, evp->dest.domain); 732 e->dest = xstrdup(buf); 733 (void)snprintf(buf, sizeof buf, "%s@%s", 734 evp->rcpt.user, evp->rcpt.domain); 735 if (strcmp(buf, e->dest)) 736 e->rcpt = xstrdup(buf); 737 e->task = task; 738 if (evp->dsn_orcpt.user[0] && evp->dsn_orcpt.domain[0]) { 739 (void)snprintf(buf, sizeof buf, "%s@%s", 740 evp->dsn_orcpt.user, evp->dsn_orcpt.domain); 741 e->dsn_orcpt = xstrdup(buf); 742 } 743 (void)strlcpy(e->dsn_envid, evp->dsn_envid, 744 sizeof e->dsn_envid); 745 e->dsn_notify = evp->dsn_notify; 746 e->dsn_ret = evp->dsn_ret; 747 748 TAILQ_INSERT_TAIL(&task->envelopes, e, entry); 749 log_debug("debug: mta: received evp:%016" PRIx64 750 " for <%s>", e->id, e->dest); 751 752 stat_increment("mta.envelope", 1); 753 754 mta_drain(relay); 755 mta_relay_unref(relay); /* from here */ 756 } 757 758 static void 759 mta_delivery_flush_event(int fd, short event, void *arg) 760 { 761 struct mta_envelope *e; 762 struct timeval tv; 763 764 if (tree_poproot(&flush_evp, NULL, (void**)(&e))) { 765 766 if (e->delivery == IMSG_MTA_DELIVERY_OK) { 767 m_create(p_queue, IMSG_MTA_DELIVERY_OK, 0, 0, -1); 768 m_add_evpid(p_queue, e->id); 769 m_add_int(p_queue, e->ext); 770 m_close(p_queue); 771 } else if (e->delivery == IMSG_MTA_DELIVERY_TEMPFAIL) { 772 m_create(p_queue, IMSG_MTA_DELIVERY_TEMPFAIL, 0, 0, -1); 773 m_add_evpid(p_queue, e->id); 774 m_add_string(p_queue, e->status); 775 m_add_int(p_queue, ESC_OTHER_STATUS); 776 m_close(p_queue); 777 } 778 else if (e->delivery == IMSG_MTA_DELIVERY_PERMFAIL) { 779 m_create(p_queue, IMSG_MTA_DELIVERY_PERMFAIL, 0, 0, -1); 780 m_add_evpid(p_queue, e->id); 781 m_add_string(p_queue, e->status); 782 m_add_int(p_queue, ESC_OTHER_STATUS); 783 m_close(p_queue); 784 } 785 else if (e->delivery == IMSG_MTA_DELIVERY_LOOP) { 786 m_create(p_queue, IMSG_MTA_DELIVERY_LOOP, 0, 0, -1); 787 m_add_evpid(p_queue, e->id); 788 m_close(p_queue); 789 } 790 else { 791 log_warnx("warn: bad delivery type %d for %016" PRIx64, 792 e->delivery, e->id); 793 fatalx("aborting"); 794 } 795 796 log_debug("debug: mta: flush for %016"PRIx64" (-> %s)", e->id, e->dest); 797 798 free(e->smtpname); 799 free(e->dest); 800 free(e->rcpt); 801 free(e->dsn_orcpt); 802 free(e); 803 804 tv.tv_sec = 0; 805 tv.tv_usec = 0; 806 evtimer_add(&ev_flush_evp, &tv); 807 } 808 } 809 810 void 811 mta_delivery_log(struct mta_envelope *e, const char *source, const char *relay, 812 int delivery, const char *status) 813 { 814 if (delivery == IMSG_MTA_DELIVERY_OK) 815 mta_log(e, "Ok", source, relay, status); 816 else if (delivery == IMSG_MTA_DELIVERY_TEMPFAIL) 817 mta_log(e, "TempFail", source, relay, status); 818 else if (delivery == IMSG_MTA_DELIVERY_PERMFAIL) 819 mta_log(e, "PermFail", source, relay, status); 820 else if (delivery == IMSG_MTA_DELIVERY_LOOP) 821 mta_log(e, "PermFail", source, relay, "Loop detected"); 822 else { 823 log_warnx("warn: bad delivery type %d for %016" PRIx64, 824 delivery, e->id); 825 fatalx("aborting"); 826 } 827 828 e->delivery = delivery; 829 if (status) 830 (void)strlcpy(e->status, status, sizeof(e->status)); 831 } 832 833 void 834 mta_delivery_notify(struct mta_envelope *e) 835 { 836 struct timeval tv; 837 838 tree_xset(&flush_evp, e->id, e); 839 if (tree_count(&flush_evp) == 1) { 840 tv.tv_sec = 0; 841 tv.tv_usec = 0; 842 evtimer_add(&ev_flush_evp, &tv); 843 } 844 } 845 846 static void 847 mta_query_mx(struct mta_relay *relay) 848 { 849 uint64_t id; 850 851 if (relay->status & RELAY_WAIT_MX) 852 return; 853 854 log_debug("debug: mta: querying MX for %s...", 855 mta_relay_to_text(relay)); 856 857 if (waitq_wait(&relay->domain->mxs, mta_on_mx, relay)) { 858 id = generate_uid(); 859 tree_xset(&wait_mx, id, relay->domain); 860 if (relay->domain->as_host) 861 m_create(p_lka, IMSG_MTA_DNS_HOST, 0, 0, -1); 862 else 863 m_create(p_lka, IMSG_MTA_DNS_MX, 0, 0, -1); 864 m_add_id(p_lka, id); 865 m_add_string(p_lka, relay->domain->name); 866 m_close(p_lka); 867 } 868 relay->status |= RELAY_WAIT_MX; 869 mta_relay_ref(relay); 870 } 871 872 static void 873 mta_query_limits(struct mta_relay *relay) 874 { 875 if (relay->status & RELAY_WAIT_LIMITS) 876 return; 877 878 relay->limits = dict_get(env->sc_limits_dict, relay->domain->name); 879 if (relay->limits == NULL) 880 relay->limits = dict_get(env->sc_limits_dict, "default"); 881 882 if (max_seen_conndelay_route < relay->limits->conndelay_route) 883 max_seen_conndelay_route = relay->limits->conndelay_route; 884 if (max_seen_discdelay_route < relay->limits->discdelay_route) 885 max_seen_discdelay_route = relay->limits->discdelay_route; 886 } 887 888 static void 889 mta_query_secret(struct mta_relay *relay) 890 { 891 if (relay->status & RELAY_WAIT_SECRET) 892 return; 893 894 log_debug("debug: mta: querying secret for %s...", 895 mta_relay_to_text(relay)); 896 897 tree_xset(&wait_secret, relay->id, relay); 898 relay->status |= RELAY_WAIT_SECRET; 899 900 m_create(p_lka, IMSG_MTA_LOOKUP_CREDENTIALS, 0, 0, -1); 901 m_add_id(p_lka, relay->id); 902 m_add_string(p_lka, relay->authtable); 903 m_add_string(p_lka, relay->authlabel); 904 m_close(p_lka); 905 906 mta_relay_ref(relay); 907 } 908 909 static void 910 mta_query_smarthost(struct envelope *evp0) 911 { 912 struct dispatcher *dispatcher; 913 struct envelope *evp; 914 915 evp = malloc(sizeof(*evp)); 916 memmove(evp, evp0, sizeof(*evp)); 917 918 dispatcher = dict_xget(env->sc_dispatchers, evp->dispatcher); 919 920 log_debug("debug: mta: querying smarthost for %s:%s...", 921 evp->dispatcher, dispatcher->u.remote.smarthost); 922 923 tree_xset(&wait_smarthost, evp->id, evp); 924 925 m_create(p_lka, IMSG_MTA_LOOKUP_SMARTHOST, 0, 0, -1); 926 m_add_id(p_lka, evp->id); 927 m_add_string(p_lka, dispatcher->u.remote.smarthost); 928 m_close(p_lka); 929 930 log_debug("debug: mta: querying smarthost"); 931 } 932 933 static void 934 mta_query_preference(struct mta_relay *relay) 935 { 936 if (relay->status & RELAY_WAIT_PREFERENCE) 937 return; 938 939 log_debug("debug: mta: querying preference for %s...", 940 mta_relay_to_text(relay)); 941 942 tree_xset(&wait_preference, relay->id, relay); 943 relay->status |= RELAY_WAIT_PREFERENCE; 944 945 m_create(p_lka, IMSG_MTA_DNS_MX_PREFERENCE, 0, 0, -1); 946 m_add_id(p_lka, relay->id); 947 m_add_string(p_lka, relay->domain->name); 948 m_add_string(p_lka, relay->backupname); 949 m_close(p_lka); 950 951 mta_relay_ref(relay); 952 } 953 954 static void 955 mta_query_source(struct mta_relay *relay) 956 { 957 log_debug("debug: mta: querying source for %s...", 958 mta_relay_to_text(relay)); 959 960 relay->sourceloop += 1; 961 962 if (relay->sourcetable == NULL) { 963 /* 964 * This is a recursive call, but it only happens once, since 965 * another source will not be queried immediately. 966 */ 967 mta_relay_ref(relay); 968 mta_on_source(relay, mta_source(NULL)); 969 return; 970 } 971 972 m_create(p_lka, IMSG_MTA_LOOKUP_SOURCE, 0, 0, -1); 973 m_add_id(p_lka, relay->id); 974 m_add_string(p_lka, relay->sourcetable); 975 m_close(p_lka); 976 977 tree_xset(&wait_source, relay->id, relay); 978 relay->status |= RELAY_WAIT_SOURCE; 979 mta_relay_ref(relay); 980 } 981 982 static void 983 mta_on_mx(void *tag, void *arg, void *data) 984 { 985 struct mta_domain *domain = data; 986 struct mta_relay *relay = arg; 987 988 log_debug("debug: mta: ... got mx (%p, %s, %s)", 989 tag, domain->name, mta_relay_to_text(relay)); 990 991 switch (domain->mxstatus) { 992 case DNS_OK: 993 break; 994 case DNS_RETRY: 995 relay->fail = IMSG_MTA_DELIVERY_TEMPFAIL; 996 relay->failstr = "Temporary failure in MX lookup"; 997 break; 998 case DNS_EINVAL: 999 relay->fail = IMSG_MTA_DELIVERY_PERMFAIL; 1000 relay->failstr = "Invalid domain name"; 1001 break; 1002 case DNS_ENONAME: 1003 relay->fail = IMSG_MTA_DELIVERY_PERMFAIL; 1004 relay->failstr = "Domain does not exist"; 1005 break; 1006 case DNS_ENOTFOUND: 1007 relay->fail = IMSG_MTA_DELIVERY_TEMPFAIL; 1008 relay->failstr = "No MX found for domain"; 1009 break; 1010 default: 1011 fatalx("bad DNS lookup error code"); 1012 break; 1013 } 1014 1015 if (domain->mxstatus) 1016 log_info("smtp-out: Failed to resolve MX for %s: %s", 1017 mta_relay_to_text(relay), relay->failstr); 1018 1019 relay->status &= ~RELAY_WAIT_MX; 1020 mta_drain(relay); 1021 mta_relay_unref(relay); /* from mta_drain() */ 1022 } 1023 1024 static void 1025 mta_on_secret(struct mta_relay *relay, const char *secret) 1026 { 1027 log_debug("debug: mta: ... got secret for %s: %s", 1028 mta_relay_to_text(relay), secret); 1029 1030 if (secret) 1031 relay->secret = strdup(secret); 1032 1033 if (relay->secret == NULL) { 1034 log_warnx("warn: Failed to retrieve secret " 1035 "for %s", mta_relay_to_text(relay)); 1036 relay->fail = IMSG_MTA_DELIVERY_TEMPFAIL; 1037 relay->failstr = "Could not retrieve credentials"; 1038 } 1039 1040 relay->status &= ~RELAY_WAIT_SECRET; 1041 mta_drain(relay); 1042 mta_relay_unref(relay); /* from mta_query_secret() */ 1043 } 1044 1045 static void 1046 mta_on_smarthost(struct envelope *evp, const char *smarthost) 1047 { 1048 if (smarthost == NULL) { 1049 log_warnx("warn: Failed to retrieve smarthost " 1050 "for envelope %"PRIx64, evp->id); 1051 m_create(p_queue, IMSG_MTA_DELIVERY_TEMPFAIL, 0, 0, -1); 1052 m_add_evpid(p_queue, evp->id); 1053 m_add_string(p_queue, "Cannot retrieve smarthost"); 1054 m_add_int(p_queue, ESC_OTHER_STATUS); 1055 m_close(p_queue); 1056 return; 1057 } 1058 1059 log_debug("debug: mta: ... got smarthost for %016"PRIx64": %s", 1060 evp->id, smarthost); 1061 mta_handle_envelope(evp, smarthost); 1062 free(evp); 1063 } 1064 1065 static void 1066 mta_on_preference(struct mta_relay *relay, int preference) 1067 { 1068 log_debug("debug: mta: ... got preference for %s: %d", 1069 mta_relay_to_text(relay), preference); 1070 1071 relay->backuppref = preference; 1072 1073 relay->status &= ~RELAY_WAIT_PREFERENCE; 1074 mta_drain(relay); 1075 mta_relay_unref(relay); /* from mta_query_preference() */ 1076 } 1077 1078 static void 1079 mta_on_source(struct mta_relay *relay, struct mta_source *source) 1080 { 1081 struct mta_connector *c; 1082 void *iter; 1083 int delay, errmask; 1084 1085 log_debug("debug: mta: ... got source for %s: %s", 1086 mta_relay_to_text(relay), source ? mta_source_to_text(source) : "NULL"); 1087 1088 relay->lastsource = time(NULL); 1089 delay = DELAY_CHECK_SOURCE_SLOW; 1090 1091 if (source) { 1092 c = mta_connector(relay, source); 1093 if (c->flags & CONNECTOR_NEW) { 1094 c->flags &= ~CONNECTOR_NEW; 1095 delay = DELAY_CHECK_SOURCE; 1096 } 1097 mta_connect(c); 1098 if ((c->flags & CONNECTOR_ERROR) == 0) 1099 relay->sourceloop = 0; 1100 else 1101 delay = DELAY_CHECK_SOURCE_FAST; 1102 mta_source_unref(source); /* from constructor */ 1103 } 1104 else { 1105 log_warnx("warn: Failed to get source address for %s", 1106 mta_relay_to_text(relay)); 1107 } 1108 1109 if (tree_count(&relay->connectors) == 0) { 1110 relay->fail = IMSG_MTA_DELIVERY_TEMPFAIL; 1111 relay->failstr = "Could not retrieve source address"; 1112 } 1113 if (tree_count(&relay->connectors) < relay->sourceloop) { 1114 relay->fail = IMSG_MTA_DELIVERY_TEMPFAIL; 1115 relay->failstr = "No valid route to remote MX"; 1116 1117 errmask = 0; 1118 iter = NULL; 1119 while (tree_iter(&relay->connectors, &iter, NULL, (void **)&c)) 1120 errmask |= c->flags; 1121 1122 if (errmask & CONNECTOR_ERROR_ROUTE_SMTP) 1123 relay->failstr = "Destination seem to reject all mails"; 1124 else if (errmask & CONNECTOR_ERROR_ROUTE_NET) 1125 relay->failstr = "Network error on destination MXs"; 1126 else if (errmask & CONNECTOR_ERROR_MX) 1127 relay->failstr = "No MX found for destination"; 1128 else if (errmask & CONNECTOR_ERROR_FAMILY) 1129 relay->failstr = "Address family mismatch on destination MXs"; 1130 else if (errmask & CONNECTOR_ERROR_BLOCKED) 1131 relay->failstr = "All routes to destination blocked"; 1132 else 1133 relay->failstr = "No valid route to destination"; 1134 } 1135 1136 relay->nextsource = relay->lastsource + delay; 1137 relay->status &= ~RELAY_WAIT_SOURCE; 1138 mta_drain(relay); 1139 mta_relay_unref(relay); /* from mta_query_source() */ 1140 } 1141 1142 static void 1143 mta_connect(struct mta_connector *c) 1144 { 1145 struct mta_route *route; 1146 struct mta_limits *l = c->relay->limits; 1147 int limits; 1148 time_t nextconn, now; 1149 1150 /* toggle the block flag */ 1151 if (mta_is_blocked(c->source, c->relay->domain->name)) 1152 c->flags |= CONNECTOR_ERROR_BLOCKED; 1153 else 1154 c->flags &= ~CONNECTOR_ERROR_BLOCKED; 1155 1156 again: 1157 1158 log_debug("debug: mta: connecting with %s", mta_connector_to_text(c)); 1159 1160 /* Do not connect if this connector has an error. */ 1161 if (c->flags & CONNECTOR_ERROR) { 1162 log_debug("debug: mta: connector error"); 1163 return; 1164 } 1165 1166 if (c->flags & CONNECTOR_WAIT) { 1167 log_debug("debug: mta: cancelling connector timeout"); 1168 runq_cancel(runq_connector, c); 1169 c->flags &= ~CONNECTOR_WAIT; 1170 } 1171 1172 /* No job. */ 1173 if (c->relay->ntask == 0) { 1174 log_debug("debug: mta: no task for connector"); 1175 return; 1176 } 1177 1178 /* Do not create more connections than necessary */ 1179 if ((c->relay->nconn_ready >= c->relay->ntask) || 1180 (c->relay->nconn > 2 && c->relay->nconn >= c->relay->ntask / 2)) { 1181 log_debug("debug: mta: enough connections already"); 1182 return; 1183 } 1184 1185 limits = 0; 1186 nextconn = now = time(NULL); 1187 1188 if (c->relay->domain->lastconn + l->conndelay_domain > nextconn) { 1189 log_debug("debug: mta: cannot use domain %s before %llus", 1190 c->relay->domain->name, 1191 (unsigned long long) c->relay->domain->lastconn + l->conndelay_domain - now); 1192 nextconn = c->relay->domain->lastconn + l->conndelay_domain; 1193 } 1194 if (c->relay->domain->nconn >= l->maxconn_per_domain) { 1195 log_debug("debug: mta: hit domain limit"); 1196 limits |= CONNECTOR_LIMIT_DOMAIN; 1197 } 1198 1199 if (c->source->lastconn + l->conndelay_source > nextconn) { 1200 log_debug("debug: mta: cannot use source %s before %llus", 1201 mta_source_to_text(c->source), 1202 (unsigned long long) c->source->lastconn + l->conndelay_source - now); 1203 nextconn = c->source->lastconn + l->conndelay_source; 1204 } 1205 if (c->source->nconn >= l->maxconn_per_source) { 1206 log_debug("debug: mta: hit source limit"); 1207 limits |= CONNECTOR_LIMIT_SOURCE; 1208 } 1209 1210 if (c->lastconn + l->conndelay_connector > nextconn) { 1211 log_debug("debug: mta: cannot use %s before %llus", 1212 mta_connector_to_text(c), 1213 (unsigned long long) c->lastconn + l->conndelay_connector - now); 1214 nextconn = c->lastconn + l->conndelay_connector; 1215 } 1216 if (c->nconn >= l->maxconn_per_connector) { 1217 log_debug("debug: mta: hit connector limit"); 1218 limits |= CONNECTOR_LIMIT_CONN; 1219 } 1220 1221 if (c->relay->lastconn + l->conndelay_relay > nextconn) { 1222 log_debug("debug: mta: cannot use %s before %llus", 1223 mta_relay_to_text(c->relay), 1224 (unsigned long long) c->relay->lastconn + l->conndelay_relay - now); 1225 nextconn = c->relay->lastconn + l->conndelay_relay; 1226 } 1227 if (c->relay->nconn >= l->maxconn_per_relay) { 1228 log_debug("debug: mta: hit relay limit"); 1229 limits |= CONNECTOR_LIMIT_RELAY; 1230 } 1231 1232 /* We can connect now, find a route */ 1233 if (!limits && nextconn <= now) 1234 route = mta_find_route(c, now, &limits, &nextconn); 1235 else 1236 route = NULL; 1237 1238 /* No route */ 1239 if (route == NULL) { 1240 1241 if (c->flags & CONNECTOR_ERROR) { 1242 /* XXX we might want to clear this flag later */ 1243 log_debug("debug: mta-routing: no route available for %s: errors on connector", 1244 mta_connector_to_text(c)); 1245 return; 1246 } 1247 else if (limits) { 1248 log_debug("debug: mta-routing: no route available for %s: limits reached", 1249 mta_connector_to_text(c)); 1250 nextconn = now + DELAY_CHECK_LIMIT; 1251 } 1252 else { 1253 log_debug("debug: mta-routing: no route available for %s: must wait a bit", 1254 mta_connector_to_text(c)); 1255 } 1256 log_debug("debug: mta: retrying to connect on %s in %llus...", 1257 mta_connector_to_text(c), 1258 (unsigned long long) nextconn - time(NULL)); 1259 c->flags |= CONNECTOR_WAIT; 1260 runq_schedule_at(runq_connector, nextconn, c); 1261 return; 1262 } 1263 1264 log_debug("debug: mta-routing: spawning new connection on %s", 1265 mta_route_to_text(route)); 1266 1267 c->nconn += 1; 1268 c->lastconn = time(NULL); 1269 1270 c->relay->nconn += 1; 1271 c->relay->lastconn = c->lastconn; 1272 c->relay->domain->nconn += 1; 1273 c->relay->domain->lastconn = c->lastconn; 1274 route->nconn += 1; 1275 route->lastconn = c->lastconn; 1276 route->src->nconn += 1; 1277 route->src->lastconn = c->lastconn; 1278 route->dst->nconn += 1; 1279 route->dst->lastconn = c->lastconn; 1280 1281 mta_session(c->relay, route); /* this never fails synchronously */ 1282 mta_relay_ref(c->relay); 1283 1284 goto again; 1285 } 1286 1287 static void 1288 mta_on_timeout(struct runq *runq, void *arg) 1289 { 1290 struct mta_connector *connector = arg; 1291 struct mta_relay *relay = arg; 1292 struct mta_route *route = arg; 1293 struct hoststat *hs = arg; 1294 1295 if (runq == runq_relay) { 1296 log_debug("debug: mta: ... timeout for %s", 1297 mta_relay_to_text(relay)); 1298 relay->status &= ~RELAY_WAIT_CONNECTOR; 1299 mta_drain(relay); 1300 mta_relay_unref(relay); /* from mta_drain() */ 1301 } 1302 else if (runq == runq_connector) { 1303 log_debug("debug: mta: ... timeout for %s", 1304 mta_connector_to_text(connector)); 1305 connector->flags &= ~CONNECTOR_WAIT; 1306 mta_connect(connector); 1307 } 1308 else if (runq == runq_route) { 1309 route->flags &= ~ROUTE_RUNQ; 1310 mta_route_enable(route); 1311 mta_route_unref(route); 1312 } 1313 else if (runq == runq_hoststat) { 1314 log_debug("debug: mta: ... timeout for hoststat %s", 1315 hs->name); 1316 mta_hoststat_remove_entry(hs); 1317 free(hs); 1318 } 1319 } 1320 1321 static void 1322 mta_route_disable(struct mta_route *route, int penalty, int reason) 1323 { 1324 unsigned long long delay; 1325 1326 route->penalty += penalty; 1327 route->lastpenalty = time(NULL); 1328 delay = (unsigned long long)DELAY_ROUTE_BASE * route->penalty * route->penalty; 1329 if (delay > DELAY_ROUTE_MAX) 1330 delay = DELAY_ROUTE_MAX; 1331 #if 0 1332 delay = 60; 1333 #endif 1334 1335 log_info("smtp-out: Disabling route %s for %llus", 1336 mta_route_to_text(route), delay); 1337 1338 if (route->flags & ROUTE_DISABLED) 1339 runq_cancel(runq_route, route); 1340 else 1341 mta_route_ref(route); 1342 1343 route->flags |= reason & ROUTE_DISABLED; 1344 runq_schedule(runq_route, delay, route); 1345 } 1346 1347 static void 1348 mta_route_enable(struct mta_route *route) 1349 { 1350 if (route->flags & ROUTE_DISABLED) { 1351 log_info("smtp-out: Enabling route %s", 1352 mta_route_to_text(route)); 1353 route->flags &= ~ROUTE_DISABLED; 1354 route->flags |= ROUTE_NEW; 1355 route->nerror = 0; 1356 } 1357 1358 if (route->penalty) { 1359 #if DELAY_QUADRATIC 1360 route->penalty -= 1; 1361 route->lastpenalty = time(NULL); 1362 #else 1363 route->penalty = 0; 1364 #endif 1365 } 1366 } 1367 1368 static void 1369 mta_drain(struct mta_relay *r) 1370 { 1371 char buf[64]; 1372 1373 log_debug("debug: mta: draining %s " 1374 "refcount=%d, ntask=%zu, nconnector=%zu, nconn=%zu", 1375 mta_relay_to_text(r), 1376 r->refcount, r->ntask, tree_count(&r->connectors), r->nconn); 1377 1378 /* 1379 * All done. 1380 */ 1381 if (r->ntask == 0) { 1382 log_debug("debug: mta: all done for %s", mta_relay_to_text(r)); 1383 return; 1384 } 1385 1386 /* 1387 * If we know that this relay is failing flush the tasks. 1388 */ 1389 if (r->fail) { 1390 mta_flush(r, r->fail, r->failstr); 1391 return; 1392 } 1393 1394 /* Query secret if needed. */ 1395 if (r->flags & RELAY_AUTH && r->secret == NULL) 1396 mta_query_secret(r); 1397 1398 /* Query our preference if needed. */ 1399 if (r->backupname && r->backuppref == -1) 1400 mta_query_preference(r); 1401 1402 /* Query the domain MXs if needed. */ 1403 if (r->domain->lastmxquery == 0) 1404 mta_query_mx(r); 1405 1406 /* Query the limits if needed. */ 1407 if (r->limits == NULL) 1408 mta_query_limits(r); 1409 1410 /* Wait until we are ready to proceed. */ 1411 if (r->status & RELAY_WAITMASK) { 1412 buf[0] = '\0'; 1413 if (r->status & RELAY_WAIT_MX) 1414 (void)strlcat(buf, " MX", sizeof buf); 1415 if (r->status & RELAY_WAIT_PREFERENCE) 1416 (void)strlcat(buf, " preference", sizeof buf); 1417 if (r->status & RELAY_WAIT_SECRET) 1418 (void)strlcat(buf, " secret", sizeof buf); 1419 if (r->status & RELAY_WAIT_SOURCE) 1420 (void)strlcat(buf, " source", sizeof buf); 1421 if (r->status & RELAY_WAIT_CONNECTOR) 1422 (void)strlcat(buf, " connector", sizeof buf); 1423 log_debug("debug: mta: %s waiting for%s", 1424 mta_relay_to_text(r), buf); 1425 return; 1426 } 1427 1428 /* 1429 * We have pending task, and it's maybe time too try a new source. 1430 */ 1431 if (r->nextsource <= time(NULL)) 1432 mta_query_source(r); 1433 else { 1434 log_debug("debug: mta: scheduling relay %s in %llus...", 1435 mta_relay_to_text(r), 1436 (unsigned long long) r->nextsource - time(NULL)); 1437 runq_schedule_at(runq_relay, r->nextsource, r); 1438 r->status |= RELAY_WAIT_CONNECTOR; 1439 mta_relay_ref(r); 1440 } 1441 } 1442 1443 static void 1444 mta_flush(struct mta_relay *relay, int fail, const char *error) 1445 { 1446 struct mta_envelope *e; 1447 struct mta_task *task; 1448 const char *domain; 1449 void *iter; 1450 struct mta_connector *c; 1451 size_t n, r; 1452 1453 log_debug("debug: mta_flush(%s, %d, \"%s\")", 1454 mta_relay_to_text(relay), fail, error); 1455 1456 if (fail != IMSG_MTA_DELIVERY_TEMPFAIL && fail != IMSG_MTA_DELIVERY_PERMFAIL) 1457 errx(1, "unexpected delivery status %d", fail); 1458 1459 n = 0; 1460 while ((task = TAILQ_FIRST(&relay->tasks))) { 1461 TAILQ_REMOVE(&relay->tasks, task, entry); 1462 while ((e = TAILQ_FIRST(&task->envelopes))) { 1463 TAILQ_REMOVE(&task->envelopes, e, entry); 1464 1465 /* 1466 * host was suspended, cache envelope id in hoststat tree 1467 * so that it can be retried when a delivery succeeds for 1468 * that domain. 1469 */ 1470 domain = strchr(e->dest, '@'); 1471 if (fail == IMSG_MTA_DELIVERY_TEMPFAIL && domain) { 1472 r = 0; 1473 iter = NULL; 1474 while (tree_iter(&relay->connectors, &iter, 1475 NULL, (void **)&c)) { 1476 if (c->flags & CONNECTOR_ERROR_ROUTE) 1477 r++; 1478 } 1479 if (tree_count(&relay->connectors) == r) 1480 mta_hoststat_cache(domain+1, e->id); 1481 } 1482 1483 mta_delivery_log(e, NULL, relay->domain->name, fail, error); 1484 mta_delivery_notify(e); 1485 1486 n++; 1487 } 1488 free(task->sender); 1489 free(task); 1490 } 1491 1492 stat_decrement("mta.task", relay->ntask); 1493 stat_decrement("mta.envelope", n); 1494 relay->ntask = 0; 1495 1496 /* release all waiting envelopes for the relay */ 1497 if (relay->state & RELAY_HOLDQ) { 1498 m_create(p_queue, IMSG_MTA_HOLDQ_RELEASE, 0, 0, -1); 1499 m_add_id(p_queue, relay->id); 1500 m_add_int(p_queue, -1); 1501 m_close(p_queue); 1502 } 1503 } 1504 1505 /* 1506 * Find a route to use for this connector 1507 */ 1508 static struct mta_route * 1509 mta_find_route(struct mta_connector *c, time_t now, int *limits, 1510 time_t *nextconn) 1511 { 1512 struct mta_route *route, *best; 1513 struct mta_limits *l = c->relay->limits; 1514 struct mta_mx *mx; 1515 int level, limit_host, limit_route; 1516 int family_mismatch, seen, suspended_route; 1517 time_t tm; 1518 1519 log_debug("debug: mta-routing: searching new route for %s...", 1520 mta_connector_to_text(c)); 1521 1522 tm = 0; 1523 limit_host = 0; 1524 limit_route = 0; 1525 suspended_route = 0; 1526 family_mismatch = 0; 1527 level = -1; 1528 best = NULL; 1529 seen = 0; 1530 1531 TAILQ_FOREACH(mx, &c->relay->domain->mxs, entry) { 1532 /* 1533 * New preference level 1534 */ 1535 if (mx->preference > level) { 1536 #ifndef IGNORE_MX_PREFERENCE 1537 /* 1538 * Use the current best MX if found. 1539 */ 1540 if (best) 1541 break; 1542 1543 /* 1544 * No candidate found. There are valid MXs at this 1545 * preference level but they reached their limit, or 1546 * we can't connect yet. 1547 */ 1548 if (limit_host || limit_route || tm) 1549 break; 1550 1551 /* 1552 * If we are a backup MX, do not relay to MXs with 1553 * a greater preference value. 1554 */ 1555 if (c->relay->backuppref >= 0 && 1556 mx->preference >= c->relay->backuppref) 1557 break; 1558 1559 /* 1560 * Start looking at MXs on this preference level. 1561 */ 1562 #endif 1563 level = mx->preference; 1564 } 1565 1566 if (mx->host->flags & HOST_IGNORE) 1567 continue; 1568 1569 /* Found a possibly valid mx */ 1570 seen++; 1571 1572 if ((c->source->sa && 1573 c->source->sa->sa_family != mx->host->sa->sa_family) || 1574 (l->family && l->family != mx->host->sa->sa_family)) { 1575 log_debug("debug: mta-routing: skipping host %s: AF mismatch", 1576 mta_host_to_text(mx->host)); 1577 family_mismatch = 1; 1578 continue; 1579 } 1580 1581 if (mx->host->nconn >= l->maxconn_per_host) { 1582 log_debug("debug: mta-routing: skipping host %s: too many connections", 1583 mta_host_to_text(mx->host)); 1584 limit_host = 1; 1585 continue; 1586 } 1587 1588 if (mx->host->lastconn + l->conndelay_host > now) { 1589 log_debug("debug: mta-routing: skipping host %s: cannot use before %llus", 1590 mta_host_to_text(mx->host), 1591 (unsigned long long) mx->host->lastconn + l->conndelay_host - now); 1592 if (tm == 0 || mx->host->lastconn + l->conndelay_host < tm) 1593 tm = mx->host->lastconn + l->conndelay_host; 1594 continue; 1595 } 1596 1597 route = mta_route(c->source, mx->host); 1598 1599 if (route->flags & ROUTE_DISABLED) { 1600 log_debug("debug: mta-routing: skipping route %s: suspend", 1601 mta_route_to_text(route)); 1602 suspended_route |= route->flags & ROUTE_DISABLED; 1603 mta_route_unref(route); /* from here */ 1604 continue; 1605 } 1606 1607 if (route->nconn && (route->flags & ROUTE_NEW)) { 1608 log_debug("debug: mta-routing: skipping route %s: not validated yet", 1609 mta_route_to_text(route)); 1610 limit_route = 1; 1611 mta_route_unref(route); /* from here */ 1612 continue; 1613 } 1614 1615 if (route->nconn >= l->maxconn_per_route) { 1616 log_debug("debug: mta-routing: skipping route %s: too many connections", 1617 mta_route_to_text(route)); 1618 limit_route = 1; 1619 mta_route_unref(route); /* from here */ 1620 continue; 1621 } 1622 1623 if (route->lastconn + l->conndelay_route > now) { 1624 log_debug("debug: mta-routing: skipping route %s: cannot use before %llus (delay after connect)", 1625 mta_route_to_text(route), 1626 (unsigned long long) route->lastconn + l->conndelay_route - now); 1627 if (tm == 0 || route->lastconn + l->conndelay_route < tm) 1628 tm = route->lastconn + l->conndelay_route; 1629 mta_route_unref(route); /* from here */ 1630 continue; 1631 } 1632 1633 if (route->lastdisc + l->discdelay_route > now) { 1634 log_debug("debug: mta-routing: skipping route %s: cannot use before %llus (delay after disconnect)", 1635 mta_route_to_text(route), 1636 (unsigned long long) route->lastdisc + l->discdelay_route - now); 1637 if (tm == 0 || route->lastdisc + l->discdelay_route < tm) 1638 tm = route->lastdisc + l->discdelay_route; 1639 mta_route_unref(route); /* from here */ 1640 continue; 1641 } 1642 1643 /* Use the route with the lowest number of connections. */ 1644 if (best && route->nconn >= best->nconn) { 1645 log_debug("debug: mta-routing: skipping route %s: current one is better", 1646 mta_route_to_text(route)); 1647 mta_route_unref(route); /* from here */ 1648 continue; 1649 } 1650 1651 if (best) 1652 mta_route_unref(best); /* from here */ 1653 best = route; 1654 log_debug("debug: mta-routing: selecting candidate route %s", 1655 mta_route_to_text(route)); 1656 } 1657 1658 if (best) 1659 return (best); 1660 1661 /* Order is important */ 1662 if (seen == 0) { 1663 log_info("smtp-out: No MX found for %s", 1664 mta_connector_to_text(c)); 1665 c->flags |= CONNECTOR_ERROR_MX; 1666 } 1667 else if (limit_route) { 1668 log_debug("debug: mta: hit route limit"); 1669 *limits |= CONNECTOR_LIMIT_ROUTE; 1670 } 1671 else if (limit_host) { 1672 log_debug("debug: mta: hit host limit"); 1673 *limits |= CONNECTOR_LIMIT_HOST; 1674 } 1675 else if (tm) { 1676 if (tm > *nextconn) 1677 *nextconn = tm; 1678 } 1679 else if (family_mismatch) { 1680 log_info("smtp-out: Address family mismatch on %s", 1681 mta_connector_to_text(c)); 1682 c->flags |= CONNECTOR_ERROR_FAMILY; 1683 } 1684 else if (suspended_route) { 1685 log_info("smtp-out: No valid route for %s", 1686 mta_connector_to_text(c)); 1687 if (suspended_route & ROUTE_DISABLED_NET) 1688 c->flags |= CONNECTOR_ERROR_ROUTE_NET; 1689 if (suspended_route & ROUTE_DISABLED_SMTP) 1690 c->flags |= CONNECTOR_ERROR_ROUTE_SMTP; 1691 } 1692 1693 return (NULL); 1694 } 1695 1696 static void 1697 mta_log(const struct mta_envelope *evp, const char *prefix, const char *source, 1698 const char *relay, const char *status) 1699 { 1700 log_info("%016"PRIx64" mta delivery evpid=%016"PRIx64" " 1701 "from=<%s> to=<%s> rcpt=<%s> source=\"%s\" " 1702 "relay=\"%s\" delay=%s result=\"%s\" stat=\"%s\"", 1703 evp->session, 1704 evp->id, 1705 evp->task->sender, 1706 evp->dest, 1707 evp->rcpt ? evp->rcpt : "-", 1708 source ? source : "-", 1709 relay, 1710 duration_to_text(time(NULL) - evp->creation), 1711 prefix, 1712 status); 1713 } 1714 1715 static struct mta_relay * 1716 mta_relay(struct envelope *e, struct relayhost *relayh) 1717 { 1718 struct dispatcher *dispatcher; 1719 struct mta_relay key, *r; 1720 1721 dispatcher = dict_xget(env->sc_dispatchers, e->dispatcher); 1722 1723 memset(&key, 0, sizeof key); 1724 1725 key.pki_name = dispatcher->u.remote.pki; 1726 key.ca_name = dispatcher->u.remote.ca; 1727 key.authtable = dispatcher->u.remote.auth; 1728 key.sourcetable = dispatcher->u.remote.source; 1729 key.helotable = dispatcher->u.remote.helo_source; 1730 key.heloname = dispatcher->u.remote.helo; 1731 1732 if (relayh->hostname[0]) { 1733 key.domain = mta_domain(relayh->hostname, 1); 1734 } 1735 else { 1736 key.domain = mta_domain(e->dest.domain, 0); 1737 if (dispatcher->u.remote.backup) { 1738 key.backupname = dispatcher->u.remote.backupmx; 1739 if (key.backupname == NULL) 1740 key.backupname = e->smtpname; 1741 } 1742 } 1743 1744 key.tls = relayh->tls; 1745 key.flags |= relayh->flags; 1746 key.port = relayh->port; 1747 key.authlabel = relayh->authlabel; 1748 if (!key.authlabel[0]) 1749 key.authlabel = NULL; 1750 1751 if ((key.tls == RELAY_TLS_STARTTLS || key.tls == RELAY_TLS_SMTPS) && 1752 dispatcher->u.remote.tls_noverify == 0) 1753 key.flags |= RELAY_TLS_VERIFY; 1754 1755 if ((r = SPLAY_FIND(mta_relay_tree, &relays, &key)) == NULL) { 1756 r = xcalloc(1, sizeof *r); 1757 TAILQ_INIT(&r->tasks); 1758 r->id = generate_uid(); 1759 r->tls = key.tls; 1760 r->flags = key.flags; 1761 r->domain = key.domain; 1762 r->backupname = key.backupname ? 1763 xstrdup(key.backupname) : NULL; 1764 r->backuppref = -1; 1765 r->port = key.port; 1766 r->pki_name = key.pki_name ? xstrdup(key.pki_name) : NULL; 1767 r->ca_name = key.ca_name ? xstrdup(key.ca_name) : NULL; 1768 if (key.authtable) 1769 r->authtable = xstrdup(key.authtable); 1770 if (key.authlabel) 1771 r->authlabel = xstrdup(key.authlabel); 1772 if (key.sourcetable) 1773 r->sourcetable = xstrdup(key.sourcetable); 1774 if (key.helotable) 1775 r->helotable = xstrdup(key.helotable); 1776 if (key.heloname) 1777 r->heloname = xstrdup(key.heloname); 1778 SPLAY_INSERT(mta_relay_tree, &relays, r); 1779 stat_increment("mta.relay", 1); 1780 } else { 1781 mta_domain_unref(key.domain); /* from here */ 1782 } 1783 1784 r->refcount++; 1785 return (r); 1786 } 1787 1788 static void 1789 mta_relay_ref(struct mta_relay *r) 1790 { 1791 r->refcount++; 1792 } 1793 1794 static void 1795 mta_relay_unref(struct mta_relay *relay) 1796 { 1797 struct mta_connector *c; 1798 1799 if (--relay->refcount) 1800 return; 1801 1802 /* Make sure they are no envelopes held for this relay */ 1803 if (relay->state & RELAY_HOLDQ) { 1804 m_create(p_queue, IMSG_MTA_HOLDQ_RELEASE, 0, 0, -1); 1805 m_add_id(p_queue, relay->id); 1806 m_add_int(p_queue, 0); 1807 m_close(p_queue); 1808 } 1809 1810 log_debug("debug: mta: freeing %s", mta_relay_to_text(relay)); 1811 SPLAY_REMOVE(mta_relay_tree, &relays, relay); 1812 1813 while ((tree_poproot(&relay->connectors, NULL, (void**)&c))) 1814 mta_connector_free(c); 1815 1816 free(relay->authlabel); 1817 free(relay->authtable); 1818 free(relay->backupname); 1819 free(relay->pki_name); 1820 free(relay->ca_name); 1821 free(relay->helotable); 1822 free(relay->heloname); 1823 free(relay->secret); 1824 free(relay->sourcetable); 1825 1826 mta_domain_unref(relay->domain); /* from constructor */ 1827 free(relay); 1828 stat_decrement("mta.relay", 1); 1829 } 1830 1831 const char * 1832 mta_relay_to_text(struct mta_relay *relay) 1833 { 1834 static char buf[1024]; 1835 char tmp[32]; 1836 const char *sep = ","; 1837 1838 (void)snprintf(buf, sizeof buf, "[relay:%s", relay->domain->name); 1839 1840 if (relay->port) { 1841 (void)strlcat(buf, sep, sizeof buf); 1842 (void)snprintf(tmp, sizeof tmp, "port=%d", (int)relay->port); 1843 (void)strlcat(buf, tmp, sizeof buf); 1844 } 1845 1846 (void)strlcat(buf, sep, sizeof buf); 1847 switch(relay->tls) { 1848 case RELAY_TLS_OPPORTUNISTIC: 1849 (void)strlcat(buf, "smtp", sizeof buf); 1850 break; 1851 case RELAY_TLS_STARTTLS: 1852 (void)strlcat(buf, "smtp+tls", sizeof buf); 1853 break; 1854 case RELAY_TLS_SMTPS: 1855 (void)strlcat(buf, "smtps", sizeof buf); 1856 break; 1857 case RELAY_TLS_NO: 1858 if (relay->flags & RELAY_LMTP) 1859 (void)strlcat(buf, "lmtp", sizeof buf); 1860 else 1861 (void)strlcat(buf, "smtp+notls", sizeof buf); 1862 break; 1863 default: 1864 (void)strlcat(buf, "???", sizeof buf); 1865 } 1866 1867 if (relay->flags & RELAY_AUTH) { 1868 (void)strlcat(buf, sep, sizeof buf); 1869 (void)strlcat(buf, "auth=", sizeof buf); 1870 (void)strlcat(buf, relay->authtable, sizeof buf); 1871 (void)strlcat(buf, ":", sizeof buf); 1872 (void)strlcat(buf, relay->authlabel, sizeof buf); 1873 } 1874 1875 if (relay->pki_name) { 1876 (void)strlcat(buf, sep, sizeof buf); 1877 (void)strlcat(buf, "pki_name=", sizeof buf); 1878 (void)strlcat(buf, relay->pki_name, sizeof buf); 1879 } 1880 1881 if (relay->domain->as_host) { 1882 (void)strlcat(buf, sep, sizeof buf); 1883 (void)strlcat(buf, "mx", sizeof buf); 1884 } 1885 1886 if (relay->backupname) { 1887 (void)strlcat(buf, sep, sizeof buf); 1888 (void)strlcat(buf, "backup=", sizeof buf); 1889 (void)strlcat(buf, relay->backupname, sizeof buf); 1890 } 1891 1892 if (relay->sourcetable) { 1893 (void)strlcat(buf, sep, sizeof buf); 1894 (void)strlcat(buf, "sourcetable=", sizeof buf); 1895 (void)strlcat(buf, relay->sourcetable, sizeof buf); 1896 } 1897 1898 if (relay->helotable) { 1899 (void)strlcat(buf, sep, sizeof buf); 1900 (void)strlcat(buf, "helotable=", sizeof buf); 1901 (void)strlcat(buf, relay->helotable, sizeof buf); 1902 } 1903 1904 if (relay->heloname) { 1905 (void)strlcat(buf, sep, sizeof buf); 1906 (void)strlcat(buf, "heloname=", sizeof buf); 1907 (void)strlcat(buf, relay->heloname, sizeof buf); 1908 } 1909 1910 (void)strlcat(buf, "]", sizeof buf); 1911 1912 return (buf); 1913 } 1914 1915 static void 1916 mta_relay_show(struct mta_relay *r, struct mproc *p, uint32_t id, time_t t) 1917 { 1918 struct mta_connector *c; 1919 void *iter; 1920 char buf[1024], flags[1024], dur[64]; 1921 time_t to; 1922 1923 flags[0] = '\0'; 1924 1925 #define SHOWSTATUS(f, n) do { \ 1926 if (r->status & (f)) { \ 1927 if (flags[0]) \ 1928 (void)strlcat(flags, ",", sizeof(flags)); \ 1929 (void)strlcat(flags, (n), sizeof(flags)); \ 1930 } \ 1931 } while(0) 1932 1933 SHOWSTATUS(RELAY_WAIT_MX, "MX"); 1934 SHOWSTATUS(RELAY_WAIT_PREFERENCE, "preference"); 1935 SHOWSTATUS(RELAY_WAIT_SECRET, "secret"); 1936 SHOWSTATUS(RELAY_WAIT_LIMITS, "limits"); 1937 SHOWSTATUS(RELAY_WAIT_SOURCE, "source"); 1938 SHOWSTATUS(RELAY_WAIT_CONNECTOR, "connector"); 1939 #undef SHOWSTATUS 1940 1941 if (runq_pending(runq_relay, r, &to)) 1942 (void)snprintf(dur, sizeof(dur), "%s", duration_to_text(to - t)); 1943 else 1944 (void)strlcpy(dur, "-", sizeof(dur)); 1945 1946 (void)snprintf(buf, sizeof(buf), "%s refcount=%d ntask=%zu nconn=%zu lastconn=%s timeout=%s wait=%s%s", 1947 mta_relay_to_text(r), 1948 r->refcount, 1949 r->ntask, 1950 r->nconn, 1951 r->lastconn ? duration_to_text(t - r->lastconn) : "-", 1952 dur, 1953 flags, 1954 (r->state & RELAY_ONHOLD) ? "ONHOLD" : ""); 1955 m_compose(p, IMSG_CTL_MTA_SHOW_RELAYS, id, 0, -1, buf, strlen(buf) + 1); 1956 1957 iter = NULL; 1958 while (tree_iter(&r->connectors, &iter, NULL, (void **)&c)) { 1959 1960 if (runq_pending(runq_connector, c, &to)) 1961 (void)snprintf(dur, sizeof(dur), "%s", duration_to_text(to - t)); 1962 else 1963 (void)strlcpy(dur, "-", sizeof(dur)); 1964 1965 flags[0] = '\0'; 1966 1967 #define SHOWFLAG(f, n) do { \ 1968 if (c->flags & (f)) { \ 1969 if (flags[0]) \ 1970 (void)strlcat(flags, ",", sizeof(flags)); \ 1971 (void)strlcat(flags, (n), sizeof(flags)); \ 1972 } \ 1973 } while(0) 1974 1975 SHOWFLAG(CONNECTOR_NEW, "NEW"); 1976 SHOWFLAG(CONNECTOR_WAIT, "WAIT"); 1977 1978 SHOWFLAG(CONNECTOR_ERROR_FAMILY, "ERROR_FAMILY"); 1979 SHOWFLAG(CONNECTOR_ERROR_SOURCE, "ERROR_SOURCE"); 1980 SHOWFLAG(CONNECTOR_ERROR_MX, "ERROR_MX"); 1981 SHOWFLAG(CONNECTOR_ERROR_ROUTE_NET, "ERROR_ROUTE_NET"); 1982 SHOWFLAG(CONNECTOR_ERROR_ROUTE_SMTP, "ERROR_ROUTE_SMTP"); 1983 SHOWFLAG(CONNECTOR_ERROR_BLOCKED, "ERROR_BLOCKED"); 1984 1985 SHOWFLAG(CONNECTOR_LIMIT_HOST, "LIMIT_HOST"); 1986 SHOWFLAG(CONNECTOR_LIMIT_ROUTE, "LIMIT_ROUTE"); 1987 SHOWFLAG(CONNECTOR_LIMIT_SOURCE, "LIMIT_SOURCE"); 1988 SHOWFLAG(CONNECTOR_LIMIT_RELAY, "LIMIT_RELAY"); 1989 SHOWFLAG(CONNECTOR_LIMIT_CONN, "LIMIT_CONN"); 1990 SHOWFLAG(CONNECTOR_LIMIT_DOMAIN, "LIMIT_DOMAIN"); 1991 #undef SHOWFLAG 1992 1993 (void)snprintf(buf, sizeof(buf), 1994 " connector %s refcount=%d nconn=%zu lastconn=%s timeout=%s flags=%s", 1995 mta_source_to_text(c->source), 1996 c->refcount, 1997 c->nconn, 1998 c->lastconn ? duration_to_text(t - c->lastconn) : "-", 1999 dur, 2000 flags); 2001 m_compose(p, IMSG_CTL_MTA_SHOW_RELAYS, id, 0, -1, buf, 2002 strlen(buf) + 1); 2003 2004 2005 } 2006 } 2007 2008 static int 2009 mta_relay_cmp(const struct mta_relay *a, const struct mta_relay *b) 2010 { 2011 int r; 2012 2013 if (a->domain < b->domain) 2014 return (-1); 2015 if (a->domain > b->domain) 2016 return (1); 2017 2018 if (a->tls < b->tls) 2019 return (-1); 2020 if (a->tls > b->tls) 2021 return (1); 2022 2023 if (a->flags < b->flags) 2024 return (-1); 2025 if (a->flags > b->flags) 2026 return (1); 2027 2028 if (a->port < b->port) 2029 return (-1); 2030 if (a->port > b->port) 2031 return (1); 2032 2033 if (a->authtable == NULL && b->authtable) 2034 return (-1); 2035 if (a->authtable && b->authtable == NULL) 2036 return (1); 2037 if (a->authtable && ((r = strcmp(a->authtable, b->authtable)))) 2038 return (r); 2039 if (a->authlabel && ((r = strcmp(a->authlabel, b->authlabel)))) 2040 return (r); 2041 if (a->sourcetable == NULL && b->sourcetable) 2042 return (-1); 2043 if (a->sourcetable && b->sourcetable == NULL) 2044 return (1); 2045 if (a->sourcetable && ((r = strcmp(a->sourcetable, b->sourcetable)))) 2046 return (r); 2047 if (a->helotable == NULL && b->helotable) 2048 return (-1); 2049 if (a->helotable && b->helotable == NULL) 2050 return (1); 2051 if (a->helotable && ((r = strcmp(a->helotable, b->helotable)))) 2052 return (r); 2053 if (a->heloname == NULL && b->heloname) 2054 return (-1); 2055 if (a->heloname && b->heloname == NULL) 2056 return (1); 2057 if (a->heloname && ((r = strcmp(a->heloname, b->heloname)))) 2058 return (r); 2059 2060 if (a->pki_name == NULL && b->pki_name) 2061 return (-1); 2062 if (a->pki_name && b->pki_name == NULL) 2063 return (1); 2064 if (a->pki_name && ((r = strcmp(a->pki_name, b->pki_name)))) 2065 return (r); 2066 2067 if (a->ca_name == NULL && b->ca_name) 2068 return (-1); 2069 if (a->ca_name && b->ca_name == NULL) 2070 return (1); 2071 if (a->ca_name && ((r = strcmp(a->ca_name, b->ca_name)))) 2072 return (r); 2073 2074 if (a->backupname && ((r = strcmp(a->backupname, b->backupname)))) 2075 return (r); 2076 2077 return (0); 2078 } 2079 2080 SPLAY_GENERATE(mta_relay_tree, mta_relay, entry, mta_relay_cmp); 2081 2082 static struct mta_host * 2083 mta_host(const struct sockaddr *sa) 2084 { 2085 struct mta_host key, *h; 2086 struct sockaddr_storage ss; 2087 2088 memmove(&ss, sa, sa->sa_len); 2089 key.sa = (struct sockaddr*)&ss; 2090 h = SPLAY_FIND(mta_host_tree, &hosts, &key); 2091 2092 if (h == NULL) { 2093 h = xcalloc(1, sizeof(*h)); 2094 h->sa = xmemdup(sa, sa->sa_len); 2095 SPLAY_INSERT(mta_host_tree, &hosts, h); 2096 stat_increment("mta.host", 1); 2097 } 2098 2099 h->refcount++; 2100 return (h); 2101 } 2102 2103 static void 2104 mta_host_ref(struct mta_host *h) 2105 { 2106 h->refcount++; 2107 } 2108 2109 static void 2110 mta_host_unref(struct mta_host *h) 2111 { 2112 if (--h->refcount) 2113 return; 2114 2115 SPLAY_REMOVE(mta_host_tree, &hosts, h); 2116 free(h->sa); 2117 free(h->ptrname); 2118 free(h); 2119 stat_decrement("mta.host", 1); 2120 } 2121 2122 const char * 2123 mta_host_to_text(struct mta_host *h) 2124 { 2125 static char buf[1024]; 2126 2127 if (h->ptrname) 2128 (void)snprintf(buf, sizeof buf, "%s (%s)", 2129 sa_to_text(h->sa), h->ptrname); 2130 else 2131 (void)snprintf(buf, sizeof buf, "%s", sa_to_text(h->sa)); 2132 2133 return (buf); 2134 } 2135 2136 static int 2137 mta_host_cmp(const struct mta_host *a, const struct mta_host *b) 2138 { 2139 if (a->sa->sa_len < b->sa->sa_len) 2140 return (-1); 2141 if (a->sa->sa_len > b->sa->sa_len) 2142 return (1); 2143 return (memcmp(a->sa, b->sa, a->sa->sa_len)); 2144 } 2145 2146 SPLAY_GENERATE(mta_host_tree, mta_host, entry, mta_host_cmp); 2147 2148 static struct mta_domain * 2149 mta_domain(char *name, int as_host) 2150 { 2151 struct mta_domain key, *d; 2152 2153 key.name = name; 2154 key.as_host = as_host; 2155 d = SPLAY_FIND(mta_domain_tree, &domains, &key); 2156 2157 if (d == NULL) { 2158 d = xcalloc(1, sizeof(*d)); 2159 d->name = xstrdup(name); 2160 d->as_host = as_host; 2161 TAILQ_INIT(&d->mxs); 2162 SPLAY_INSERT(mta_domain_tree, &domains, d); 2163 stat_increment("mta.domain", 1); 2164 } 2165 2166 d->refcount++; 2167 return (d); 2168 } 2169 2170 #if 0 2171 static void 2172 mta_domain_ref(struct mta_domain *d) 2173 { 2174 d->refcount++; 2175 } 2176 #endif 2177 2178 static void 2179 mta_domain_unref(struct mta_domain *d) 2180 { 2181 struct mta_mx *mx; 2182 2183 if (--d->refcount) 2184 return; 2185 2186 while ((mx = TAILQ_FIRST(&d->mxs))) { 2187 TAILQ_REMOVE(&d->mxs, mx, entry); 2188 mta_host_unref(mx->host); /* from IMSG_DNS_HOST */ 2189 free(mx); 2190 } 2191 2192 SPLAY_REMOVE(mta_domain_tree, &domains, d); 2193 free(d->name); 2194 free(d); 2195 stat_decrement("mta.domain", 1); 2196 } 2197 2198 static int 2199 mta_domain_cmp(const struct mta_domain *a, const struct mta_domain *b) 2200 { 2201 if (a->as_host < b->as_host) 2202 return (-1); 2203 if (a->as_host > b->as_host) 2204 return (1); 2205 return (strcasecmp(a->name, b->name)); 2206 } 2207 2208 SPLAY_GENERATE(mta_domain_tree, mta_domain, entry, mta_domain_cmp); 2209 2210 static struct mta_source * 2211 mta_source(const struct sockaddr *sa) 2212 { 2213 struct mta_source key, *s; 2214 struct sockaddr_storage ss; 2215 2216 if (sa) { 2217 memmove(&ss, sa, sa->sa_len); 2218 key.sa = (struct sockaddr*)&ss; 2219 } else 2220 key.sa = NULL; 2221 s = SPLAY_FIND(mta_source_tree, &sources, &key); 2222 2223 if (s == NULL) { 2224 s = xcalloc(1, sizeof(*s)); 2225 if (sa) 2226 s->sa = xmemdup(sa, sa->sa_len); 2227 SPLAY_INSERT(mta_source_tree, &sources, s); 2228 stat_increment("mta.source", 1); 2229 } 2230 2231 s->refcount++; 2232 return (s); 2233 } 2234 2235 static void 2236 mta_source_ref(struct mta_source *s) 2237 { 2238 s->refcount++; 2239 } 2240 2241 static void 2242 mta_source_unref(struct mta_source *s) 2243 { 2244 if (--s->refcount) 2245 return; 2246 2247 SPLAY_REMOVE(mta_source_tree, &sources, s); 2248 free(s->sa); 2249 free(s); 2250 stat_decrement("mta.source", 1); 2251 } 2252 2253 static const char * 2254 mta_source_to_text(struct mta_source *s) 2255 { 2256 static char buf[1024]; 2257 2258 if (s->sa == NULL) 2259 return "[]"; 2260 (void)snprintf(buf, sizeof buf, "%s", sa_to_text(s->sa)); 2261 return (buf); 2262 } 2263 2264 static int 2265 mta_source_cmp(const struct mta_source *a, const struct mta_source *b) 2266 { 2267 if (a->sa == NULL) 2268 return ((b->sa == NULL) ? 0 : -1); 2269 if (b->sa == NULL) 2270 return (1); 2271 if (a->sa->sa_len < b->sa->sa_len) 2272 return (-1); 2273 if (a->sa->sa_len > b->sa->sa_len) 2274 return (1); 2275 return (memcmp(a->sa, b->sa, a->sa->sa_len)); 2276 } 2277 2278 SPLAY_GENERATE(mta_source_tree, mta_source, entry, mta_source_cmp); 2279 2280 static struct mta_connector * 2281 mta_connector(struct mta_relay *relay, struct mta_source *source) 2282 { 2283 struct mta_connector *c; 2284 2285 c = tree_get(&relay->connectors, (uintptr_t)(source)); 2286 if (c == NULL) { 2287 c = xcalloc(1, sizeof(*c)); 2288 c->relay = relay; 2289 c->source = source; 2290 c->flags |= CONNECTOR_NEW; 2291 mta_source_ref(source); 2292 tree_xset(&relay->connectors, (uintptr_t)(source), c); 2293 stat_increment("mta.connector", 1); 2294 log_debug("debug: mta: new %s", mta_connector_to_text(c)); 2295 } 2296 2297 return (c); 2298 } 2299 2300 static void 2301 mta_connector_free(struct mta_connector *c) 2302 { 2303 log_debug("debug: mta: freeing %s", 2304 mta_connector_to_text(c)); 2305 2306 if (c->flags & CONNECTOR_WAIT) { 2307 log_debug("debug: mta: cancelling timeout for %s", 2308 mta_connector_to_text(c)); 2309 runq_cancel(runq_connector, c); 2310 } 2311 mta_source_unref(c->source); /* from constructor */ 2312 free(c); 2313 2314 stat_decrement("mta.connector", 1); 2315 } 2316 2317 static const char * 2318 mta_connector_to_text(struct mta_connector *c) 2319 { 2320 static char buf[1024]; 2321 2322 (void)snprintf(buf, sizeof buf, "[connector:%s->%s,0x%x]", 2323 mta_source_to_text(c->source), 2324 mta_relay_to_text(c->relay), 2325 c->flags); 2326 return (buf); 2327 } 2328 2329 static struct mta_route * 2330 mta_route(struct mta_source *src, struct mta_host *dst) 2331 { 2332 struct mta_route key, *r; 2333 static uint64_t rid = 0; 2334 2335 key.src = src; 2336 key.dst = dst; 2337 r = SPLAY_FIND(mta_route_tree, &routes, &key); 2338 2339 if (r == NULL) { 2340 r = xcalloc(1, sizeof(*r)); 2341 r->src = src; 2342 r->dst = dst; 2343 r->flags |= ROUTE_NEW; 2344 r->id = ++rid; 2345 SPLAY_INSERT(mta_route_tree, &routes, r); 2346 mta_source_ref(src); 2347 mta_host_ref(dst); 2348 stat_increment("mta.route", 1); 2349 } 2350 else if (r->flags & ROUTE_RUNQ) { 2351 log_debug("debug: mta: mta_route_ref(): cancelling runq for route %s", 2352 mta_route_to_text(r)); 2353 r->flags &= ~(ROUTE_RUNQ | ROUTE_KEEPALIVE); 2354 runq_cancel(runq_route, r); 2355 r->refcount--; /* from mta_route_unref() */ 2356 } 2357 2358 r->refcount++; 2359 return (r); 2360 } 2361 2362 static void 2363 mta_route_ref(struct mta_route *r) 2364 { 2365 r->refcount++; 2366 } 2367 2368 static void 2369 mta_route_unref(struct mta_route *r) 2370 { 2371 time_t sched, now; 2372 int delay; 2373 2374 if (--r->refcount) 2375 return; 2376 2377 /* 2378 * Nothing references this route, but we might want to keep it alive 2379 * for a while. 2380 */ 2381 now = time(NULL); 2382 sched = 0; 2383 2384 if (r->penalty) { 2385 #if DELAY_QUADRATIC 2386 delay = DELAY_ROUTE_BASE * r->penalty * r->penalty; 2387 #else 2388 delay = 15 * 60; 2389 #endif 2390 if (delay > DELAY_ROUTE_MAX) 2391 delay = DELAY_ROUTE_MAX; 2392 sched = r->lastpenalty + delay; 2393 log_debug("debug: mta: mta_route_unref(): keeping route %s alive for %llus (penalty %d)", 2394 mta_route_to_text(r), (unsigned long long) sched - now, r->penalty); 2395 } else if (!(r->flags & ROUTE_KEEPALIVE)) { 2396 if (r->lastconn + max_seen_conndelay_route > now) 2397 sched = r->lastconn + max_seen_conndelay_route; 2398 if (r->lastdisc + max_seen_discdelay_route > now && 2399 r->lastdisc + max_seen_discdelay_route < sched) 2400 sched = r->lastdisc + max_seen_discdelay_route; 2401 2402 if (sched > now) 2403 log_debug("debug: mta: mta_route_unref(): keeping route %s alive for %llus (imposed delay)", 2404 mta_route_to_text(r), (unsigned long long) sched - now); 2405 } 2406 2407 if (sched > now) { 2408 r->flags |= ROUTE_RUNQ; 2409 runq_schedule_at(runq_route, sched, r); 2410 r->refcount++; 2411 return; 2412 } 2413 2414 log_debug("debug: mta: mta_route_unref(): really discarding route %s", 2415 mta_route_to_text(r)); 2416 2417 SPLAY_REMOVE(mta_route_tree, &routes, r); 2418 mta_source_unref(r->src); /* from constructor */ 2419 mta_host_unref(r->dst); /* from constructor */ 2420 free(r); 2421 stat_decrement("mta.route", 1); 2422 } 2423 2424 static const char * 2425 mta_route_to_text(struct mta_route *r) 2426 { 2427 static char buf[1024]; 2428 2429 (void)snprintf(buf, sizeof buf, "%s <-> %s", 2430 mta_source_to_text(r->src), 2431 mta_host_to_text(r->dst)); 2432 2433 return (buf); 2434 } 2435 2436 static int 2437 mta_route_cmp(const struct mta_route *a, const struct mta_route *b) 2438 { 2439 if (a->src < b->src) 2440 return (-1); 2441 if (a->src > b->src) 2442 return (1); 2443 2444 if (a->dst < b->dst) 2445 return (-1); 2446 if (a->dst > b->dst) 2447 return (1); 2448 2449 return (0); 2450 } 2451 2452 SPLAY_GENERATE(mta_route_tree, mta_route, entry, mta_route_cmp); 2453 2454 void 2455 mta_block(struct mta_source *src, char *dom) 2456 { 2457 struct mta_block key, *b; 2458 2459 key.source = src; 2460 key.domain = dom; 2461 2462 b = SPLAY_FIND(mta_block_tree, &blocks, &key); 2463 if (b != NULL) 2464 return; 2465 2466 b = xcalloc(1, sizeof(*b)); 2467 if (dom) 2468 b->domain = xstrdup(dom); 2469 b->source = src; 2470 mta_source_ref(src); 2471 SPLAY_INSERT(mta_block_tree, &blocks, b); 2472 } 2473 2474 void 2475 mta_unblock(struct mta_source *src, char *dom) 2476 { 2477 struct mta_block key, *b; 2478 2479 key.source = src; 2480 key.domain = dom; 2481 2482 b = SPLAY_FIND(mta_block_tree, &blocks, &key); 2483 if (b == NULL) 2484 return; 2485 2486 SPLAY_REMOVE(mta_block_tree, &blocks, b); 2487 2488 mta_source_unref(b->source); 2489 free(b->domain); 2490 free(b); 2491 } 2492 2493 int 2494 mta_is_blocked(struct mta_source *src, char *dom) 2495 { 2496 struct mta_block key; 2497 2498 key.source = src; 2499 key.domain = dom; 2500 2501 if (SPLAY_FIND(mta_block_tree, &blocks, &key)) 2502 return (1); 2503 2504 return (0); 2505 } 2506 2507 static 2508 int 2509 mta_block_cmp(const struct mta_block *a, const struct mta_block *b) 2510 { 2511 if (a->source < b->source) 2512 return (-1); 2513 if (a->source > b->source) 2514 return (1); 2515 if (!a->domain && b->domain) 2516 return (-1); 2517 if (a->domain && !b->domain) 2518 return (1); 2519 if (a->domain == b->domain) 2520 return (0); 2521 return (strcasecmp(a->domain, b->domain)); 2522 } 2523 2524 SPLAY_GENERATE(mta_block_tree, mta_block, entry, mta_block_cmp); 2525 2526 2527 2528 /* hoststat errors are not critical, we do best effort */ 2529 void 2530 mta_hoststat_update(const char *host, const char *error) 2531 { 2532 struct hoststat *hs = NULL; 2533 char buf[HOST_NAME_MAX+1]; 2534 2535 if (!lowercase(buf, host, sizeof buf)) 2536 return; 2537 2538 hs = dict_get(&hoststat, buf); 2539 if (hs == NULL) { 2540 if ((hs = calloc(1, sizeof *hs)) == NULL) 2541 return; 2542 tree_init(&hs->deferred); 2543 runq_schedule(runq_hoststat, HOSTSTAT_EXPIRE_DELAY, hs); 2544 } 2545 (void)strlcpy(hs->name, buf, sizeof hs->name); 2546 (void)strlcpy(hs->error, error, sizeof hs->error); 2547 hs->tm = time(NULL); 2548 dict_set(&hoststat, buf, hs); 2549 2550 runq_cancel(runq_hoststat, hs); 2551 runq_schedule(runq_hoststat, HOSTSTAT_EXPIRE_DELAY, hs); 2552 } 2553 2554 void 2555 mta_hoststat_cache(const char *host, uint64_t evpid) 2556 { 2557 struct hoststat *hs = NULL; 2558 char buf[HOST_NAME_MAX+1]; 2559 2560 if (!lowercase(buf, host, sizeof buf)) 2561 return; 2562 2563 hs = dict_get(&hoststat, buf); 2564 if (hs == NULL) 2565 return; 2566 2567 if (tree_count(&hs->deferred) >= env->sc_mta_max_deferred) 2568 return; 2569 2570 tree_set(&hs->deferred, evpid, NULL); 2571 } 2572 2573 void 2574 mta_hoststat_uncache(const char *host, uint64_t evpid) 2575 { 2576 struct hoststat *hs = NULL; 2577 char buf[HOST_NAME_MAX+1]; 2578 2579 if (!lowercase(buf, host, sizeof buf)) 2580 return; 2581 2582 hs = dict_get(&hoststat, buf); 2583 if (hs == NULL) 2584 return; 2585 2586 tree_pop(&hs->deferred, evpid); 2587 } 2588 2589 void 2590 mta_hoststat_reschedule(const char *host) 2591 { 2592 struct hoststat *hs = NULL; 2593 char buf[HOST_NAME_MAX+1]; 2594 uint64_t evpid; 2595 2596 if (!lowercase(buf, host, sizeof buf)) 2597 return; 2598 2599 hs = dict_get(&hoststat, buf); 2600 if (hs == NULL) 2601 return; 2602 2603 while (tree_poproot(&hs->deferred, &evpid, NULL)) { 2604 m_compose(p_queue, IMSG_MTA_SCHEDULE, 0, 0, -1, 2605 &evpid, sizeof evpid); 2606 } 2607 } 2608 2609 static void 2610 mta_hoststat_remove_entry(struct hoststat *hs) 2611 { 2612 while (tree_poproot(&hs->deferred, NULL, NULL)) 2613 ; 2614 dict_pop(&hoststat, hs->name); 2615 runq_cancel(runq_hoststat, hs); 2616 } 2617