1 /* $OpenBSD: relay.c,v 1.175 2014/07/14 00:11:12 bluhm Exp $ */ 2 3 /* 4 * Copyright (c) 2006 - 2014 Reyk Floeter <reyk@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/types.h> 20 #include <sys/queue.h> 21 #include <sys/time.h> 22 #include <sys/stat.h> 23 #include <sys/socket.h> 24 #include <sys/un.h> 25 #include <sys/tree.h> 26 #include <sys/hash.h> 27 28 #include <net/if.h> 29 #include <netinet/in_systm.h> 30 #include <netinet/in.h> 31 #include <netinet/ip.h> 32 #include <netinet/tcp.h> 33 #include <arpa/inet.h> 34 35 #include <errno.h> 36 #include <fcntl.h> 37 #include <stdlib.h> 38 #include <string.h> 39 #include <unistd.h> 40 #include <stdio.h> 41 #include <err.h> 42 #include <pwd.h> 43 #include <event.h> 44 #include <fnmatch.h> 45 46 #include <openssl/dh.h> 47 #include <openssl/ssl.h> 48 49 #include "relayd.h" 50 51 void relay_statistics(int, short, void *); 52 int relay_dispatch_parent(int, struct privsep_proc *, 53 struct imsg *); 54 int relay_dispatch_pfe(int, struct privsep_proc *, 55 struct imsg *); 56 int relay_dispatch_ca(int, struct privsep_proc *, 57 struct imsg *); 58 void relay_shutdown(void); 59 60 void relay_protodebug(struct relay *); 61 void relay_ruledebug(struct relay_rule *); 62 void relay_init(struct privsep *, struct privsep_proc *p, void *); 63 void relay_launch(void); 64 int relay_socket(struct sockaddr_storage *, in_port_t, 65 struct protocol *, int, int); 66 int relay_socket_listen(struct sockaddr_storage *, in_port_t, 67 struct protocol *); 68 int relay_socket_connect(struct sockaddr_storage *, in_port_t, 69 struct protocol *, int); 70 71 void relay_accept(int, short, void *); 72 void relay_input(struct rsession *); 73 74 u_int32_t relay_hash_addr(struct sockaddr_storage *, u_int32_t); 75 76 DH * relay_ssl_get_dhparams(int); 77 void relay_ssl_callback_info(const SSL *, int, int); 78 DH *relay_ssl_callback_dh(SSL *, int, int); 79 SSL_CTX *relay_ssl_ctx_create(struct relay *); 80 void relay_ssl_transaction(struct rsession *, 81 struct ctl_relay_event *); 82 void relay_ssl_accept(int, short, void *); 83 void relay_connect_retry(int, short, void *); 84 void relay_ssl_connect(int, short, void *); 85 void relay_ssl_connected(struct ctl_relay_event *); 86 void relay_ssl_readcb(int, short, void *); 87 void relay_ssl_writecb(int, short, void *); 88 89 char *relay_load_file(const char *, off_t *); 90 extern void bufferevent_read_pressure_cb(struct evbuffer *, size_t, 91 size_t, void *); 92 93 volatile int relay_sessions; 94 volatile int relay_inflight = 0; 95 objid_t relay_conid; 96 97 static struct relayd *env = NULL; 98 int proc_id; 99 100 static struct privsep_proc procs[] = { 101 { "parent", PROC_PARENT, relay_dispatch_parent }, 102 { "pfe", PROC_PFE, relay_dispatch_pfe }, 103 { "ca", PROC_CA, relay_dispatch_ca } 104 }; 105 106 pid_t 107 relay(struct privsep *ps, struct privsep_proc *p) 108 { 109 pid_t pid; 110 env = ps->ps_env; 111 pid = proc_run(ps, p, procs, nitems(procs), relay_init, NULL); 112 relay_http(env); 113 return (pid); 114 } 115 116 void 117 relay_shutdown(void) 118 { 119 config_purge(env, CONFIG_ALL); 120 usleep(200); /* XXX relay needs to shutdown last */ 121 } 122 123 void 124 relay_ruledebug(struct relay_rule *rule) 125 { 126 struct kv *kv = NULL; 127 u_int i; 128 129 fprintf(stderr, "\t\t"); 130 131 switch (rule->rule_action) { 132 case RULE_ACTION_MATCH: 133 fprintf(stderr, "match "); 134 break; 135 case RULE_ACTION_BLOCK: 136 fprintf(stderr, "block "); 137 break; 138 case RULE_ACTION_PASS: 139 fprintf(stderr, "pass "); 140 break; 141 } 142 143 switch (rule->rule_dir) { 144 case RELAY_DIR_ANY: 145 break; 146 case RELAY_DIR_REQUEST: 147 fprintf(stderr, "request "); 148 break; 149 case RELAY_DIR_RESPONSE: 150 fprintf(stderr, "response "); 151 break; 152 default: 153 return; 154 /* NOTREACHED */ 155 break; 156 } 157 158 if (rule->rule_flags & RULE_FLAG_QUICK) 159 fprintf(stderr, "quick "); 160 161 for (i = 1; i < KEY_TYPE_MAX; i++) { 162 kv = &rule->rule_kv[i]; 163 if (kv->kv_type != i) 164 continue; 165 166 switch (kv->kv_type) { 167 case KEY_TYPE_COOKIE: 168 fprintf(stderr, "cookie "); 169 break; 170 case KEY_TYPE_HEADER: 171 fprintf(stderr, "header "); 172 break; 173 case KEY_TYPE_PATH: 174 fprintf(stderr, "path "); 175 break; 176 case KEY_TYPE_QUERY: 177 fprintf(stderr, "query "); 178 break; 179 case KEY_TYPE_URL: 180 fprintf(stderr, "url "); 181 break; 182 default: 183 continue; 184 } 185 186 switch (kv->kv_option) { 187 case KEY_OPTION_APPEND: 188 fprintf(stderr, "append "); 189 break; 190 case KEY_OPTION_SET: 191 fprintf(stderr, "set "); 192 break; 193 case KEY_OPTION_REMOVE: 194 fprintf(stderr, "remove "); 195 break; 196 case KEY_OPTION_HASH: 197 fprintf(stderr, "hash "); 198 break; 199 case KEY_OPTION_LOG: 200 fprintf(stderr, "log "); 201 break; 202 case KEY_OPTION_NONE: 203 break; 204 } 205 206 switch (kv->kv_digest) { 207 case DIGEST_SHA1: 208 case DIGEST_MD5: 209 fprintf(stderr, "digest "); 210 break; 211 default: 212 break; 213 } 214 215 fprintf(stderr, "%s%s%s%s%s%s ", 216 kv->kv_key == NULL ? "" : "\"", 217 kv->kv_key == NULL ? "" : kv->kv_key, 218 kv->kv_key == NULL ? "" : "\"", 219 kv->kv_value == NULL ? "" : " value \"", 220 kv->kv_value == NULL ? "" : kv->kv_value, 221 kv->kv_value == NULL ? "" : "\""); 222 } 223 224 if (rule->rule_tablename[0]) 225 fprintf(stderr, "forward to <%s> ", rule->rule_tablename); 226 227 if (rule->rule_tag == -1) 228 fprintf(stderr, "no tag "); 229 else if (rule->rule_tag && rule->rule_tagname[0]) 230 fprintf(stderr, "tag \"%s\" ", 231 rule->rule_tagname); 232 233 if (rule->rule_tagged && rule->rule_taggedname[0]) 234 fprintf(stderr, "tagged \"%s\" ", 235 rule->rule_taggedname); 236 237 if (rule->rule_label == -1) 238 fprintf(stderr, "no label "); 239 else if (rule->rule_label && rule->rule_labelname[0]) 240 fprintf(stderr, "label \"%s\" ", 241 rule->rule_labelname); 242 243 fprintf(stderr, "\n"); 244 } 245 246 void 247 relay_protodebug(struct relay *rlay) 248 { 249 struct protocol *proto = rlay->rl_proto; 250 struct relay_rule *rule = NULL; 251 252 fprintf(stderr, "protocol %d: name %s\n", 253 proto->id, proto->name); 254 fprintf(stderr, "\tflags: %s, relay flags: %s\n", 255 printb_flags(proto->flags, F_BITS), 256 printb_flags(rlay->rl_conf.flags, F_BITS)); 257 if (proto->tcpflags) 258 fprintf(stderr, "\ttcp flags: %s\n", 259 printb_flags(proto->tcpflags, TCPFLAG_BITS)); 260 if ((rlay->rl_conf.flags & (F_SSL|F_SSLCLIENT)) && proto->sslflags) 261 fprintf(stderr, "\tssl flags: %s\n", 262 printb_flags(proto->sslflags, SSLFLAG_BITS)); 263 if (proto->cache != -1) 264 fprintf(stderr, "\tssl session cache: %d\n", proto->cache); 265 fprintf(stderr, "\ttype: "); 266 switch (proto->type) { 267 case RELAY_PROTO_TCP: 268 fprintf(stderr, "tcp\n"); 269 break; 270 case RELAY_PROTO_HTTP: 271 fprintf(stderr, "http\n"); 272 break; 273 case RELAY_PROTO_DNS: 274 fprintf(stderr, "dns\n"); 275 break; 276 } 277 278 rule = TAILQ_FIRST(&proto->rules); 279 while (rule != NULL) { 280 relay_ruledebug(rule); 281 rule = TAILQ_NEXT(rule, rule_entry); 282 } 283 } 284 285 int 286 relay_privinit(struct relay *rlay) 287 { 288 extern int debug; 289 290 log_debug("%s: adding relay %s", __func__, rlay->rl_conf.name); 291 292 if (debug) 293 relay_protodebug(rlay); 294 295 switch (rlay->rl_proto->type) { 296 case RELAY_PROTO_DNS: 297 relay_udp_privinit(env, rlay); 298 break; 299 case RELAY_PROTO_TCP: 300 break; 301 case RELAY_PROTO_HTTP: 302 break; 303 } 304 305 if (rlay->rl_conf.flags & F_UDP) 306 rlay->rl_s = relay_udp_bind(&rlay->rl_conf.ss, 307 rlay->rl_conf.port, rlay->rl_proto); 308 else 309 rlay->rl_s = relay_socket_listen(&rlay->rl_conf.ss, 310 rlay->rl_conf.port, rlay->rl_proto); 311 if (rlay->rl_s == -1) 312 return (-1); 313 314 return (0); 315 } 316 317 void 318 relay_init(struct privsep *ps, struct privsep_proc *p, void *arg) 319 { 320 struct timeval tv; 321 322 if (config_init(ps->ps_env) == -1) 323 fatal("failed to initialize configuration"); 324 325 /* Set to current prefork id */ 326 proc_id = p->p_instance; 327 328 /* We use a custom shutdown callback */ 329 p->p_shutdown = relay_shutdown; 330 331 /* Unlimited file descriptors (use system limits) */ 332 socket_rlimit(-1); 333 334 /* Schedule statistics timer */ 335 evtimer_set(&env->sc_statev, relay_statistics, NULL); 336 bcopy(&env->sc_statinterval, &tv, sizeof(tv)); 337 evtimer_add(&env->sc_statev, &tv); 338 } 339 340 void 341 relay_statistics(int fd, short events, void *arg) 342 { 343 struct relay *rlay; 344 struct ctl_stats crs, *cur; 345 struct timeval tv, tv_now; 346 int resethour = 0, resetday = 0; 347 struct rsession *con, *next_con; 348 349 /* 350 * This is a hack to calculate some average statistics. 351 * It doesn't try to be very accurate, but could be improved... 352 */ 353 354 timerclear(&tv); 355 getmonotime(&tv_now); 356 357 TAILQ_FOREACH(rlay, env->sc_relays, rl_entry) { 358 bzero(&crs, sizeof(crs)); 359 resethour = resetday = 0; 360 361 cur = &rlay->rl_stats[proc_id]; 362 cur->cnt += cur->last; 363 cur->tick++; 364 cur->avg = (cur->last + cur->avg) / 2; 365 cur->last_hour += cur->last; 366 if ((cur->tick % (3600 / env->sc_statinterval.tv_sec)) == 0) { 367 cur->avg_hour = (cur->last_hour + cur->avg_hour) / 2; 368 resethour++; 369 } 370 cur->last_day += cur->last; 371 if ((cur->tick % (86400 / env->sc_statinterval.tv_sec)) == 0) { 372 cur->avg_day = (cur->last_day + cur->avg_day) / 2; 373 resethour++; 374 } 375 bcopy(cur, &crs, sizeof(crs)); 376 377 cur->last = 0; 378 if (resethour) 379 cur->last_hour = 0; 380 if (resetday) 381 cur->last_day = 0; 382 383 crs.id = rlay->rl_conf.id; 384 crs.proc = proc_id; 385 proc_compose_imsg(env->sc_ps, PROC_PFE, -1, IMSG_STATISTICS, -1, 386 &crs, sizeof(crs)); 387 388 for (con = SPLAY_ROOT(&rlay->rl_sessions); 389 con != NULL; con = next_con) { 390 next_con = SPLAY_NEXT(session_tree, 391 &rlay->rl_sessions, con); 392 timersub(&tv_now, &con->se_tv_last, &tv); 393 if (timercmp(&tv, &rlay->rl_conf.timeout, >=)) 394 relay_close(con, "hard timeout"); 395 } 396 } 397 398 /* Schedule statistics timer */ 399 evtimer_set(&env->sc_statev, relay_statistics, NULL); 400 bcopy(&env->sc_statinterval, &tv, sizeof(tv)); 401 evtimer_add(&env->sc_statev, &tv); 402 } 403 404 void 405 relay_launch(void) 406 { 407 void (*callback)(int, short, void *); 408 struct relay *rlay; 409 struct host *host; 410 struct relay_table *rlt; 411 412 TAILQ_FOREACH(rlay, env->sc_relays, rl_entry) { 413 if ((rlay->rl_conf.flags & (F_SSL|F_SSLCLIENT)) && 414 (rlay->rl_ssl_ctx = relay_ssl_ctx_create(rlay)) == NULL) 415 fatal("relay_init: failed to create SSL context"); 416 417 TAILQ_FOREACH(rlt, &rlay->rl_tables, rlt_entry) { 418 /* 419 * set rule->rule_table in advance and save time 420 * looking up for this later on rule/connection 421 * evalution 422 */ 423 rule_settable(&rlay->rl_proto->rules, rlt); 424 425 switch (rlt->rlt_mode) { 426 case RELAY_DSTMODE_ROUNDROBIN: 427 case RELAY_DSTMODE_RANDOM: 428 rlt->rlt_key = 0; 429 break; 430 case RELAY_DSTMODE_LOADBALANCE: 431 case RELAY_DSTMODE_HASH: 432 case RELAY_DSTMODE_SRCHASH: 433 rlt->rlt_key = 434 hash32_str(rlay->rl_conf.name, HASHINIT); 435 rlt->rlt_key = 436 hash32_str(rlt->rlt_table->conf.name, 437 rlt->rlt_key); 438 break; 439 } 440 rlt->rlt_nhosts = 0; 441 TAILQ_FOREACH(host, &rlt->rlt_table->hosts, entry) { 442 if (rlt->rlt_nhosts >= RELAY_MAXHOSTS) 443 fatal("relay_init: " 444 "too many hosts in table"); 445 host->idx = rlt->rlt_nhosts; 446 rlt->rlt_host[rlt->rlt_nhosts++] = host; 447 } 448 log_info("adding %d hosts from table %s%s", 449 rlt->rlt_nhosts, rlt->rlt_table->conf.name, 450 rlt->rlt_table->conf.check ? "" : " (no check)"); 451 } 452 453 switch (rlay->rl_proto->type) { 454 case RELAY_PROTO_DNS: 455 relay_udp_init(rlay); 456 break; 457 case RELAY_PROTO_TCP: 458 case RELAY_PROTO_HTTP: 459 relay_http_init(rlay); 460 /* Use defaults */ 461 break; 462 } 463 464 log_debug("%s: running relay %s", __func__, 465 rlay->rl_conf.name); 466 467 rlay->rl_up = HOST_UP; 468 469 if (rlay->rl_conf.flags & F_UDP) 470 callback = relay_udp_server; 471 else 472 callback = relay_accept; 473 474 event_set(&rlay->rl_ev, rlay->rl_s, EV_READ, 475 callback, rlay); 476 event_add(&rlay->rl_ev, NULL); 477 evtimer_set(&rlay->rl_evt, callback, rlay); 478 } 479 } 480 481 int 482 relay_socket_af(struct sockaddr_storage *ss, in_port_t port) 483 { 484 switch (ss->ss_family) { 485 case AF_INET: 486 ((struct sockaddr_in *)ss)->sin_port = port; 487 ((struct sockaddr_in *)ss)->sin_len = 488 sizeof(struct sockaddr_in); 489 break; 490 case AF_INET6: 491 ((struct sockaddr_in6 *)ss)->sin6_port = port; 492 ((struct sockaddr_in6 *)ss)->sin6_len = 493 sizeof(struct sockaddr_in6); 494 break; 495 default: 496 return (-1); 497 } 498 499 return (0); 500 } 501 502 in_port_t 503 relay_socket_getport(struct sockaddr_storage *ss) 504 { 505 switch (ss->ss_family) { 506 case AF_INET: 507 return (((struct sockaddr_in *)ss)->sin_port); 508 case AF_INET6: 509 return (((struct sockaddr_in6 *)ss)->sin6_port); 510 default: 511 return (0); 512 } 513 514 /* NOTREACHED */ 515 return (0); 516 } 517 518 int 519 relay_socket(struct sockaddr_storage *ss, in_port_t port, 520 struct protocol *proto, int fd, int reuseport) 521 { 522 struct linger lng; 523 int s = -1, val; 524 525 if (relay_socket_af(ss, port) == -1) 526 goto bad; 527 528 s = fd == -1 ? socket(ss->ss_family, SOCK_STREAM, IPPROTO_TCP) : fd; 529 if (s == -1) 530 goto bad; 531 532 /* 533 * Socket options 534 */ 535 bzero(&lng, sizeof(lng)); 536 if (setsockopt(s, SOL_SOCKET, SO_LINGER, &lng, sizeof(lng)) == -1) 537 goto bad; 538 if (reuseport) { 539 val = 1; 540 if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT, &val, 541 sizeof(int)) == -1) 542 goto bad; 543 } 544 if (fcntl(s, F_SETFL, O_NONBLOCK) == -1) 545 goto bad; 546 if (proto->tcpflags & TCPFLAG_BUFSIZ) { 547 val = proto->tcpbufsiz; 548 if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, 549 &val, sizeof(val)) == -1) 550 goto bad; 551 val = proto->tcpbufsiz; 552 if (setsockopt(s, SOL_SOCKET, SO_SNDBUF, 553 &val, sizeof(val)) == -1) 554 goto bad; 555 } 556 557 /* 558 * IP options 559 */ 560 if (proto->tcpflags & TCPFLAG_IPTTL) { 561 val = (int)proto->tcpipttl; 562 if (setsockopt(s, IPPROTO_IP, IP_TTL, 563 &val, sizeof(val)) == -1) 564 goto bad; 565 } 566 if (proto->tcpflags & TCPFLAG_IPMINTTL) { 567 val = (int)proto->tcpipminttl; 568 if (setsockopt(s, IPPROTO_IP, IP_MINTTL, 569 &val, sizeof(val)) == -1) 570 goto bad; 571 } 572 573 /* 574 * TCP options 575 */ 576 if (proto->tcpflags & (TCPFLAG_NODELAY|TCPFLAG_NNODELAY)) { 577 if (proto->tcpflags & TCPFLAG_NNODELAY) 578 val = 0; 579 else 580 val = 1; 581 if (setsockopt(s, IPPROTO_TCP, TCP_NODELAY, 582 &val, sizeof(val)) == -1) 583 goto bad; 584 } 585 if (proto->tcpflags & (TCPFLAG_SACK|TCPFLAG_NSACK)) { 586 if (proto->tcpflags & TCPFLAG_NSACK) 587 val = 0; 588 else 589 val = 1; 590 if (setsockopt(s, IPPROTO_TCP, TCP_SACK_ENABLE, 591 &val, sizeof(val)) == -1) 592 goto bad; 593 } 594 595 return (s); 596 597 bad: 598 if (s != -1) 599 close(s); 600 return (-1); 601 } 602 603 int 604 relay_socket_connect(struct sockaddr_storage *ss, in_port_t port, 605 struct protocol *proto, int fd) 606 { 607 int s; 608 609 if ((s = relay_socket(ss, port, proto, fd, 0)) == -1) 610 return (-1); 611 612 if (connect(s, (struct sockaddr *)ss, ss->ss_len) == -1) { 613 if (errno != EINPROGRESS) 614 goto bad; 615 } 616 617 return (s); 618 619 bad: 620 close(s); 621 return (-1); 622 } 623 624 int 625 relay_socket_listen(struct sockaddr_storage *ss, in_port_t port, 626 struct protocol *proto) 627 { 628 int s; 629 630 if ((s = relay_socket(ss, port, proto, -1, 1)) == -1) 631 return (-1); 632 633 if (bind(s, (struct sockaddr *)ss, ss->ss_len) == -1) 634 goto bad; 635 if (listen(s, proto->tcpbacklog) == -1) 636 goto bad; 637 638 return (s); 639 640 bad: 641 close(s); 642 return (-1); 643 } 644 645 void 646 relay_connected(int fd, short sig, void *arg) 647 { 648 struct rsession *con = arg; 649 struct relay *rlay = con->se_relay; 650 evbuffercb outrd = relay_read; 651 evbuffercb outwr = relay_write; 652 struct bufferevent *bev; 653 struct ctl_relay_event *out = &con->se_out; 654 socklen_t len; 655 int error; 656 657 if (sig == EV_TIMEOUT) { 658 relay_abort_http(con, 504, "connect timeout", 0); 659 return; 660 } 661 662 len = sizeof(error); 663 if (getsockopt(fd, SOL_SOCKET, SO_ERROR, &error, 664 &len) == -1 || error) { 665 if (error) 666 errno = error; 667 relay_abort_http(con, 500, "socket error", 0); 668 return; 669 } 670 671 if ((rlay->rl_conf.flags & F_SSLCLIENT) && (out->ssl == NULL)) { 672 relay_ssl_transaction(con, out); 673 return; 674 } 675 676 DPRINTF("%s: session %d: successful", __func__, con->se_id); 677 678 switch (rlay->rl_proto->type) { 679 case RELAY_PROTO_HTTP: 680 if (relay_httpdesc_init(out) == -1) { 681 relay_close(con, 682 "failed to allocate http descriptor"); 683 return; 684 } 685 con->se_out.toread = TOREAD_HTTP_HEADER; 686 outrd = relay_read_http; 687 break; 688 case RELAY_PROTO_TCP: 689 /* Use defaults */ 690 break; 691 default: 692 fatalx("relay_connected: unknown protocol"); 693 } 694 695 /* 696 * Relay <-> Server 697 */ 698 bev = bufferevent_new(fd, outrd, outwr, relay_error, &con->se_out); 699 if (bev == NULL) { 700 relay_abort_http(con, 500, 701 "failed to allocate output buffer event", 0); 702 return; 703 } 704 evbuffer_free(bev->output); 705 bev->output = con->se_out.output; 706 if (bev->output == NULL) 707 fatal("relay_connected: invalid output buffer"); 708 con->se_out.bev = bev; 709 710 /* Initialize the SSL wrapper */ 711 if ((rlay->rl_conf.flags & F_SSLCLIENT) && (out->ssl != NULL)) 712 relay_ssl_connected(out); 713 714 bufferevent_settimeout(bev, 715 rlay->rl_conf.timeout.tv_sec, rlay->rl_conf.timeout.tv_sec); 716 bufferevent_enable(bev, EV_READ|EV_WRITE); 717 718 if (relay_splice(&con->se_out) == -1) 719 relay_close(con, strerror(errno)); 720 } 721 722 void 723 relay_input(struct rsession *con) 724 { 725 struct relay *rlay = con->se_relay; 726 evbuffercb inrd = relay_read; 727 evbuffercb inwr = relay_write; 728 729 switch (rlay->rl_proto->type) { 730 case RELAY_PROTO_HTTP: 731 if (relay_httpdesc_init(&con->se_in) == -1) { 732 relay_close(con, 733 "failed to allocate http descriptor"); 734 return; 735 } 736 con->se_in.toread = TOREAD_HTTP_HEADER; 737 inrd = relay_read_http; 738 break; 739 case RELAY_PROTO_TCP: 740 /* Use defaults */ 741 break; 742 default: 743 fatalx("relay_input: unknown protocol"); 744 } 745 746 /* 747 * Client <-> Relay 748 */ 749 con->se_in.bev = bufferevent_new(con->se_in.s, inrd, inwr, 750 relay_error, &con->se_in); 751 if (con->se_in.bev == NULL) { 752 relay_close(con, "failed to allocate input buffer event"); 753 return; 754 } 755 756 /* Initialize the SSL wrapper */ 757 if ((rlay->rl_conf.flags & F_SSL) && con->se_in.ssl != NULL) 758 relay_ssl_connected(&con->se_in); 759 760 bufferevent_settimeout(con->se_in.bev, 761 rlay->rl_conf.timeout.tv_sec, rlay->rl_conf.timeout.tv_sec); 762 bufferevent_enable(con->se_in.bev, EV_READ|EV_WRITE); 763 764 if (relay_splice(&con->se_in) == -1) 765 relay_close(con, strerror(errno)); 766 } 767 768 void 769 relay_write(struct bufferevent *bev, void *arg) 770 { 771 struct ctl_relay_event *cre = arg; 772 struct rsession *con = cre->con; 773 774 getmonotime(&con->se_tv_last); 775 776 if (con->se_done) 777 goto done; 778 if (relay_splice(cre->dst) == -1) 779 goto fail; 780 return; 781 done: 782 relay_close(con, "last write (done)"); 783 return; 784 fail: 785 relay_close(con, strerror(errno)); 786 } 787 788 void 789 relay_dump(struct ctl_relay_event *cre, const void *buf, size_t len) 790 { 791 if (!len) 792 return; 793 794 /* 795 * This function will dump the specified message directly 796 * to the underlying session, without waiting for success 797 * of non-blocking events etc. This is useful to print an 798 * error message before gracefully closing the session. 799 */ 800 if (cre->ssl != NULL) 801 (void)SSL_write(cre->ssl, buf, len); 802 else 803 (void)write(cre->s, buf, len); 804 } 805 806 void 807 relay_read(struct bufferevent *bev, void *arg) 808 { 809 struct ctl_relay_event *cre = arg; 810 struct rsession *con = cre->con; 811 struct evbuffer *src = EVBUFFER_INPUT(bev); 812 813 getmonotime(&con->se_tv_last); 814 cre->timedout = 0; 815 816 if (!EVBUFFER_LENGTH(src)) 817 return; 818 if (relay_bufferevent_write_buffer(cre->dst, src) == -1) 819 goto fail; 820 if (con->se_done) 821 goto done; 822 if (cre->dst->bev) 823 bufferevent_enable(cre->dst->bev, EV_READ); 824 return; 825 done: 826 relay_close(con, "last read (done)"); 827 return; 828 fail: 829 relay_close(con, strerror(errno)); 830 } 831 832 int 833 relay_splice(struct ctl_relay_event *cre) 834 { 835 struct rsession *con = cre->con; 836 struct relay *rlay = con->se_relay; 837 struct protocol *proto = rlay->rl_proto; 838 struct splice sp; 839 840 if ((rlay->rl_conf.flags & (F_SSL|F_SSLCLIENT)) || 841 (proto->tcpflags & TCPFLAG_NSPLICE)) 842 return (0); 843 844 if (cre->splicelen >= 0) 845 return (0); 846 847 /* still not connected */ 848 if (cre->bev == NULL || cre->dst->bev == NULL) 849 return (0); 850 851 if (! (cre->toread == TOREAD_UNLIMITED || cre->toread > 0)) { 852 DPRINTF("%s: session %d: splice dir %d, nothing to read %lld", 853 __func__, con->se_id, cre->dir, cre->toread); 854 return (0); 855 } 856 857 /* do not splice before buffers have not been completely flushed */ 858 if (EVBUFFER_LENGTH(cre->bev->input) || 859 EVBUFFER_LENGTH(cre->dst->bev->output)) { 860 DPRINTF("%s: session %d: splice dir %d, dirty buffer", 861 __func__, con->se_id, cre->dir); 862 bufferevent_disable(cre->bev, EV_READ); 863 return (0); 864 } 865 866 bzero(&sp, sizeof(sp)); 867 sp.sp_fd = cre->dst->s; 868 sp.sp_max = cre->toread > 0 ? cre->toread : 0; 869 bcopy(&rlay->rl_conf.timeout, &sp.sp_idle, sizeof(sp.sp_idle)); 870 if (setsockopt(cre->s, SOL_SOCKET, SO_SPLICE, &sp, sizeof(sp)) == -1) { 871 log_debug("%s: session %d: splice dir %d failed: %s", 872 __func__, con->se_id, cre->dir, strerror(errno)); 873 return (-1); 874 } 875 cre->splicelen = 0; 876 bufferevent_enable(cre->bev, EV_READ); 877 878 DPRINTF("%s: session %d: splice dir %d, maximum %lld, successful", 879 __func__, con->se_id, cre->dir, cre->toread); 880 881 return (0); 882 } 883 884 int 885 relay_splicelen(struct ctl_relay_event *cre) 886 { 887 struct rsession *con = cre->con; 888 off_t len; 889 socklen_t optlen; 890 891 if (cre->splicelen < 0) 892 return (0); 893 894 optlen = sizeof(len); 895 if (getsockopt(cre->s, SOL_SOCKET, SO_SPLICE, &len, &optlen) == -1) { 896 log_debug("%s: session %d: splice dir %d get length failed: %s", 897 __func__, con->se_id, cre->dir, strerror(errno)); 898 return (-1); 899 } 900 901 DPRINTF("%s: session %d: splice dir %d, length %lld", 902 __func__, con->se_id, cre->dir, len); 903 904 if (len > cre->splicelen) { 905 getmonotime(&con->se_tv_last); 906 907 cre->splicelen = len; 908 return (1); 909 } 910 911 return (0); 912 } 913 914 int 915 relay_spliceadjust(struct ctl_relay_event *cre) 916 { 917 if (cre->splicelen < 0) 918 return (0); 919 if (relay_splicelen(cre) == -1) 920 return (-1); 921 if (cre->splicelen > 0 && cre->toread > 0) 922 cre->toread -= cre->splicelen; 923 cre->splicelen = -1; 924 925 return (0); 926 } 927 928 void 929 relay_error(struct bufferevent *bev, short error, void *arg) 930 { 931 struct ctl_relay_event *cre = arg; 932 struct rsession *con = cre->con; 933 struct evbuffer *dst; 934 935 if (error & EVBUFFER_TIMEOUT) { 936 if (cre->splicelen >= 0) { 937 bufferevent_enable(bev, EV_READ); 938 } else if (cre->dst->splicelen >= 0) { 939 switch (relay_splicelen(cre->dst)) { 940 case -1: 941 goto fail; 942 case 0: 943 relay_close(con, "buffer event timeout"); 944 break; 945 case 1: 946 cre->timedout = 1; 947 bufferevent_enable(bev, EV_READ); 948 break; 949 } 950 } else { 951 relay_close(con, "buffer event timeout"); 952 } 953 return; 954 } 955 if (error & EVBUFFER_ERROR && errno == ETIMEDOUT) { 956 if (cre->dst->splicelen >= 0) { 957 switch (relay_splicelen(cre->dst)) { 958 case -1: 959 goto fail; 960 case 0: 961 relay_close(con, "splice timeout"); 962 return; 963 case 1: 964 bufferevent_enable(bev, EV_READ); 965 break; 966 } 967 } else if (cre->dst->timedout) { 968 relay_close(con, "splice timeout"); 969 return; 970 } 971 if (relay_spliceadjust(cre) == -1) 972 goto fail; 973 if (relay_splice(cre) == -1) 974 goto fail; 975 return; 976 } 977 if (error & EVBUFFER_ERROR && errno == EFBIG) { 978 if (relay_spliceadjust(cre) == -1) 979 goto fail; 980 bufferevent_enable(cre->bev, EV_READ); 981 return; 982 } 983 if (error & (EVBUFFER_READ|EVBUFFER_WRITE|EVBUFFER_EOF)) { 984 bufferevent_disable(bev, EV_READ|EV_WRITE); 985 986 con->se_done = 1; 987 if (cre->dst->bev != NULL) { 988 dst = EVBUFFER_OUTPUT(cre->dst->bev); 989 if (EVBUFFER_LENGTH(dst)) 990 return; 991 } else 992 return; 993 994 relay_close(con, "done"); 995 return; 996 } 997 relay_close(con, "buffer event error"); 998 return; 999 fail: 1000 relay_close(con, strerror(errno)); 1001 } 1002 1003 void 1004 relay_accept(int fd, short event, void *arg) 1005 { 1006 struct relay *rlay = arg; 1007 struct rsession *con = NULL; 1008 struct ctl_natlook *cnl = NULL; 1009 socklen_t slen; 1010 struct timeval tv; 1011 struct sockaddr_storage ss; 1012 int s = -1; 1013 1014 event_add(&rlay->rl_ev, NULL); 1015 if ((event & EV_TIMEOUT)) 1016 return; 1017 1018 slen = sizeof(ss); 1019 if ((s = accept_reserve(fd, (struct sockaddr *)&ss, 1020 &slen, FD_RESERVE, &relay_inflight)) == -1) { 1021 /* 1022 * Pause accept if we are out of file descriptors, or 1023 * libevent will haunt us here too. 1024 */ 1025 if (errno == ENFILE || errno == EMFILE) { 1026 struct timeval evtpause = { 1, 0 }; 1027 1028 event_del(&rlay->rl_ev); 1029 evtimer_add(&rlay->rl_evt, &evtpause); 1030 log_debug("%s: deferring connections", __func__); 1031 } 1032 return; 1033 } 1034 if (relay_sessions >= RELAY_MAX_SESSIONS || 1035 rlay->rl_conf.flags & F_DISABLE) 1036 goto err; 1037 1038 if (fcntl(s, F_SETFL, O_NONBLOCK) == -1) 1039 goto err; 1040 1041 if ((con = calloc(1, sizeof(*con))) == NULL) 1042 goto err; 1043 1044 con->se_in.s = s; 1045 con->se_in.ssl = NULL; 1046 con->se_out.s = -1; 1047 con->se_out.ssl = NULL; 1048 con->se_in.dst = &con->se_out; 1049 con->se_out.dst = &con->se_in; 1050 con->se_in.con = con; 1051 con->se_out.con = con; 1052 con->se_in.splicelen = -1; 1053 con->se_out.splicelen = -1; 1054 con->se_in.toread = TOREAD_UNLIMITED; 1055 con->se_out.toread = TOREAD_UNLIMITED; 1056 con->se_relay = rlay; 1057 con->se_id = ++relay_conid; 1058 con->se_relayid = rlay->rl_conf.id; 1059 con->se_pid = getpid(); 1060 con->se_in.dir = RELAY_DIR_REQUEST; 1061 con->se_out.dir = RELAY_DIR_RESPONSE; 1062 con->se_retry = rlay->rl_conf.dstretry; 1063 con->se_bnds = -1; 1064 con->se_out.port = rlay->rl_conf.dstport; 1065 switch (ss.ss_family) { 1066 case AF_INET: 1067 con->se_in.port = ((struct sockaddr_in *)&ss)->sin_port; 1068 break; 1069 case AF_INET6: 1070 con->se_in.port = ((struct sockaddr_in6 *)&ss)->sin6_port; 1071 break; 1072 } 1073 bcopy(&ss, &con->se_in.ss, sizeof(con->se_in.ss)); 1074 1075 getmonotime(&con->se_tv_start); 1076 bcopy(&con->se_tv_start, &con->se_tv_last, sizeof(con->se_tv_last)); 1077 1078 relay_sessions++; 1079 SPLAY_INSERT(session_tree, &rlay->rl_sessions, con); 1080 1081 /* Increment the per-relay session counter */ 1082 rlay->rl_stats[proc_id].last++; 1083 1084 /* Pre-allocate output buffer */ 1085 con->se_out.output = evbuffer_new(); 1086 if (con->se_out.output == NULL) { 1087 relay_close(con, "failed to allocate output buffer"); 1088 return; 1089 } 1090 1091 /* Pre-allocate log buffer */ 1092 con->se_haslog = 0; 1093 con->se_log = evbuffer_new(); 1094 if (con->se_log == NULL) { 1095 relay_close(con, "failed to allocate log buffer"); 1096 return; 1097 } 1098 1099 if (rlay->rl_conf.flags & F_DIVERT) { 1100 slen = sizeof(con->se_out.ss); 1101 if (getsockname(s, (struct sockaddr *)&con->se_out.ss, 1102 &slen) == -1) { 1103 relay_close(con, "peer lookup failed"); 1104 return; 1105 } 1106 con->se_out.port = relay_socket_getport(&con->se_out.ss); 1107 1108 /* Detect loop and fall back to the alternate forward target */ 1109 if (bcmp(&rlay->rl_conf.ss, &con->se_out.ss, 1110 sizeof(con->se_out.ss)) == 0 && 1111 con->se_out.port == rlay->rl_conf.port) 1112 con->se_out.ss.ss_family = AF_UNSPEC; 1113 } else if (rlay->rl_conf.flags & F_NATLOOK) { 1114 if ((cnl = calloc(1, sizeof(*cnl))) == NULL) { 1115 relay_close(con, "failed to allocate nat lookup"); 1116 return; 1117 } 1118 1119 con->se_cnl = cnl; 1120 bzero(cnl, sizeof(*cnl)); 1121 cnl->in = -1; 1122 cnl->id = con->se_id; 1123 cnl->proc = proc_id; 1124 cnl->proto = IPPROTO_TCP; 1125 1126 bcopy(&con->se_in.ss, &cnl->src, sizeof(cnl->src)); 1127 slen = sizeof(cnl->dst); 1128 if (getsockname(s, 1129 (struct sockaddr *)&cnl->dst, &slen) == -1) { 1130 relay_close(con, "failed to get local address"); 1131 return; 1132 } 1133 1134 proc_compose_imsg(env->sc_ps, PROC_PFE, -1, IMSG_NATLOOK, -1, 1135 cnl, sizeof(*cnl)); 1136 1137 /* Schedule timeout */ 1138 evtimer_set(&con->se_ev, relay_natlook, con); 1139 bcopy(&rlay->rl_conf.timeout, &tv, sizeof(tv)); 1140 evtimer_add(&con->se_ev, &tv); 1141 return; 1142 } 1143 1144 if (rlay->rl_conf.flags & F_SSLINSPECT) { 1145 relay_preconnect(con); 1146 return; 1147 } 1148 1149 relay_session(con); 1150 return; 1151 err: 1152 if (s != -1) { 1153 close(s); 1154 if (con != NULL) 1155 free(con); 1156 /* 1157 * the session struct was not completly set up, but still 1158 * counted as an inflight session. account for this. 1159 */ 1160 relay_inflight--; 1161 log_debug("%s: inflight decremented, now %d", 1162 __func__, relay_inflight); 1163 } 1164 } 1165 1166 u_int32_t 1167 relay_hash_addr(struct sockaddr_storage *ss, u_int32_t p) 1168 { 1169 struct sockaddr_in *sin4; 1170 struct sockaddr_in6 *sin6; 1171 1172 if (ss->ss_family == AF_INET) { 1173 sin4 = (struct sockaddr_in *)ss; 1174 p = hash32_buf(&sin4->sin_addr, 1175 sizeof(struct in_addr), p); 1176 } else { 1177 sin6 = (struct sockaddr_in6 *)ss; 1178 p = hash32_buf(&sin6->sin6_addr, 1179 sizeof(struct in6_addr), p); 1180 } 1181 1182 return (p); 1183 } 1184 1185 int 1186 relay_from_table(struct rsession *con) 1187 { 1188 struct relay *rlay = con->se_relay; 1189 struct host *host; 1190 struct relay_table *rlt = NULL; 1191 struct table *table = NULL; 1192 u_int32_t p = con->se_hashkey; 1193 int idx = -1; 1194 1195 /* the table is already selected */ 1196 if (con->se_table != NULL) { 1197 rlt = con->se_table; 1198 table = rlt->rlt_table; 1199 if (table->conf.check && !table->up) 1200 table = NULL; 1201 goto gottable; 1202 } 1203 1204 /* otherwise grep the first active table */ 1205 TAILQ_FOREACH(rlt, &rlay->rl_tables, rlt_entry) { 1206 table = rlt->rlt_table; 1207 if ((rlt->rlt_flags & F_USED) == 0 || 1208 (table->conf.check && !table->up)) 1209 table = NULL; 1210 else 1211 break; 1212 } 1213 1214 gottable: 1215 if (table == NULL) { 1216 log_debug("%s: session %d: no active hosts", 1217 __func__, con->se_id); 1218 return (-1); 1219 } 1220 if (!con->se_hashkeyset) { 1221 p = con->se_hashkey = rlt->rlt_key; 1222 con->se_hashkeyset = 1; 1223 } 1224 1225 switch (rlt->rlt_mode) { 1226 case RELAY_DSTMODE_ROUNDROBIN: 1227 if ((int)rlt->rlt_key >= rlt->rlt_nhosts) 1228 rlt->rlt_key = 0; 1229 idx = (int)rlt->rlt_key; 1230 break; 1231 case RELAY_DSTMODE_RANDOM: 1232 idx = (int)arc4random_uniform(rlt->rlt_nhosts); 1233 break; 1234 case RELAY_DSTMODE_SRCHASH: 1235 case RELAY_DSTMODE_LOADBALANCE: 1236 /* Source IP address without port */ 1237 p = relay_hash_addr(&con->se_in.ss, p); 1238 if (rlt->rlt_mode == RELAY_DSTMODE_SRCHASH) 1239 break; 1240 /* FALLTHROUGH */ 1241 case RELAY_DSTMODE_HASH: 1242 /* Local "destination" IP address and port */ 1243 p = relay_hash_addr(&rlay->rl_conf.ss, p); 1244 p = hash32_buf(&rlay->rl_conf.port, 1245 sizeof(rlay->rl_conf.port), p); 1246 break; 1247 default: 1248 fatalx("relay_from_table: unsupported mode"); 1249 /* NOTREACHED */ 1250 } 1251 if (idx == -1 && (idx = p % rlt->rlt_nhosts) >= RELAY_MAXHOSTS) 1252 return (-1); 1253 host = rlt->rlt_host[idx]; 1254 DPRINTF("%s: session %d: table %s host %s, p 0x%08x, idx %d", 1255 __func__, con->se_id, table->conf.name, host->conf.name, p, idx); 1256 while (host != NULL) { 1257 DPRINTF("%s: session %d: host %s", __func__, 1258 con->se_id, host->conf.name); 1259 if (!table->conf.check || host->up == HOST_UP) 1260 goto found; 1261 host = TAILQ_NEXT(host, entry); 1262 } 1263 TAILQ_FOREACH(host, &table->hosts, entry) { 1264 DPRINTF("%s: session %d: next host %s", 1265 __func__, con->se_id, host->conf.name); 1266 if (!table->conf.check || host->up == HOST_UP) 1267 goto found; 1268 } 1269 1270 /* Should not happen */ 1271 fatalx("relay_from_table: no active hosts, desynchronized"); 1272 1273 found: 1274 if (rlt->rlt_mode == RELAY_DSTMODE_ROUNDROBIN) 1275 rlt->rlt_key = host->idx + 1; 1276 con->se_retry = host->conf.retry; 1277 con->se_out.port = table->conf.port; 1278 bcopy(&host->conf.ss, &con->se_out.ss, sizeof(con->se_out.ss)); 1279 1280 return (0); 1281 } 1282 1283 void 1284 relay_natlook(int fd, short event, void *arg) 1285 { 1286 struct rsession *con = arg; 1287 struct relay *rlay = con->se_relay; 1288 struct ctl_natlook *cnl = con->se_cnl; 1289 1290 if (cnl == NULL) 1291 fatalx("invalid NAT lookup"); 1292 1293 if (con->se_out.ss.ss_family == AF_UNSPEC && cnl->in == -1 && 1294 rlay->rl_conf.dstss.ss_family == AF_UNSPEC && 1295 TAILQ_EMPTY(&rlay->rl_tables)) { 1296 relay_close(con, "session NAT lookup failed"); 1297 return; 1298 } 1299 if (cnl->in != -1) { 1300 bcopy(&cnl->rdst, &con->se_out.ss, sizeof(con->se_out.ss)); 1301 con->se_out.port = cnl->rdport; 1302 } 1303 free(con->se_cnl); 1304 con->se_cnl = NULL; 1305 1306 relay_session(con); 1307 } 1308 1309 void 1310 relay_session(struct rsession *con) 1311 { 1312 struct relay *rlay = con->se_relay; 1313 struct ctl_relay_event *in = &con->se_in, *out = &con->se_out; 1314 1315 if (bcmp(&rlay->rl_conf.ss, &out->ss, sizeof(out->ss)) == 0 && 1316 out->port == rlay->rl_conf.port) { 1317 log_debug("%s: session %d: looping", __func__, con->se_id); 1318 relay_close(con, "session aborted"); 1319 return; 1320 } 1321 1322 if (rlay->rl_conf.flags & F_UDP) { 1323 /* 1324 * Call the UDP protocol-specific handler 1325 */ 1326 if (rlay->rl_proto->request == NULL) 1327 fatalx("invalide UDP session"); 1328 if ((*rlay->rl_proto->request)(con) == -1) 1329 relay_close(con, "session failed"); 1330 return; 1331 } 1332 1333 if ((rlay->rl_conf.flags & F_SSL) && (in->ssl == NULL)) { 1334 relay_ssl_transaction(con, in); 1335 return; 1336 } 1337 1338 if (rlay->rl_proto->type != RELAY_PROTO_HTTP) { 1339 if (rlay->rl_conf.fwdmode == FWD_TRANS) 1340 relay_bindanyreq(con, 0, IPPROTO_TCP); 1341 else if (relay_connect(con) == -1) { 1342 relay_close(con, "session failed"); 1343 return; 1344 } 1345 } 1346 1347 relay_input(con); 1348 } 1349 1350 void 1351 relay_bindanyreq(struct rsession *con, in_port_t port, int proto) 1352 { 1353 struct relay *rlay = con->se_relay; 1354 struct ctl_bindany bnd; 1355 struct timeval tv; 1356 1357 bzero(&bnd, sizeof(bnd)); 1358 bnd.bnd_id = con->se_id; 1359 bnd.bnd_proc = proc_id; 1360 bnd.bnd_port = port; 1361 bnd.bnd_proto = proto; 1362 bcopy(&con->se_in.ss, &bnd.bnd_ss, sizeof(bnd.bnd_ss)); 1363 proc_compose_imsg(env->sc_ps, PROC_PARENT, -1, IMSG_BINDANY, 1364 -1, &bnd, sizeof(bnd)); 1365 1366 /* Schedule timeout */ 1367 evtimer_set(&con->se_ev, relay_bindany, con); 1368 bcopy(&rlay->rl_conf.timeout, &tv, sizeof(tv)); 1369 evtimer_add(&con->se_ev, &tv); 1370 } 1371 1372 void 1373 relay_bindany(int fd, short event, void *arg) 1374 { 1375 struct rsession *con = arg; 1376 1377 if (con->se_bnds == -1) { 1378 relay_close(con, "bindany failed, invalid socket"); 1379 return; 1380 } 1381 if (relay_connect(con) == -1) 1382 relay_close(con, "session failed"); 1383 } 1384 1385 void 1386 relay_connect_retry(int fd, short sig, void *arg) 1387 { 1388 struct timeval evtpause = { 1, 0 }; 1389 struct rsession *con = arg; 1390 struct relay *rlay = con->se_relay; 1391 int bnds = -1; 1392 1393 if (relay_inflight < 1) 1394 fatalx("relay_connect_retry: no connection in flight"); 1395 1396 DPRINTF("%s: retry %d of %d, inflight: %d",__func__, 1397 con->se_retrycount, con->se_retry, relay_inflight); 1398 1399 if (sig != EV_TIMEOUT) 1400 fatalx("relay_connect_retry: called without timeout"); 1401 1402 evtimer_del(&con->se_inflightevt); 1403 1404 /* 1405 * XXX we might want to check if the inbound socket is still 1406 * available: client could have closed it while we were waiting? 1407 */ 1408 1409 DPRINTF("%s: got EV_TIMEOUT", __func__); 1410 1411 if (getdtablecount() + FD_RESERVE + 1412 relay_inflight > getdtablesize()) { 1413 if (con->se_retrycount < RELAY_OUTOF_FD_RETRIES) { 1414 evtimer_add(&con->se_inflightevt, &evtpause); 1415 return; 1416 } 1417 /* we waited for RELAY_OUTOF_FD_RETRIES seconds, give up */ 1418 event_add(&rlay->rl_ev, NULL); 1419 relay_abort_http(con, 504, "connection timed out", 0); 1420 return; 1421 } 1422 1423 if (rlay->rl_conf.fwdmode == FWD_TRANS) { 1424 /* con->se_bnds cannot be unset */ 1425 bnds = con->se_bnds; 1426 } 1427 1428 retry: 1429 if ((con->se_out.s = relay_socket_connect(&con->se_out.ss, 1430 con->se_out.port, rlay->rl_proto, bnds)) == -1) { 1431 log_debug("%s: session %d: " 1432 "forward failed: %s, %s", __func__, 1433 con->se_id, strerror(errno), 1434 con->se_retry ? "next retry" : "last retry"); 1435 1436 con->se_retrycount++; 1437 1438 if ((errno == ENFILE || errno == EMFILE) && 1439 (con->se_retrycount < con->se_retry)) { 1440 event_del(&rlay->rl_ev); 1441 evtimer_add(&con->se_inflightevt, &evtpause); 1442 evtimer_add(&rlay->rl_evt, &evtpause); 1443 return; 1444 } else if (con->se_retrycount < con->se_retry) 1445 goto retry; 1446 event_add(&rlay->rl_ev, NULL); 1447 relay_abort_http(con, 504, "connect failed", 0); 1448 return; 1449 } 1450 1451 relay_inflight--; 1452 DPRINTF("%s: inflight decremented, now %d",__func__, relay_inflight); 1453 1454 event_add(&rlay->rl_ev, NULL); 1455 1456 if (errno == EINPROGRESS) 1457 event_again(&con->se_ev, con->se_out.s, EV_WRITE|EV_TIMEOUT, 1458 relay_connected, &con->se_tv_start, &rlay->rl_conf.timeout, 1459 con); 1460 else 1461 relay_connected(con->se_out.s, EV_WRITE, con); 1462 1463 return; 1464 } 1465 1466 int 1467 relay_preconnect(struct rsession *con) 1468 { 1469 log_debug("%s: session %d: process %d", __func__, 1470 con->se_id, privsep_process); 1471 return (relay_connect(con)); 1472 } 1473 1474 int 1475 relay_connect(struct rsession *con) 1476 { 1477 struct relay *rlay = con->se_relay; 1478 struct timeval evtpause = { 1, 0 }; 1479 int bnds = -1, ret; 1480 1481 /* Connection is already established but session not active */ 1482 if ((rlay->rl_conf.flags & F_SSLINSPECT) && con->se_out.s != -1) { 1483 if (con->se_out.ssl == NULL) { 1484 log_debug("%s: ssl connect failed", __func__); 1485 return (-1); 1486 } 1487 relay_connected(con->se_out.s, EV_WRITE, con); 1488 return (0); 1489 } 1490 1491 if (relay_inflight < 1) 1492 fatalx("relay_connect: no connection in flight"); 1493 1494 getmonotime(&con->se_tv_start); 1495 1496 if (!TAILQ_EMPTY(&rlay->rl_tables)) { 1497 if (relay_from_table(con) != 0) 1498 return (-1); 1499 } else if (con->se_out.ss.ss_family == AF_UNSPEC) { 1500 bcopy(&rlay->rl_conf.dstss, &con->se_out.ss, 1501 sizeof(con->se_out.ss)); 1502 con->se_out.port = rlay->rl_conf.dstport; 1503 } 1504 1505 if (rlay->rl_conf.fwdmode == FWD_TRANS) { 1506 if (con->se_bnds == -1) { 1507 log_debug("%s: could not bind any sock", __func__); 1508 return (-1); 1509 } 1510 bnds = con->se_bnds; 1511 } 1512 1513 /* Do the IPv4-to-IPv6 or IPv6-to-IPv4 translation if requested */ 1514 if (rlay->rl_conf.dstaf.ss_family != AF_UNSPEC) { 1515 if (con->se_out.ss.ss_family == AF_INET && 1516 rlay->rl_conf.dstaf.ss_family == AF_INET6) 1517 ret = map4to6(&con->se_out.ss, &rlay->rl_conf.dstaf); 1518 else if (con->se_out.ss.ss_family == AF_INET6 && 1519 rlay->rl_conf.dstaf.ss_family == AF_INET) 1520 ret = map6to4(&con->se_out.ss); 1521 else 1522 ret = 0; 1523 if (ret != 0) { 1524 log_debug("%s: mapped to invalid address", __func__); 1525 return (-1); 1526 } 1527 } 1528 1529 retry: 1530 if ((con->se_out.s = relay_socket_connect(&con->se_out.ss, 1531 con->se_out.port, rlay->rl_proto, bnds)) == -1) { 1532 if (errno == ENFILE || errno == EMFILE) { 1533 log_debug("%s: session %d: forward failed: %s", 1534 __func__, con->se_id, strerror(errno)); 1535 evtimer_set(&con->se_inflightevt, relay_connect_retry, 1536 con); 1537 event_del(&rlay->rl_ev); 1538 evtimer_add(&con->se_inflightevt, &evtpause); 1539 evtimer_add(&rlay->rl_evt, &evtpause); 1540 return (0); 1541 } else { 1542 if (con->se_retry) { 1543 con->se_retry--; 1544 log_debug("%s: session %d: " 1545 "forward failed: %s, %s", __func__, 1546 con->se_id, strerror(errno), 1547 con->se_retry ? 1548 "next retry" : "last retry"); 1549 goto retry; 1550 } 1551 log_debug("%s: session %d: forward failed: %s", 1552 __func__, con->se_id, strerror(errno)); 1553 return (-1); 1554 } 1555 } 1556 1557 relay_inflight--; 1558 DPRINTF("%s: inflight decremented, now %d",__func__, 1559 relay_inflight); 1560 1561 if (errno == EINPROGRESS) 1562 event_again(&con->se_ev, con->se_out.s, EV_WRITE|EV_TIMEOUT, 1563 relay_connected, &con->se_tv_start, &rlay->rl_conf.timeout, 1564 con); 1565 else 1566 relay_connected(con->se_out.s, EV_WRITE, con); 1567 1568 return (0); 1569 } 1570 1571 void 1572 relay_close(struct rsession *con, const char *msg) 1573 { 1574 char ibuf[128], obuf[128], *ptr = NULL; 1575 struct relay *rlay = con->se_relay; 1576 struct protocol *proto = rlay->rl_proto; 1577 1578 SPLAY_REMOVE(session_tree, &rlay->rl_sessions, con); 1579 1580 event_del(&con->se_ev); 1581 if (con->se_in.bev != NULL) 1582 bufferevent_disable(con->se_in.bev, EV_READ|EV_WRITE); 1583 if (con->se_out.bev != NULL) 1584 bufferevent_disable(con->se_out.bev, EV_READ|EV_WRITE); 1585 1586 if ((env->sc_opts & RELAYD_OPT_LOGUPDATE) && msg != NULL) { 1587 bzero(&ibuf, sizeof(ibuf)); 1588 bzero(&obuf, sizeof(obuf)); 1589 (void)print_host(&con->se_in.ss, ibuf, sizeof(ibuf)); 1590 (void)print_host(&con->se_out.ss, obuf, sizeof(obuf)); 1591 if (EVBUFFER_LENGTH(con->se_log) && 1592 evbuffer_add_printf(con->se_log, "\r\n") != -1) 1593 ptr = evbuffer_readline(con->se_log); 1594 log_info("relay %s, " 1595 "session %d (%d active), %s, %s -> %s:%d, " 1596 "%s%s%s", rlay->rl_conf.name, con->se_id, relay_sessions, 1597 con->se_tag != 0 ? tag_id2name(con->se_tag) : "0", ibuf, 1598 obuf, ntohs(con->se_out.port), msg, ptr == NULL ? "" : ",", 1599 ptr == NULL ? "" : ptr); 1600 if (ptr != NULL) 1601 free(ptr); 1602 } 1603 1604 if (proto->close != NULL) 1605 (*proto->close)(con); 1606 1607 if (con->se_priv != NULL) 1608 free(con->se_priv); 1609 if (con->se_in.bev != NULL) 1610 bufferevent_free(con->se_in.bev); 1611 else if (con->se_in.output != NULL) 1612 evbuffer_free(con->se_in.output); 1613 if (con->se_in.ssl != NULL) { 1614 /* XXX handle non-blocking shutdown */ 1615 if (SSL_shutdown(con->se_in.ssl) == 0) 1616 SSL_shutdown(con->se_in.ssl); 1617 SSL_free(con->se_in.ssl); 1618 } 1619 if (con->se_in.sslcert != NULL) 1620 X509_free(con->se_in.sslcert); 1621 if (con->se_in.s != -1) { 1622 close(con->se_in.s); 1623 if (con->se_out.s == -1) { 1624 /* 1625 * the output was never connected, 1626 * thus this was an inflight session. 1627 */ 1628 relay_inflight--; 1629 log_debug("%s: sessions inflight decremented, now %d", 1630 __func__, relay_inflight); 1631 } 1632 } 1633 if (con->se_in.buf != NULL) 1634 free(con->se_in.buf); 1635 1636 if (con->se_out.bev != NULL) 1637 bufferevent_free(con->se_out.bev); 1638 else if (con->se_out.output != NULL) 1639 evbuffer_free(con->se_out.output); 1640 if (con->se_out.ssl != NULL) { 1641 /* XXX handle non-blocking shutdown */ 1642 if (SSL_shutdown(con->se_out.ssl) == 0) 1643 SSL_shutdown(con->se_out.ssl); 1644 SSL_free(con->se_out.ssl); 1645 } 1646 if (con->se_out.sslcert != NULL) 1647 X509_free(con->se_out.sslcert); 1648 if (con->se_out.s != -1) { 1649 close(con->se_out.s); 1650 1651 /* Some file descriptors are available again. */ 1652 if (evtimer_pending(&rlay->rl_evt, NULL)) { 1653 evtimer_del(&rlay->rl_evt); 1654 event_add(&rlay->rl_ev, NULL); 1655 } 1656 } 1657 1658 if (con->se_out.buf != NULL) 1659 free(con->se_out.buf); 1660 1661 if (con->se_log != NULL) 1662 evbuffer_free(con->se_log); 1663 1664 if (con->se_cnl != NULL) { 1665 #if 0 1666 proc_compose_imsg(env->sc_ps, PROC_PFE, -1, IMSG_KILLSTATES, -1, 1667 cnl, sizeof(*cnl)); 1668 #endif 1669 free(con->se_cnl); 1670 } 1671 1672 free(con); 1673 relay_sessions--; 1674 } 1675 1676 int 1677 relay_dispatch_pfe(int fd, struct privsep_proc *p, struct imsg *imsg) 1678 { 1679 struct relay *rlay; 1680 struct rsession *con, se; 1681 struct ctl_natlook cnl; 1682 struct timeval tv; 1683 struct host *host; 1684 struct table *table; 1685 struct ctl_status st; 1686 objid_t id; 1687 int cid; 1688 1689 switch (imsg->hdr.type) { 1690 case IMSG_HOST_DISABLE: 1691 memcpy(&id, imsg->data, sizeof(id)); 1692 if ((host = host_find(env, id)) == NULL) 1693 fatalx("relay_dispatch_pfe: desynchronized"); 1694 if ((table = table_find(env, host->conf.tableid)) == 1695 NULL) 1696 fatalx("relay_dispatch_pfe: invalid table id"); 1697 if (host->up == HOST_UP) 1698 table->up--; 1699 host->flags |= F_DISABLE; 1700 host->up = HOST_UNKNOWN; 1701 break; 1702 case IMSG_HOST_ENABLE: 1703 memcpy(&id, imsg->data, sizeof(id)); 1704 if ((host = host_find(env, id)) == NULL) 1705 fatalx("relay_dispatch_pfe: desynchronized"); 1706 host->flags &= ~(F_DISABLE); 1707 host->up = HOST_UNKNOWN; 1708 break; 1709 case IMSG_TABLE_DISABLE: 1710 memcpy(&id, imsg->data, sizeof(id)); 1711 if ((table = table_find(env, id)) == NULL) 1712 fatalx("relay_dispatch_pfe: desynchronized"); 1713 table->conf.flags |= F_DISABLE; 1714 table->up = 0; 1715 TAILQ_FOREACH(host, &table->hosts, entry) 1716 host->up = HOST_UNKNOWN; 1717 break; 1718 case IMSG_TABLE_ENABLE: 1719 memcpy(&id, imsg->data, sizeof(id)); 1720 if ((table = table_find(env, id)) == NULL) 1721 fatalx("relay_dispatch_pfe: desynchronized"); 1722 table->conf.flags &= ~(F_DISABLE); 1723 table->up = 0; 1724 TAILQ_FOREACH(host, &table->hosts, entry) 1725 host->up = HOST_UNKNOWN; 1726 break; 1727 case IMSG_HOST_STATUS: 1728 IMSG_SIZE_CHECK(imsg, &st); 1729 memcpy(&st, imsg->data, sizeof(st)); 1730 if ((host = host_find(env, st.id)) == NULL) 1731 fatalx("relay_dispatch_pfe: invalid host id"); 1732 if (host->flags & F_DISABLE) 1733 break; 1734 if (host->up == st.up) { 1735 log_debug("%s: host %d => %d", __func__, 1736 host->conf.id, host->up); 1737 fatalx("relay_dispatch_pfe: desynchronized"); 1738 } 1739 1740 if ((table = table_find(env, host->conf.tableid)) 1741 == NULL) 1742 fatalx("relay_dispatch_pfe: invalid table id"); 1743 1744 DPRINTF("%s: [%d] state %d for " 1745 "host %u %s", __func__, proc_id, st.up, 1746 host->conf.id, host->conf.name); 1747 1748 if ((st.up == HOST_UNKNOWN && host->up == HOST_DOWN) || 1749 (st.up == HOST_DOWN && host->up == HOST_UNKNOWN)) { 1750 host->up = st.up; 1751 break; 1752 } 1753 if (st.up == HOST_UP) 1754 table->up++; 1755 else 1756 table->up--; 1757 host->up = st.up; 1758 break; 1759 case IMSG_NATLOOK: 1760 bcopy(imsg->data, &cnl, sizeof(cnl)); 1761 if ((con = session_find(env, cnl.id)) == NULL || 1762 con->se_cnl == NULL) { 1763 log_debug("%s: session %d: expired", 1764 __func__, cnl.id); 1765 break; 1766 } 1767 bcopy(&cnl, con->se_cnl, sizeof(*con->se_cnl)); 1768 evtimer_del(&con->se_ev); 1769 evtimer_set(&con->se_ev, relay_natlook, con); 1770 bzero(&tv, sizeof(tv)); 1771 evtimer_add(&con->se_ev, &tv); 1772 break; 1773 case IMSG_CTL_SESSION: 1774 IMSG_SIZE_CHECK(imsg, &cid); 1775 memcpy(&cid, imsg->data, sizeof(cid)); 1776 TAILQ_FOREACH(rlay, env->sc_relays, rl_entry) { 1777 SPLAY_FOREACH(con, session_tree, 1778 &rlay->rl_sessions) { 1779 memcpy(&se, con, sizeof(se)); 1780 se.se_cid = cid; 1781 proc_compose_imsg(env->sc_ps, p->p_id, -1, 1782 IMSG_CTL_SESSION, 1783 -1, &se, sizeof(se)); 1784 } 1785 } 1786 proc_compose_imsg(env->sc_ps, p->p_id, -1, IMSG_CTL_END, 1787 -1, &cid, sizeof(cid)); 1788 break; 1789 default: 1790 return (-1); 1791 } 1792 1793 return (0); 1794 } 1795 1796 int 1797 relay_dispatch_ca(int fd, struct privsep_proc *p, struct imsg *imsg) 1798 { 1799 return (-1); 1800 } 1801 1802 int 1803 relay_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg) 1804 { 1805 struct rsession *con; 1806 struct timeval tv; 1807 objid_t id; 1808 1809 switch (imsg->hdr.type) { 1810 case IMSG_BINDANY: 1811 bcopy(imsg->data, &id, sizeof(id)); 1812 if ((con = session_find(env, id)) == NULL) { 1813 log_debug("%s: session %d: expired", 1814 __func__, id); 1815 break; 1816 } 1817 1818 /* Will validate the result later */ 1819 con->se_bnds = imsg->fd; 1820 1821 evtimer_del(&con->se_ev); 1822 evtimer_set(&con->se_ev, relay_bindany, con); 1823 bzero(&tv, sizeof(tv)); 1824 evtimer_add(&con->se_ev, &tv); 1825 break; 1826 case IMSG_CFG_TABLE: 1827 config_gettable(env, imsg); 1828 break; 1829 case IMSG_CFG_HOST: 1830 config_gethost(env, imsg); 1831 break; 1832 case IMSG_CFG_PROTO: 1833 config_getproto(env, imsg); 1834 break; 1835 case IMSG_CFG_RULE: 1836 config_getrule(env, imsg); 1837 break; 1838 case IMSG_CFG_RELAY: 1839 config_getrelay(env, imsg); 1840 break; 1841 case IMSG_CFG_RELAY_TABLE: 1842 config_getrelaytable(env, imsg); 1843 break; 1844 case IMSG_CFG_DONE: 1845 config_getcfg(env, imsg); 1846 break; 1847 case IMSG_CTL_START: 1848 relay_launch(); 1849 break; 1850 case IMSG_CTL_RESET: 1851 config_getreset(env, imsg); 1852 break; 1853 default: 1854 return (-1); 1855 } 1856 1857 return (0); 1858 } 1859 1860 void 1861 relay_ssl_callback_info(const SSL *ssl, int where, int rc) 1862 { 1863 struct ctl_relay_event *cre; 1864 int ssl_state; 1865 1866 cre = (struct ctl_relay_event *)SSL_get_app_data(ssl); 1867 1868 if (cre == NULL || cre->sslreneg_state == SSLRENEG_ALLOW) 1869 return; 1870 1871 ssl_state = SSL_get_state(ssl); 1872 1873 /* Check renegotiations */ 1874 if ((where & SSL_CB_ACCEPT_LOOP) && 1875 (cre->sslreneg_state == SSLRENEG_DENY)) { 1876 if ((ssl_state == SSL3_ST_SR_CLNT_HELLO_A) || 1877 (ssl_state == SSL23_ST_SR_CLNT_HELLO_A)) { 1878 /* 1879 * This is a client initiated renegotiation 1880 * that we do not allow 1881 */ 1882 cre->sslreneg_state = SSLRENEG_ABORT; 1883 } 1884 } else if ((where & SSL_CB_HANDSHAKE_DONE) && 1885 (cre->sslreneg_state == SSLRENEG_INIT)) { 1886 /* 1887 * This is right after the first handshake, 1888 * disallow any further negotiations. 1889 */ 1890 cre->sslreneg_state = SSLRENEG_DENY; 1891 } 1892 } 1893 1894 DH * 1895 relay_ssl_get_dhparams(int keylen) 1896 { 1897 DH *dh; 1898 BIGNUM *(*prime)(BIGNUM *); 1899 const char *gen; 1900 1901 gen = "2"; 1902 if (keylen >= 8192) 1903 prime = get_rfc3526_prime_8192; 1904 else if (keylen >= 4096) 1905 prime = get_rfc3526_prime_4096; 1906 else if (keylen >= 3072) 1907 prime = get_rfc3526_prime_3072; 1908 else if (keylen >= 2048) 1909 prime = get_rfc3526_prime_2048; 1910 else if (keylen >= 1536) 1911 prime = get_rfc3526_prime_1536; 1912 else 1913 prime = get_rfc2409_prime_1024; 1914 1915 if ((dh = DH_new()) == NULL) 1916 return (NULL); 1917 1918 dh->p = (*prime)(NULL); 1919 BN_dec2bn(&dh->g, gen); 1920 1921 if (dh->p == NULL || dh->g == NULL) { 1922 DH_free(dh); 1923 return (NULL); 1924 } 1925 1926 return (dh); 1927 } 1928 1929 DH * 1930 relay_ssl_callback_dh(SSL *ssl, int export, int keylen) 1931 { 1932 struct ctl_relay_event *cre; 1933 EVP_PKEY *pkey; 1934 int keytype, maxlen; 1935 DH *dh = NULL; 1936 1937 /* Get maximum key length from config */ 1938 if ((cre = (struct ctl_relay_event *)SSL_get_app_data(ssl)) == NULL) 1939 return (NULL); 1940 maxlen = cre->con->se_relay->rl_proto->ssldhparams; 1941 1942 /* Get the private key length from the cert */ 1943 if ((pkey = SSL_get_privatekey(ssl))) { 1944 keytype = EVP_PKEY_type(pkey->type); 1945 if (keytype == EVP_PKEY_RSA || keytype == EVP_PKEY_DSA) 1946 keylen = EVP_PKEY_bits(pkey); 1947 else 1948 return (NULL); 1949 } 1950 1951 /* get built-in params based on the shorter key length */ 1952 dh = relay_ssl_get_dhparams(MIN(keylen, maxlen)); 1953 1954 return (dh); 1955 } 1956 1957 SSL_CTX * 1958 relay_ssl_ctx_create(struct relay *rlay) 1959 { 1960 struct protocol *proto = rlay->rl_proto; 1961 SSL_CTX *ctx; 1962 EC_KEY *ecdhkey; 1963 1964 ctx = SSL_CTX_new(SSLv23_method()); 1965 if (ctx == NULL) 1966 goto err; 1967 1968 /* Modify session timeout and cache size*/ 1969 SSL_CTX_set_timeout(ctx, 1970 (long)MIN(rlay->rl_conf.timeout.tv_sec, LONG_MAX)); 1971 if (proto->cache < -1) { 1972 SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_OFF); 1973 } else if (proto->cache >= -1) { 1974 SSL_CTX_set_session_cache_mode(ctx, SSL_SESS_CACHE_SERVER); 1975 if (proto->cache >= 0) 1976 SSL_CTX_sess_set_cache_size(ctx, proto->cache); 1977 } 1978 1979 /* Enable all workarounds and set SSL options */ 1980 SSL_CTX_set_options(ctx, SSL_OP_ALL); 1981 SSL_CTX_set_options(ctx, 1982 SSL_OP_NO_SESSION_RESUMPTION_ON_RENEGOTIATION); 1983 if (proto->sslflags & SSLFLAG_CIPHER_SERVER_PREF) 1984 SSL_CTX_set_options(ctx, SSL_OP_CIPHER_SERVER_PREFERENCE); 1985 1986 /* Set the allowed SSL protocols */ 1987 if ((proto->sslflags & SSLFLAG_SSLV2) == 0) 1988 SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv2); 1989 if ((proto->sslflags & SSLFLAG_SSLV3) == 0) 1990 SSL_CTX_set_options(ctx, SSL_OP_NO_SSLv3); 1991 if ((proto->sslflags & SSLFLAG_TLSV1) == 0) 1992 SSL_CTX_set_options(ctx, SSL_OP_NO_TLSv1); 1993 1994 /* add the SSL info callback */ 1995 SSL_CTX_set_info_callback(ctx, relay_ssl_callback_info); 1996 1997 if (proto->sslecdhcurve > 0) { 1998 /* Enable ECDHE support for TLS perfect forward secrecy */ 1999 if ((ecdhkey = 2000 EC_KEY_new_by_curve_name(proto->sslecdhcurve)) == NULL) 2001 goto err; 2002 SSL_CTX_set_tmp_ecdh(ctx, ecdhkey); 2003 SSL_CTX_set_options(ctx, SSL_OP_SINGLE_ECDH_USE); 2004 EC_KEY_free(ecdhkey); 2005 } 2006 2007 if (proto->ssldhparams > 0) { 2008 /* Enable EDH params (forward secrecy for older clients) */ 2009 SSL_CTX_set_tmp_dh_callback(ctx, relay_ssl_callback_dh); 2010 } 2011 2012 if (!SSL_CTX_set_cipher_list(ctx, proto->sslciphers)) 2013 goto err; 2014 2015 /* Verify the server certificate if we have a CA chain */ 2016 if ((rlay->rl_conf.flags & F_SSLCLIENT) && 2017 (rlay->rl_ssl_ca != NULL)) { 2018 if (!ssl_ctx_load_verify_memory(ctx, 2019 rlay->rl_ssl_ca, rlay->rl_conf.ssl_ca_len)) 2020 goto err; 2021 SSL_CTX_set_verify(ctx, SSL_VERIFY_PEER, NULL); 2022 } 2023 2024 if ((rlay->rl_conf.flags & F_SSL) == 0) 2025 return (ctx); 2026 2027 log_debug("%s: loading certificate", __func__); 2028 if (!ssl_ctx_use_certificate_chain(ctx, 2029 rlay->rl_ssl_cert, rlay->rl_conf.ssl_cert_len)) 2030 goto err; 2031 2032 log_debug("%s: loading private key", __func__); 2033 if (!ssl_ctx_fake_private_key(ctx, 2034 &rlay->rl_conf.ssl_keyid, sizeof(rlay->rl_conf.ssl_keyid), 2035 rlay->rl_ssl_cert, rlay->rl_conf.ssl_cert_len, 2036 &rlay->rl_ssl_x509, &rlay->rl_ssl_pkey)) 2037 goto err; 2038 2039 if (!SSL_CTX_check_private_key(ctx)) 2040 goto err; 2041 2042 if (rlay->rl_conf.ssl_cacert_len) { 2043 log_debug("%s: loading CA private key", __func__); 2044 if (!ssl_load_pkey(&rlay->rl_conf.ssl_cakeyid, 2045 sizeof(rlay->rl_conf.ssl_cakeyid), 2046 rlay->rl_ssl_cacert, rlay->rl_conf.ssl_cacert_len, 2047 &rlay->rl_ssl_cacertx509, &rlay->rl_ssl_capkey)) 2048 goto err; 2049 } 2050 2051 /* Set session context to the local relay name */ 2052 if (!SSL_CTX_set_session_id_context(ctx, rlay->rl_conf.name, 2053 strlen(rlay->rl_conf.name))) 2054 goto err; 2055 2056 /* The text versions of the keys/certs are not needed anymore */ 2057 purge_key(&rlay->rl_ssl_cert, rlay->rl_conf.ssl_cert_len); 2058 purge_key(&rlay->rl_ssl_cacert, rlay->rl_conf.ssl_cacert_len); 2059 2060 return (ctx); 2061 2062 err: 2063 if (ctx != NULL) 2064 SSL_CTX_free(ctx); 2065 ssl_error(rlay->rl_conf.name, "relay_ssl_ctx_create"); 2066 return (NULL); 2067 } 2068 2069 void 2070 relay_ssl_transaction(struct rsession *con, struct ctl_relay_event *cre) 2071 { 2072 struct relay *rlay = con->se_relay; 2073 struct protocol *proto = rlay->rl_proto; 2074 SSL *ssl; 2075 const SSL_METHOD *method; 2076 void (*cb)(int, short, void *); 2077 u_int flag; 2078 2079 ssl = SSL_new(rlay->rl_ssl_ctx); 2080 if (ssl == NULL) 2081 goto err; 2082 2083 if (cre->dir == RELAY_DIR_REQUEST) { 2084 cb = relay_ssl_accept; 2085 method = SSLv23_server_method(); 2086 flag = EV_READ; 2087 2088 /* Use session-specific certificate for SSL inspection. */ 2089 if (cre->sslcert != NULL) 2090 SSL_use_certificate(ssl, cre->sslcert); 2091 } else { 2092 cb = relay_ssl_connect; 2093 method = SSLv23_client_method(); 2094 flag = EV_WRITE; 2095 } 2096 2097 if (!SSL_set_ssl_method(ssl, method)) 2098 goto err; 2099 if (!SSL_set_fd(ssl, cre->s)) 2100 goto err; 2101 2102 if (cre->dir == RELAY_DIR_REQUEST) { 2103 if ((proto->sslflags & SSLFLAG_CLIENT_RENEG) == 0) 2104 /* Only allow negotiation during the first handshake */ 2105 cre->sslreneg_state = SSLRENEG_INIT; 2106 else 2107 /* Allow client initiated renegotiations */ 2108 cre->sslreneg_state = SSLRENEG_ALLOW; 2109 SSL_set_accept_state(ssl); 2110 } else { 2111 /* Always allow renegotiations if we're the client */ 2112 cre->sslreneg_state = SSLRENEG_ALLOW; 2113 SSL_set_connect_state(ssl); 2114 } 2115 2116 SSL_set_app_data(ssl, cre); 2117 cre->ssl = ssl; 2118 2119 DPRINTF("%s: session %d: scheduling on %s", __func__, con->se_id, 2120 (flag == EV_READ) ? "EV_READ" : "EV_WRITE"); 2121 event_again(&con->se_ev, cre->s, EV_TIMEOUT|flag, cb, 2122 &con->se_tv_start, &rlay->rl_conf.timeout, con); 2123 return; 2124 2125 err: 2126 if (ssl != NULL) 2127 SSL_free(ssl); 2128 ssl_error(rlay->rl_conf.name, "relay_ssl_transaction"); 2129 relay_close(con, "session ssl failed"); 2130 } 2131 2132 void 2133 relay_ssl_accept(int fd, short event, void *arg) 2134 { 2135 struct rsession *con = arg; 2136 struct relay *rlay = con->se_relay; 2137 int retry_flag = 0; 2138 int ssl_err = 0; 2139 int ret; 2140 2141 if (event == EV_TIMEOUT) { 2142 relay_close(con, "SSL accept timeout"); 2143 return; 2144 } 2145 2146 ret = SSL_accept(con->se_in.ssl); 2147 if (ret <= 0) { 2148 ssl_err = SSL_get_error(con->se_in.ssl, ret); 2149 2150 switch (ssl_err) { 2151 case SSL_ERROR_WANT_READ: 2152 retry_flag = EV_READ; 2153 goto retry; 2154 case SSL_ERROR_WANT_WRITE: 2155 retry_flag = EV_WRITE; 2156 goto retry; 2157 case SSL_ERROR_ZERO_RETURN: 2158 case SSL_ERROR_SYSCALL: 2159 if (ret == 0) { 2160 relay_close(con, "closed"); 2161 return; 2162 } 2163 /* FALLTHROUGH */ 2164 default: 2165 ssl_error(rlay->rl_conf.name, "relay_ssl_accept"); 2166 relay_close(con, "SSL accept error"); 2167 return; 2168 } 2169 } 2170 2171 2172 #ifdef DEBUG 2173 log_info( 2174 #else 2175 log_debug( 2176 #endif 2177 "relay %s, session %d established (%d active)", 2178 rlay->rl_conf.name, con->se_id, relay_sessions); 2179 2180 relay_session(con); 2181 return; 2182 2183 retry: 2184 DPRINTF("%s: session %d: scheduling on %s", __func__, con->se_id, 2185 (retry_flag == EV_READ) ? "EV_READ" : "EV_WRITE"); 2186 event_again(&con->se_ev, fd, EV_TIMEOUT|retry_flag, relay_ssl_accept, 2187 &con->se_tv_start, &rlay->rl_conf.timeout, con); 2188 } 2189 2190 void 2191 relay_ssl_connect(int fd, short event, void *arg) 2192 { 2193 struct rsession *con = arg; 2194 struct relay *rlay = con->se_relay; 2195 int retry_flag = 0; 2196 int ssl_err = 0; 2197 int ret; 2198 X509 *servercert = NULL; 2199 2200 if (event == EV_TIMEOUT) { 2201 relay_close(con, "SSL connect timeout"); 2202 return; 2203 } 2204 2205 ret = SSL_connect(con->se_out.ssl); 2206 if (ret <= 0) { 2207 ssl_err = SSL_get_error(con->se_out.ssl, ret); 2208 2209 switch (ssl_err) { 2210 case SSL_ERROR_WANT_READ: 2211 retry_flag = EV_READ; 2212 goto retry; 2213 case SSL_ERROR_WANT_WRITE: 2214 retry_flag = EV_WRITE; 2215 goto retry; 2216 case SSL_ERROR_ZERO_RETURN: 2217 case SSL_ERROR_SYSCALL: 2218 if (ret == 0) { 2219 relay_close(con, "closed"); 2220 return; 2221 } 2222 /* FALLTHROUGH */ 2223 default: 2224 ssl_error(rlay->rl_conf.name, "relay_ssl_connect"); 2225 relay_close(con, "SSL connect error"); 2226 return; 2227 } 2228 } 2229 2230 #ifdef DEBUG 2231 log_info( 2232 #else 2233 log_debug( 2234 #endif 2235 "relay %s, ssl session %d connected (%d active)", 2236 rlay->rl_conf.name, con->se_id, relay_sessions); 2237 2238 if (rlay->rl_conf.flags & F_SSLINSPECT) { 2239 if ((servercert = 2240 SSL_get_peer_certificate(con->se_out.ssl)) != NULL) { 2241 con->se_in.sslcert = 2242 ssl_update_certificate(servercert, 2243 rlay->rl_ssl_pkey, rlay->rl_ssl_capkey, 2244 rlay->rl_ssl_cacertx509); 2245 } else 2246 con->se_in.sslcert = NULL; 2247 if (servercert != NULL) 2248 X509_free(servercert); 2249 if (con->se_in.sslcert == NULL) 2250 relay_close(con, "could not create certificate"); 2251 else 2252 relay_session(con); 2253 return; 2254 } 2255 2256 relay_connected(fd, EV_WRITE, con); 2257 return; 2258 2259 retry: 2260 DPRINTF("%s: session %d: scheduling on %s", __func__, con->se_id, 2261 (retry_flag == EV_READ) ? "EV_READ" : "EV_WRITE"); 2262 event_again(&con->se_ev, fd, EV_TIMEOUT|retry_flag, relay_ssl_connect, 2263 &con->se_tv_start, &rlay->rl_conf.timeout, con); 2264 } 2265 2266 void 2267 relay_ssl_connected(struct ctl_relay_event *cre) 2268 { 2269 /* 2270 * Hack libevent - we overwrite the internal bufferevent I/O 2271 * functions to handle the SSL abstraction. 2272 */ 2273 event_set(&cre->bev->ev_read, cre->s, EV_READ, 2274 relay_ssl_readcb, cre->bev); 2275 event_set(&cre->bev->ev_write, cre->s, EV_WRITE, 2276 relay_ssl_writecb, cre->bev); 2277 } 2278 2279 void 2280 relay_ssl_readcb(int fd, short event, void *arg) 2281 { 2282 char rbuf[IBUF_READ_SIZE]; 2283 struct bufferevent *bufev = arg; 2284 struct ctl_relay_event *cre = bufev->cbarg; 2285 struct rsession *con = cre->con; 2286 struct relay *rlay = con->se_relay; 2287 int ret = 0, ssl_err = 0; 2288 short what = EVBUFFER_READ; 2289 int howmuch = IBUF_READ_SIZE; 2290 size_t len; 2291 2292 if (event == EV_TIMEOUT) { 2293 what |= EVBUFFER_TIMEOUT; 2294 goto err; 2295 } 2296 2297 if (cre->sslreneg_state == SSLRENEG_ABORT) { 2298 what |= EVBUFFER_ERROR; 2299 goto err; 2300 } 2301 2302 if (bufev->wm_read.high != 0) 2303 howmuch = MIN(sizeof(rbuf), bufev->wm_read.high); 2304 2305 ret = SSL_read(cre->ssl, rbuf, howmuch); 2306 if (ret <= 0) { 2307 ssl_err = SSL_get_error(cre->ssl, ret); 2308 2309 switch (ssl_err) { 2310 case SSL_ERROR_WANT_READ: 2311 DPRINTF("%s: session %d: want read", 2312 __func__, con->se_id); 2313 goto retry; 2314 case SSL_ERROR_WANT_WRITE: 2315 DPRINTF("%s: session %d: want write", 2316 __func__, con->se_id); 2317 goto retry; 2318 default: 2319 if (ret == 0) 2320 what |= EVBUFFER_EOF; 2321 else { 2322 ssl_error(rlay->rl_conf.name, 2323 "relay_ssl_readcb"); 2324 what |= EVBUFFER_ERROR; 2325 } 2326 goto err; 2327 } 2328 } 2329 2330 if (evbuffer_add(bufev->input, rbuf, ret) == -1) { 2331 what |= EVBUFFER_ERROR; 2332 goto err; 2333 } 2334 2335 relay_bufferevent_add(&bufev->ev_read, bufev->timeout_read); 2336 2337 len = EVBUFFER_LENGTH(bufev->input); 2338 if (bufev->wm_read.low != 0 && len < bufev->wm_read.low) 2339 return; 2340 if (bufev->wm_read.high != 0 && len > bufev->wm_read.high) { 2341 struct evbuffer *buf = bufev->input; 2342 event_del(&bufev->ev_read); 2343 evbuffer_setcb(buf, bufferevent_read_pressure_cb, bufev); 2344 return; 2345 } 2346 2347 if (bufev->readcb != NULL) 2348 (*bufev->readcb)(bufev, bufev->cbarg); 2349 return; 2350 2351 retry: 2352 relay_bufferevent_add(&bufev->ev_read, bufev->timeout_read); 2353 return; 2354 2355 err: 2356 (*bufev->errorcb)(bufev, what, bufev->cbarg); 2357 } 2358 2359 void 2360 relay_ssl_writecb(int fd, short event, void *arg) 2361 { 2362 struct bufferevent *bufev = arg; 2363 struct ctl_relay_event *cre = bufev->cbarg; 2364 struct rsession *con = cre->con; 2365 struct relay *rlay = con->se_relay; 2366 int ret = 0, ssl_err; 2367 short what = EVBUFFER_WRITE; 2368 2369 if (event == EV_TIMEOUT) { 2370 what |= EVBUFFER_TIMEOUT; 2371 goto err; 2372 } 2373 2374 if (cre->sslreneg_state == SSLRENEG_ABORT) { 2375 what |= EVBUFFER_ERROR; 2376 goto err; 2377 } 2378 2379 if (EVBUFFER_LENGTH(bufev->output)) { 2380 if (cre->buf == NULL) { 2381 cre->buflen = EVBUFFER_LENGTH(bufev->output); 2382 if ((cre->buf = malloc(cre->buflen)) == NULL) { 2383 what |= EVBUFFER_ERROR; 2384 goto err; 2385 } 2386 bcopy(EVBUFFER_DATA(bufev->output), 2387 cre->buf, cre->buflen); 2388 } 2389 2390 ret = SSL_write(cre->ssl, cre->buf, cre->buflen); 2391 if (ret <= 0) { 2392 ssl_err = SSL_get_error(cre->ssl, ret); 2393 2394 switch (ssl_err) { 2395 case SSL_ERROR_WANT_READ: 2396 DPRINTF("%s: session %d: want read", 2397 __func__, con->se_id); 2398 goto retry; 2399 case SSL_ERROR_WANT_WRITE: 2400 DPRINTF("%s: session %d: want write", 2401 __func__, con->se_id); 2402 goto retry; 2403 default: 2404 if (ret == 0) 2405 what |= EVBUFFER_EOF; 2406 else { 2407 ssl_error(rlay->rl_conf.name, 2408 "relay_ssl_writecb"); 2409 what |= EVBUFFER_ERROR; 2410 } 2411 goto err; 2412 } 2413 } 2414 evbuffer_drain(bufev->output, ret); 2415 } 2416 if (cre->buf != NULL) { 2417 free(cre->buf); 2418 cre->buf = NULL; 2419 cre->buflen = 0; 2420 } 2421 2422 if (EVBUFFER_LENGTH(bufev->output) != 0) 2423 relay_bufferevent_add(&bufev->ev_write, bufev->timeout_write); 2424 2425 if (bufev->writecb != NULL && 2426 EVBUFFER_LENGTH(bufev->output) <= bufev->wm_write.low) 2427 (*bufev->writecb)(bufev, bufev->cbarg); 2428 return; 2429 2430 retry: 2431 if (cre->buflen != 0) 2432 relay_bufferevent_add(&bufev->ev_write, bufev->timeout_write); 2433 return; 2434 2435 err: 2436 if (cre->buf != NULL) { 2437 free(cre->buf); 2438 cre->buf = NULL; 2439 cre->buflen = 0; 2440 } 2441 (*bufev->errorcb)(bufev, what, bufev->cbarg); 2442 } 2443 2444 int 2445 relay_bufferevent_add(struct event *ev, int timeout) 2446 { 2447 struct timeval tv, *ptv = NULL; 2448 2449 if (timeout) { 2450 timerclear(&tv); 2451 tv.tv_sec = timeout; 2452 ptv = &tv; 2453 } 2454 2455 return (event_add(ev, ptv)); 2456 } 2457 2458 #ifdef notyet 2459 int 2460 relay_bufferevent_printf(struct ctl_relay_event *cre, const char *fmt, ...) 2461 { 2462 int ret; 2463 va_list ap; 2464 2465 va_start(ap, fmt); 2466 ret = evbuffer_add_vprintf(cre->output, fmt, ap); 2467 va_end(ap); 2468 2469 if (cre->bev != NULL && 2470 ret != -1 && EVBUFFER_LENGTH(cre->output) > 0 && 2471 (cre->bev->enabled & EV_WRITE)) 2472 bufferevent_enable(cre->bev, EV_WRITE); 2473 2474 return (ret); 2475 } 2476 #endif 2477 2478 int 2479 relay_bufferevent_print(struct ctl_relay_event *cre, const char *str) 2480 { 2481 if (cre->bev == NULL) 2482 return (evbuffer_add(cre->output, str, strlen(str))); 2483 return (bufferevent_write(cre->bev, str, strlen(str))); 2484 } 2485 2486 int 2487 relay_bufferevent_write_buffer(struct ctl_relay_event *cre, 2488 struct evbuffer *buf) 2489 { 2490 if (cre->bev == NULL) 2491 return (evbuffer_add_buffer(cre->output, buf)); 2492 return (bufferevent_write_buffer(cre->bev, buf)); 2493 } 2494 2495 int 2496 relay_bufferevent_write_chunk(struct ctl_relay_event *cre, 2497 struct evbuffer *buf, size_t size) 2498 { 2499 int ret; 2500 ret = relay_bufferevent_write(cre, buf->buffer, size); 2501 if (ret != -1) 2502 evbuffer_drain(buf, size); 2503 return (ret); 2504 } 2505 2506 int 2507 relay_bufferevent_write(struct ctl_relay_event *cre, void *data, size_t size) 2508 { 2509 if (cre->bev == NULL) 2510 return (evbuffer_add(cre->output, data, size)); 2511 return (bufferevent_write(cre->bev, data, size)); 2512 } 2513 2514 int 2515 relay_cmp_af(struct sockaddr_storage *a, struct sockaddr_storage *b) 2516 { 2517 int ret = -1; 2518 struct sockaddr_in ia, ib; 2519 struct sockaddr_in6 ia6, ib6; 2520 2521 switch (a->ss_family) { 2522 case AF_INET: 2523 bcopy(a, &ia, sizeof(struct sockaddr_in)); 2524 bcopy(b, &ib, sizeof(struct sockaddr_in)); 2525 2526 ret = memcmp(&ia.sin_addr, &ib.sin_addr, 2527 sizeof(ia.sin_addr)); 2528 if (ret == 0) 2529 ret = memcmp(&ia.sin_port, &ib.sin_port, 2530 sizeof(ia.sin_port)); 2531 break; 2532 case AF_INET6: 2533 bcopy(a, &ia6, sizeof(struct sockaddr_in6)); 2534 bcopy(b, &ib6, sizeof(struct sockaddr_in6)); 2535 2536 ret = memcmp(&ia6.sin6_addr, &ib6.sin6_addr, 2537 sizeof(ia6.sin6_addr)); 2538 if (ret == 0) 2539 ret = memcmp(&ia6.sin6_port, &ib6.sin6_port, 2540 sizeof(ia6.sin6_port)); 2541 break; 2542 default: 2543 break; 2544 } 2545 2546 return (ret); 2547 } 2548 2549 char * 2550 relay_load_file(const char *name, off_t *len) 2551 { 2552 struct stat st; 2553 off_t size; 2554 u_int8_t *buf = NULL; 2555 int fd; 2556 2557 if ((fd = open(name, O_RDONLY)) == -1) 2558 return (NULL); 2559 if (fstat(fd, &st) != 0) 2560 goto fail; 2561 size = st.st_size; 2562 if ((buf = calloc(1, size + 1)) == NULL) 2563 goto fail; 2564 if (read(fd, buf, size) != size) 2565 goto fail; 2566 2567 close(fd); 2568 2569 *len = size; 2570 return (buf); 2571 2572 fail: 2573 if (buf != NULL) 2574 free(buf); 2575 close(fd); 2576 return (NULL); 2577 } 2578 2579 int 2580 relay_load_certfiles(struct relay *rlay) 2581 { 2582 char certfile[PATH_MAX]; 2583 char hbuf[sizeof("ffff:ffff:ffff:ffff:ffff:ffff:255.255.255.255")]; 2584 struct protocol *proto = rlay->rl_proto; 2585 int useport = htons(rlay->rl_conf.port); 2586 2587 if (rlay->rl_conf.flags & F_SSLCLIENT) { 2588 if (strlen(proto->sslca)) { 2589 if ((rlay->rl_ssl_ca = 2590 relay_load_file(proto->sslca, 2591 &rlay->rl_conf.ssl_ca_len)) == NULL) 2592 return (-1); 2593 log_debug("%s: using ca %s", __func__, proto->sslca); 2594 } 2595 if (strlen(proto->sslcacert)) { 2596 if ((rlay->rl_ssl_cacert = 2597 relay_load_file(proto->sslcacert, 2598 &rlay->rl_conf.ssl_cacert_len)) == NULL) 2599 return (-1); 2600 log_debug("%s: using ca certificate %s", __func__, 2601 proto->sslcacert); 2602 } 2603 if (strlen(proto->sslcakey) && proto->sslcapass != NULL) { 2604 if ((rlay->rl_ssl_cakey = 2605 ssl_load_key(env, proto->sslcakey, 2606 &rlay->rl_conf.ssl_cakey_len, 2607 proto->sslcapass)) == NULL) 2608 return (-1); 2609 log_debug("%s: using ca key %s", __func__, 2610 proto->sslcakey); 2611 } 2612 } 2613 2614 if ((rlay->rl_conf.flags & F_SSL) == 0) 2615 return (0); 2616 2617 if (print_host(&rlay->rl_conf.ss, hbuf, sizeof(hbuf)) == NULL) 2618 return (-1); 2619 2620 if (snprintf(certfile, sizeof(certfile), 2621 "/etc/ssl/%s:%u.crt", hbuf, useport) == -1) 2622 return (-1); 2623 if ((rlay->rl_ssl_cert = relay_load_file(certfile, 2624 &rlay->rl_conf.ssl_cert_len)) == NULL) { 2625 if (snprintf(certfile, sizeof(certfile), 2626 "/etc/ssl/%s.crt", hbuf) == -1) 2627 return (-1); 2628 if ((rlay->rl_ssl_cert = relay_load_file(certfile, 2629 &rlay->rl_conf.ssl_cert_len)) == NULL) 2630 return (-1); 2631 useport = 0; 2632 } 2633 log_debug("%s: using certificate %s", __func__, certfile); 2634 2635 if (useport) { 2636 if (snprintf(certfile, sizeof(certfile), 2637 "/etc/ssl/private/%s:%u.key", hbuf, useport) == -1) 2638 return -1; 2639 } else { 2640 if (snprintf(certfile, sizeof(certfile), 2641 "/etc/ssl/private/%s.key", hbuf) == -1) 2642 return -1; 2643 } 2644 if ((rlay->rl_ssl_key = ssl_load_key(env, certfile, 2645 &rlay->rl_conf.ssl_key_len, NULL)) == NULL) 2646 return (-1); 2647 log_debug("%s: using private key %s", __func__, certfile); 2648 2649 return (0); 2650 } 2651 2652 int 2653 relay_session_cmp(struct rsession *a, struct rsession *b) 2654 { 2655 struct relay *rlay = b->se_relay; 2656 struct protocol *proto = rlay->rl_proto; 2657 2658 if (proto != NULL && proto->cmp != NULL) 2659 return ((*proto->cmp)(a, b)); 2660 2661 return ((int)a->se_id - b->se_id); 2662 } 2663 2664 void 2665 relay_log(struct rsession *con, char *msg) 2666 { 2667 if (con->se_haslog && con->se_log != NULL) { 2668 evbuffer_add(con->se_log, msg, strlen(msg)); 2669 } 2670 } 2671 2672 SPLAY_GENERATE(session_tree, rsession, se_nodes, relay_session_cmp); 2673