1 /* $OpenBSD: server.c,v 1.113 2017/11/29 16:55:08 beck Exp $ */ 2 3 /* 4 * Copyright (c) 2006 - 2015 Reyk Floeter <reyk@openbsd.org> 5 * 6 * Permission to use, copy, modify, and distribute this software for any 7 * purpose with or without fee is hereby granted, provided that the above 8 * copyright notice and this permission notice appear in all copies. 9 * 10 * THE SOFTWARE IS PROVIDED "AS IS" AND THE AUTHOR DISCLAIMS ALL WARRANTIES 11 * WITH REGARD TO THIS SOFTWARE INCLUDING ALL IMPLIED WARRANTIES OF 12 * MERCHANTABILITY AND FITNESS. IN NO EVENT SHALL THE AUTHOR BE LIABLE FOR 13 * ANY SPECIAL, DIRECT, INDIRECT, OR CONSEQUENTIAL DAMAGES OR ANY DAMAGES 14 * WHATSOEVER RESULTING FROM LOSS OF USE, DATA OR PROFITS, WHETHER IN AN 15 * ACTION OF CONTRACT, NEGLIGENCE OR OTHER TORTIOUS ACTION, ARISING OUT OF 16 * OR IN CONNECTION WITH THE USE OR PERFORMANCE OF THIS SOFTWARE. 17 */ 18 19 #include <sys/types.h> 20 #include <sys/queue.h> 21 #include <sys/time.h> 22 #include <sys/stat.h> 23 #include <sys/socket.h> 24 #include <sys/uio.h> 25 #include <sys/tree.h> 26 27 #include <netinet/in.h> 28 #include <netinet/tcp.h> 29 #include <arpa/inet.h> 30 31 #include <stdio.h> 32 #include <stdlib.h> 33 #include <stdarg.h> 34 #include <limits.h> 35 #include <errno.h> 36 #include <fcntl.h> 37 #include <string.h> 38 #include <syslog.h> 39 #include <unistd.h> 40 #include <event.h> 41 #include <imsg.h> 42 #include <tls.h> 43 #include <vis.h> 44 45 #include "httpd.h" 46 47 #define MINIMUM(a, b) (((a) < (b)) ? (a) : (b)) 48 49 int server_dispatch_parent(int, struct privsep_proc *, 50 struct imsg *); 51 int server_dispatch_logger(int, struct privsep_proc *, 52 struct imsg *); 53 void server_shutdown(void); 54 55 void server_init(struct privsep *, struct privsep_proc *p, void *); 56 void server_launch(void); 57 int server_socket(struct sockaddr_storage *, in_port_t, 58 struct server_config *, int, int); 59 int server_socket_listen(struct sockaddr_storage *, in_port_t, 60 struct server_config *); 61 struct server *server_byid(uint32_t); 62 63 int server_tls_init(struct server *); 64 void server_tls_readcb(int, short, void *); 65 void server_tls_writecb(int, short, void *); 66 void server_tls_handshake(int, short, void *); 67 68 void server_accept(int, short, void *); 69 void server_input(struct client *); 70 void server_inflight_dec(struct client *, const char *); 71 72 extern void bufferevent_read_pressure_cb(struct evbuffer *, size_t, 73 size_t, void *); 74 75 volatile int server_clients; 76 volatile int server_inflight = 0; 77 uint32_t server_cltid; 78 79 static struct privsep_proc procs[] = { 80 { "parent", PROC_PARENT, server_dispatch_parent }, 81 { "logger", PROC_LOGGER, server_dispatch_logger } 82 }; 83 84 void 85 server(struct privsep *ps, struct privsep_proc *p) 86 { 87 proc_run(ps, p, procs, nitems(procs), server_init, NULL); 88 server_http(); 89 } 90 91 void 92 server_shutdown(void) 93 { 94 config_purge(httpd_env, CONFIG_ALL); 95 usleep(200); /* XXX server needs to shutdown last */ 96 } 97 98 int 99 server_privinit(struct server *srv) 100 { 101 struct server *s; 102 103 if (srv->srv_conf.flags & SRVFLAG_LOCATION) 104 return (0); 105 106 log_debug("%s: adding server %s", __func__, srv->srv_conf.name); 107 108 /* 109 * There's no need to open a new socket if a server with the 110 * same address already exists. 111 */ 112 TAILQ_FOREACH(s, httpd_env->sc_servers, srv_entry) { 113 if (s != srv && s->srv_s != -1 && 114 s->srv_conf.port == srv->srv_conf.port && 115 sockaddr_cmp((struct sockaddr *)&s->srv_conf.ss, 116 (struct sockaddr *)&srv->srv_conf.ss, 117 s->srv_conf.prefixlen) == 0) 118 return (0); 119 } 120 121 /* Open listening socket in the privileged process */ 122 if ((srv->srv_s = server_socket_listen(&srv->srv_conf.ss, 123 srv->srv_conf.port, &srv->srv_conf)) == -1) 124 return (-1); 125 126 return (0); 127 } 128 129 int 130 server_tls_cmp(struct server *s1, struct server *s2, int match_keypair) 131 { 132 struct server_config *sc1, *sc2; 133 134 sc1 = &s1->srv_conf; 135 sc2 = &s2->srv_conf; 136 137 if (sc1->tls_protocols != sc2->tls_protocols) 138 return (-1); 139 if (sc1->tls_ticket_lifetime != sc2->tls_ticket_lifetime) 140 return (-1); 141 if (strcmp(sc1->tls_ciphers, sc2->tls_ciphers) != 0) 142 return (-1); 143 if (strcmp(sc1->tls_dhe_params, sc2->tls_dhe_params) != 0) 144 return (-1); 145 if (strcmp(sc1->tls_ecdhe_curves, sc2->tls_ecdhe_curves) != 0) 146 return (-1); 147 148 if (match_keypair) { 149 if (strcmp(sc1->tls_cert_file, sc2->tls_cert_file) != 0) 150 return (-1); 151 if (strcmp(sc1->tls_key_file, sc2->tls_key_file) != 0) 152 return (-1); 153 } 154 155 return (0); 156 } 157 158 int 159 server_tls_load_keypair(struct server *srv) 160 { 161 if ((srv->srv_conf.flags & SRVFLAG_TLS) == 0) 162 return (0); 163 164 if ((srv->srv_conf.tls_cert = tls_load_file(srv->srv_conf.tls_cert_file, 165 &srv->srv_conf.tls_cert_len, NULL)) == NULL) 166 return (-1); 167 log_debug("%s: using certificate %s", __func__, 168 srv->srv_conf.tls_cert_file); 169 170 /* XXX allow to specify password for encrypted key */ 171 if ((srv->srv_conf.tls_key = tls_load_file(srv->srv_conf.tls_key_file, 172 &srv->srv_conf.tls_key_len, NULL)) == NULL) 173 return (-1); 174 log_debug("%s: using private key %s", __func__, 175 srv->srv_conf.tls_key_file); 176 177 return (0); 178 } 179 180 int 181 server_tls_load_ocsp(struct server *srv) 182 { 183 if ((srv->srv_conf.flags & SRVFLAG_TLS) == 0) 184 return (0); 185 186 if (srv->srv_conf.tls_ocsp_staple_file == NULL) 187 return (0); 188 189 if ((srv->srv_conf.tls_ocsp_staple = tls_load_file( 190 srv->srv_conf.tls_ocsp_staple_file, 191 &srv->srv_conf.tls_ocsp_staple_len, NULL)) == NULL) { 192 log_warnx("%s: Failed to load ocsp staple from %s", __func__, 193 srv->srv_conf.tls_ocsp_staple_file); 194 return (-1); 195 } 196 197 if (srv->srv_conf.tls_ocsp_staple_len == 0) { 198 log_warnx("%s: ignoring 0 length ocsp staple from %s", __func__, 199 srv->srv_conf.tls_ocsp_staple_file); 200 return (0); 201 } 202 203 log_debug("%s: using ocsp staple from %s", __func__, 204 srv->srv_conf.tls_ocsp_staple_file); 205 206 return (0); 207 } 208 209 int 210 server_tls_init(struct server *srv) 211 { 212 struct server_config *srv_conf; 213 214 if ((srv->srv_conf.flags & SRVFLAG_TLS) == 0) 215 return (0); 216 217 log_debug("%s: setting up tls for %s", __func__, srv->srv_conf.name); 218 219 if (tls_init() != 0) { 220 log_warnx("%s: failed to initialise tls", __func__); 221 return (-1); 222 } 223 if ((srv->srv_tls_config = tls_config_new()) == NULL) { 224 log_warnx("%s: failed to get tls config", __func__); 225 return (-1); 226 } 227 if ((srv->srv_tls_ctx = tls_server()) == NULL) { 228 log_warnx("%s: failed to get tls server", __func__); 229 return (-1); 230 } 231 232 if (tls_config_set_protocols(srv->srv_tls_config, 233 srv->srv_conf.tls_protocols) != 0) { 234 log_warnx("%s: failed to set tls protocols: %s", 235 __func__, tls_config_error(srv->srv_tls_config)); 236 return (-1); 237 } 238 if (tls_config_set_ciphers(srv->srv_tls_config, 239 srv->srv_conf.tls_ciphers) != 0) { 240 log_warnx("%s: failed to set tls ciphers: %s", 241 __func__, tls_config_error(srv->srv_tls_config)); 242 return (-1); 243 } 244 if (tls_config_set_dheparams(srv->srv_tls_config, 245 srv->srv_conf.tls_dhe_params) != 0) { 246 log_warnx("%s: failed to set tls dhe params: %s", 247 __func__, tls_config_error(srv->srv_tls_config)); 248 return (-1); 249 } 250 if (tls_config_set_ecdhecurves(srv->srv_tls_config, 251 srv->srv_conf.tls_ecdhe_curves) != 0) { 252 log_warnx("%s: failed to set tls ecdhe curves: %s", 253 __func__, tls_config_error(srv->srv_tls_config)); 254 return (-1); 255 } 256 257 if (tls_config_set_keypair_ocsp_mem(srv->srv_tls_config, 258 srv->srv_conf.tls_cert, srv->srv_conf.tls_cert_len, 259 srv->srv_conf.tls_key, srv->srv_conf.tls_key_len, 260 srv->srv_conf.tls_ocsp_staple, 261 srv->srv_conf.tls_ocsp_staple_len) != 0) { 262 log_warnx("%s: failed to set tls certificate/key: %s", 263 __func__, tls_config_error(srv->srv_tls_config)); 264 return (-1); 265 } 266 267 TAILQ_FOREACH(srv_conf, &srv->srv_hosts, entry) { 268 if (srv_conf->tls_cert == NULL || srv_conf->tls_key == NULL) 269 continue; 270 log_debug("%s: adding keypair for server %s", __func__, 271 srv->srv_conf.name); 272 if (tls_config_add_keypair_ocsp_mem(srv->srv_tls_config, 273 srv_conf->tls_cert, srv_conf->tls_cert_len, 274 srv_conf->tls_key, srv_conf->tls_key_len, 275 srv_conf->tls_ocsp_staple, 276 srv_conf->tls_ocsp_staple_len) != 0) { 277 log_warnx("%s: failed to add tls keypair", __func__); 278 return (-1); 279 } 280 } 281 282 /* set common session ID among all processes */ 283 if (tls_config_set_session_id(srv->srv_tls_config, 284 httpd_env->sc_tls_sid, sizeof(httpd_env->sc_tls_sid)) == -1) { 285 log_warnx("%s: could not set the TLS session ID: %s", 286 __func__, tls_config_error(srv->srv_tls_config)); 287 return (-1); 288 } 289 290 /* ticket support */ 291 if (srv->srv_conf.tls_ticket_lifetime) { 292 if (tls_config_set_session_lifetime(srv->srv_tls_config, 293 srv->srv_conf.tls_ticket_lifetime) == -1) { 294 log_warnx("%s: could not set the TLS session lifetime: " 295 "%s", __func__, 296 tls_config_error(srv->srv_tls_config)); 297 return (-1); 298 } 299 tls_config_add_ticket_key(srv->srv_tls_config, 300 srv->srv_conf.tls_ticket_key.tt_keyrev, 301 srv->srv_conf.tls_ticket_key.tt_key, 302 sizeof(srv->srv_conf.tls_ticket_key.tt_key)); 303 explicit_bzero(&srv->srv_conf.tls_ticket_key, 304 sizeof(srv->srv_conf.tls_ticket_key)); 305 } 306 307 if (tls_configure(srv->srv_tls_ctx, srv->srv_tls_config) != 0) { 308 log_warnx("%s: failed to configure tls - %s", __func__, 309 tls_error(srv->srv_tls_ctx)); 310 return (-1); 311 } 312 313 /* We're now done with the public/private key... */ 314 tls_config_clear_keys(srv->srv_tls_config); 315 freezero(srv->srv_conf.tls_cert, srv->srv_conf.tls_cert_len); 316 freezero(srv->srv_conf.tls_key, srv->srv_conf.tls_key_len); 317 srv->srv_conf.tls_cert = NULL; 318 srv->srv_conf.tls_key = NULL; 319 srv->srv_conf.tls_cert_len = 0; 320 srv->srv_conf.tls_key_len = 0; 321 322 return (0); 323 } 324 325 void 326 server_generate_ticket_key(struct server_config *srv_conf) 327 { 328 struct server_tls_ticket *key = &srv_conf->tls_ticket_key; 329 330 key->tt_id = srv_conf->id; 331 key->tt_keyrev = arc4random(); 332 arc4random_buf(key->tt_key, sizeof(key->tt_key)); 333 } 334 335 void 336 server_init(struct privsep *ps, struct privsep_proc *p, void *arg) 337 { 338 server_http(); 339 340 if (config_init(ps->ps_env) == -1) 341 fatal("failed to initialize configuration"); 342 343 /* We use a custom shutdown callback */ 344 p->p_shutdown = server_shutdown; 345 346 /* Unlimited file descriptors (use system limits) */ 347 socket_rlimit(-1); 348 349 if (pledge("stdio rpath inet unix recvfd", NULL) == -1) 350 fatal("pledge"); 351 352 #if 0 353 /* Schedule statistics timer */ 354 evtimer_set(&ps->ps_env->sc_statev, server_statistics, NULL); 355 memcpy(&tv, &ps->ps_env->sc_statinterval, sizeof(tv)); 356 evtimer_add(&ps->ps_env->sc_statev, &tv); 357 #endif 358 } 359 360 void 361 server_launch(void) 362 { 363 struct server *srv; 364 365 TAILQ_FOREACH(srv, httpd_env->sc_servers, srv_entry) { 366 log_debug("%s: configuring server %s", __func__, 367 srv->srv_conf.name); 368 369 server_tls_init(srv); 370 server_http_init(srv); 371 372 log_debug("%s: running server %s", __func__, 373 srv->srv_conf.name); 374 375 event_set(&srv->srv_ev, srv->srv_s, EV_READ, 376 server_accept, srv); 377 event_add(&srv->srv_ev, NULL); 378 evtimer_set(&srv->srv_evt, server_accept, srv); 379 } 380 } 381 382 void 383 server_purge(struct server *srv) 384 { 385 struct client *clt; 386 struct server_config *srv_conf; 387 388 /* shutdown and remove server */ 389 if (event_initialized(&srv->srv_ev)) 390 event_del(&srv->srv_ev); 391 if (evtimer_initialized(&srv->srv_evt)) 392 evtimer_del(&srv->srv_evt); 393 394 if (srv->srv_s != -1) 395 close(srv->srv_s); 396 TAILQ_REMOVE(httpd_env->sc_servers, srv, srv_entry); 397 398 /* cleanup sessions */ 399 while ((clt = 400 SPLAY_ROOT(&srv->srv_clients)) != NULL) 401 server_close(clt, NULL); 402 403 /* cleanup hosts */ 404 while ((srv_conf = 405 TAILQ_FIRST(&srv->srv_hosts)) != NULL) { 406 TAILQ_REMOVE(&srv->srv_hosts, srv_conf, entry); 407 408 /* It might point to our own "default" entry */ 409 if (srv_conf != &srv->srv_conf) { 410 serverconfig_free(srv_conf); 411 free(srv_conf); 412 } 413 } 414 415 tls_config_free(srv->srv_tls_config); 416 tls_free(srv->srv_tls_ctx); 417 418 free(srv); 419 } 420 421 void 422 serverconfig_free(struct server_config *srv_conf) 423 { 424 free(srv_conf->return_uri); 425 free(srv_conf->tls_cert_file); 426 free(srv_conf->tls_key_file); 427 free(srv_conf->tls_ocsp_staple_file); 428 free(srv_conf->tls_ocsp_staple); 429 freezero(srv_conf->tls_cert, srv_conf->tls_cert_len); 430 freezero(srv_conf->tls_key, srv_conf->tls_key_len); 431 } 432 433 void 434 serverconfig_reset(struct server_config *srv_conf) 435 { 436 srv_conf->auth = NULL; 437 srv_conf->return_uri = NULL; 438 srv_conf->tls_cert = NULL; 439 srv_conf->tls_cert_file = NULL; 440 srv_conf->tls_key = NULL; 441 srv_conf->tls_key_file = NULL; 442 srv_conf->tls_ocsp_staple = NULL; 443 srv_conf->tls_ocsp_staple_file = NULL; 444 } 445 446 struct server * 447 server_byaddr(struct sockaddr *addr, in_port_t port) 448 { 449 struct server *srv; 450 451 TAILQ_FOREACH(srv, httpd_env->sc_servers, srv_entry) { 452 if (port == srv->srv_conf.port && 453 sockaddr_cmp((struct sockaddr *)&srv->srv_conf.ss, 454 addr, srv->srv_conf.prefixlen) == 0) 455 return (srv); 456 } 457 458 return (NULL); 459 } 460 461 struct server_config * 462 serverconfig_byid(uint32_t id) 463 { 464 struct server *srv; 465 struct server_config *srv_conf; 466 467 TAILQ_FOREACH(srv, httpd_env->sc_servers, srv_entry) { 468 if (srv->srv_conf.id == id) 469 return (&srv->srv_conf); 470 TAILQ_FOREACH(srv_conf, &srv->srv_hosts, entry) { 471 if (srv_conf->id == id) 472 return (srv_conf); 473 } 474 } 475 476 return (NULL); 477 } 478 479 struct server * 480 server_byid(uint32_t id) 481 { 482 struct server *srv; 483 484 TAILQ_FOREACH(srv, httpd_env->sc_servers, srv_entry) { 485 if (srv->srv_conf.id == id) 486 return (srv); 487 } 488 return (NULL); 489 } 490 491 int 492 server_foreach(int (*srv_cb)(struct server *, 493 struct server_config *, void *), void *arg) 494 { 495 struct server *srv; 496 struct server_config *srv_conf; 497 498 TAILQ_FOREACH(srv, httpd_env->sc_servers, srv_entry) { 499 if ((srv_cb)(srv, &srv->srv_conf, arg) == -1) 500 return (-1); 501 TAILQ_FOREACH(srv_conf, &srv->srv_hosts, entry) { 502 if ((srv_cb)(srv, srv_conf, arg) == -1) 503 return (-1); 504 } 505 } 506 507 return (0); 508 } 509 510 struct server * 511 server_match(struct server *s2, int match_name) 512 { 513 struct server *s1; 514 515 /* Attempt to find matching server. */ 516 TAILQ_FOREACH(s1, httpd_env->sc_servers, srv_entry) { 517 if ((s1->srv_conf.flags & SRVFLAG_LOCATION) != 0) 518 continue; 519 if (match_name) { 520 if (strcmp(s1->srv_conf.name, s2->srv_conf.name) != 0) 521 continue; 522 } 523 if (s1->srv_conf.port != s2->srv_conf.port) 524 continue; 525 if (sockaddr_cmp( 526 (struct sockaddr *)&s1->srv_conf.ss, 527 (struct sockaddr *)&s2->srv_conf.ss, 528 s1->srv_conf.prefixlen) != 0) 529 continue; 530 531 return (s1); 532 } 533 534 return (NULL); 535 } 536 537 int 538 server_socket_af(struct sockaddr_storage *ss, in_port_t port) 539 { 540 switch (ss->ss_family) { 541 case AF_INET: 542 ((struct sockaddr_in *)ss)->sin_port = port; 543 ((struct sockaddr_in *)ss)->sin_len = 544 sizeof(struct sockaddr_in); 545 break; 546 case AF_INET6: 547 ((struct sockaddr_in6 *)ss)->sin6_port = port; 548 ((struct sockaddr_in6 *)ss)->sin6_len = 549 sizeof(struct sockaddr_in6); 550 break; 551 default: 552 return (-1); 553 } 554 555 return (0); 556 } 557 558 in_port_t 559 server_socket_getport(struct sockaddr_storage *ss) 560 { 561 switch (ss->ss_family) { 562 case AF_INET: 563 return (((struct sockaddr_in *)ss)->sin_port); 564 case AF_INET6: 565 return (((struct sockaddr_in6 *)ss)->sin6_port); 566 default: 567 return (0); 568 } 569 570 /* NOTREACHED */ 571 return (0); 572 } 573 574 int 575 server_socket(struct sockaddr_storage *ss, in_port_t port, 576 struct server_config *srv_conf, int fd, int reuseport) 577 { 578 struct linger lng; 579 int s = -1, val; 580 581 if (server_socket_af(ss, port) == -1) 582 goto bad; 583 584 s = fd == -1 ? socket(ss->ss_family, SOCK_STREAM | SOCK_NONBLOCK, 585 IPPROTO_TCP) : fd; 586 if (s == -1) 587 goto bad; 588 589 /* 590 * Socket options 591 */ 592 memset(&lng, 0, sizeof(lng)); 593 if (setsockopt(s, SOL_SOCKET, SO_LINGER, &lng, sizeof(lng)) == -1) 594 goto bad; 595 if (reuseport) { 596 val = 1; 597 if (setsockopt(s, SOL_SOCKET, SO_REUSEPORT, &val, 598 sizeof(int)) == -1) 599 goto bad; 600 } 601 if (srv_conf->tcpflags & TCPFLAG_BUFSIZ) { 602 val = srv_conf->tcpbufsiz; 603 if (setsockopt(s, SOL_SOCKET, SO_RCVBUF, 604 &val, sizeof(val)) == -1) 605 goto bad; 606 val = srv_conf->tcpbufsiz; 607 if (setsockopt(s, SOL_SOCKET, SO_SNDBUF, 608 &val, sizeof(val)) == -1) 609 goto bad; 610 } 611 612 /* 613 * IP options 614 */ 615 if (srv_conf->tcpflags & TCPFLAG_IPTTL) { 616 val = (int)srv_conf->tcpipttl; 617 switch (ss->ss_family) { 618 case AF_INET: 619 if (setsockopt(s, IPPROTO_IP, IP_TTL, 620 &val, sizeof(val)) == -1) 621 goto bad; 622 break; 623 case AF_INET6: 624 if (setsockopt(s, IPPROTO_IPV6, IPV6_UNICAST_HOPS, 625 &val, sizeof(val)) == -1) 626 goto bad; 627 break; 628 } 629 } 630 if (srv_conf->tcpflags & TCPFLAG_IPMINTTL) { 631 val = (int)srv_conf->tcpipminttl; 632 switch (ss->ss_family) { 633 case AF_INET: 634 if (setsockopt(s, IPPROTO_IP, IP_MINTTL, 635 &val, sizeof(val)) == -1) 636 goto bad; 637 break; 638 case AF_INET6: 639 if (setsockopt(s, IPPROTO_IPV6, IPV6_MINHOPCOUNT, 640 &val, sizeof(val)) == -1) 641 goto bad; 642 break; 643 } 644 } 645 646 /* 647 * TCP options 648 */ 649 if (srv_conf->tcpflags & (TCPFLAG_NODELAY|TCPFLAG_NNODELAY)) { 650 if (srv_conf->tcpflags & TCPFLAG_NNODELAY) 651 val = 0; 652 else 653 val = 1; 654 if (setsockopt(s, IPPROTO_TCP, TCP_NODELAY, 655 &val, sizeof(val)) == -1) 656 goto bad; 657 } 658 if (srv_conf->tcpflags & (TCPFLAG_SACK|TCPFLAG_NSACK)) { 659 if (srv_conf->tcpflags & TCPFLAG_NSACK) 660 val = 0; 661 else 662 val = 1; 663 if (setsockopt(s, IPPROTO_TCP, TCP_SACK_ENABLE, 664 &val, sizeof(val)) == -1) 665 goto bad; 666 } 667 668 return (s); 669 670 bad: 671 if (s != -1) 672 close(s); 673 return (-1); 674 } 675 676 int 677 server_socket_listen(struct sockaddr_storage *ss, in_port_t port, 678 struct server_config *srv_conf) 679 { 680 int s; 681 682 if ((s = server_socket(ss, port, srv_conf, -1, 1)) == -1) 683 return (-1); 684 685 if (bind(s, (struct sockaddr *)ss, ss->ss_len) == -1) 686 goto bad; 687 if (listen(s, srv_conf->tcpbacklog) == -1) 688 goto bad; 689 690 return (s); 691 692 bad: 693 close(s); 694 return (-1); 695 } 696 697 int 698 server_socket_connect(struct sockaddr_storage *ss, in_port_t port, 699 struct server_config *srv_conf) 700 { 701 int s; 702 703 if ((s = server_socket(ss, port, srv_conf, -1, 0)) == -1) 704 return (-1); 705 706 if (connect(s, (struct sockaddr *)ss, ss->ss_len) == -1) { 707 if (errno != EINPROGRESS) 708 goto bad; 709 } 710 711 return (s); 712 713 bad: 714 close(s); 715 return (-1); 716 } 717 718 void 719 server_tls_readcb(int fd, short event, void *arg) 720 { 721 struct bufferevent *bufev = arg; 722 struct client *clt = bufev->cbarg; 723 char rbuf[IBUF_READ_SIZE]; 724 int what = EVBUFFER_READ; 725 int howmuch = IBUF_READ_SIZE; 726 ssize_t ret; 727 size_t len; 728 729 if (event == EV_TIMEOUT) { 730 what |= EVBUFFER_TIMEOUT; 731 goto err; 732 } 733 734 if (bufev->wm_read.high != 0) 735 howmuch = MINIMUM(sizeof(rbuf), bufev->wm_read.high); 736 737 ret = tls_read(clt->clt_tls_ctx, rbuf, howmuch); 738 if (ret == TLS_WANT_POLLIN || ret == TLS_WANT_POLLOUT) { 739 goto retry; 740 } else if (ret < 0) { 741 what |= EVBUFFER_ERROR; 742 goto err; 743 } 744 len = ret; 745 746 if (len == 0) { 747 what |= EVBUFFER_EOF; 748 goto err; 749 } 750 751 if (evbuffer_add(bufev->input, rbuf, len) == -1) { 752 what |= EVBUFFER_ERROR; 753 goto err; 754 } 755 756 server_bufferevent_add(&bufev->ev_read, bufev->timeout_read); 757 758 len = EVBUFFER_LENGTH(bufev->input); 759 if (bufev->wm_read.low != 0 && len < bufev->wm_read.low) 760 return; 761 if (bufev->wm_read.high != 0 && len > bufev->wm_read.high) { 762 struct evbuffer *buf = bufev->input; 763 event_del(&bufev->ev_read); 764 evbuffer_setcb(buf, bufferevent_read_pressure_cb, bufev); 765 return; 766 } 767 768 if (bufev->readcb != NULL) 769 (*bufev->readcb)(bufev, bufev->cbarg); 770 return; 771 772 retry: 773 server_bufferevent_add(&bufev->ev_read, bufev->timeout_read); 774 return; 775 776 err: 777 (*bufev->errorcb)(bufev, what, bufev->cbarg); 778 } 779 780 void 781 server_tls_writecb(int fd, short event, void *arg) 782 { 783 struct bufferevent *bufev = arg; 784 struct client *clt = bufev->cbarg; 785 ssize_t ret; 786 short what = EVBUFFER_WRITE; 787 size_t len; 788 789 if (event == EV_TIMEOUT) { 790 what |= EVBUFFER_TIMEOUT; 791 goto err; 792 } 793 794 if (EVBUFFER_LENGTH(bufev->output)) { 795 ret = tls_write(clt->clt_tls_ctx, 796 EVBUFFER_DATA(bufev->output), 797 EVBUFFER_LENGTH(bufev->output)); 798 if (ret == TLS_WANT_POLLIN || ret == TLS_WANT_POLLOUT) { 799 goto retry; 800 } else if (ret < 0) { 801 what |= EVBUFFER_ERROR; 802 goto err; 803 } 804 len = ret; 805 evbuffer_drain(bufev->output, len); 806 } 807 808 if (EVBUFFER_LENGTH(bufev->output) != 0) 809 server_bufferevent_add(&bufev->ev_write, bufev->timeout_write); 810 811 if (bufev->writecb != NULL && 812 EVBUFFER_LENGTH(bufev->output) <= bufev->wm_write.low) 813 (*bufev->writecb)(bufev, bufev->cbarg); 814 return; 815 816 retry: 817 server_bufferevent_add(&bufev->ev_write, bufev->timeout_write); 818 return; 819 820 err: 821 (*bufev->errorcb)(bufev, what, bufev->cbarg); 822 } 823 824 void 825 server_input(struct client *clt) 826 { 827 struct server_config *srv_conf = clt->clt_srv_conf; 828 evbuffercb inrd = server_read; 829 evbuffercb inwr = server_write; 830 socklen_t slen; 831 832 if (server_httpdesc_init(clt) == -1) { 833 server_close(clt, "failed to allocate http descriptor"); 834 return; 835 } 836 837 clt->clt_toread = TOREAD_HTTP_HEADER; 838 inrd = server_read_http; 839 840 slen = sizeof(clt->clt_sndbufsiz); 841 if (getsockopt(clt->clt_s, SOL_SOCKET, SO_SNDBUF, 842 &clt->clt_sndbufsiz, &slen) == -1) { 843 server_close(clt, "failed to get send buffer size"); 844 return; 845 } 846 847 /* 848 * Client <-> Server 849 */ 850 clt->clt_bev = bufferevent_new(clt->clt_s, inrd, inwr, 851 server_error, clt); 852 if (clt->clt_bev == NULL) { 853 server_close(clt, "failed to allocate input buffer event"); 854 return; 855 } 856 857 if (srv_conf->flags & SRVFLAG_TLS) { 858 event_set(&clt->clt_bev->ev_read, clt->clt_s, EV_READ, 859 server_tls_readcb, clt->clt_bev); 860 event_set(&clt->clt_bev->ev_write, clt->clt_s, EV_WRITE, 861 server_tls_writecb, clt->clt_bev); 862 } 863 864 /* Adjust write watermark to the socket buffer output size */ 865 bufferevent_setwatermark(clt->clt_bev, EV_WRITE, 866 SERVER_MIN_PREFETCHED * clt->clt_sndbufsiz, 0); 867 /* Read at most amount of data that fits in one fcgi record. */ 868 bufferevent_setwatermark(clt->clt_bev, EV_READ, 0, FCGI_CONTENT_SIZE); 869 870 bufferevent_settimeout(clt->clt_bev, 871 srv_conf->requesttimeout.tv_sec, srv_conf->requesttimeout.tv_sec); 872 bufferevent_enable(clt->clt_bev, EV_READ|EV_WRITE); 873 } 874 875 void 876 server_write(struct bufferevent *bev, void *arg) 877 { 878 struct client *clt = arg; 879 struct evbuffer *dst = EVBUFFER_OUTPUT(bev); 880 881 if (EVBUFFER_LENGTH(dst) == 0 && 882 clt->clt_toread == TOREAD_HTTP_NONE) 883 goto done; 884 885 getmonotime(&clt->clt_tv_last); 886 887 if (clt->clt_done) 888 goto done; 889 890 if (clt->clt_srvbev && clt->clt_srvbev_throttled) { 891 bufferevent_enable(clt->clt_srvbev, EV_READ); 892 clt->clt_srvbev_throttled = 0; 893 } 894 895 return; 896 done: 897 (*bev->errorcb)(bev, EVBUFFER_WRITE, bev->cbarg); 898 return; 899 } 900 901 void 902 server_dump(struct client *clt, const void *buf, size_t len) 903 { 904 if (!len) 905 return; 906 907 /* 908 * This function will dump the specified message directly 909 * to the underlying client, without waiting for success 910 * of non-blocking events etc. This is useful to print an 911 * error message before gracefully closing the client. 912 */ 913 if (clt->clt_tls_ctx != NULL) 914 (void)tls_write(clt->clt_tls_ctx, buf, len); 915 else 916 (void)write(clt->clt_s, buf, len); 917 } 918 919 void 920 server_read(struct bufferevent *bev, void *arg) 921 { 922 struct client *clt = arg; 923 struct evbuffer *src = EVBUFFER_INPUT(bev); 924 925 getmonotime(&clt->clt_tv_last); 926 927 if (!EVBUFFER_LENGTH(src)) 928 return; 929 if (server_bufferevent_write_buffer(clt, src) == -1) 930 goto fail; 931 if (clt->clt_done) 932 goto done; 933 934 if (EVBUFFER_LENGTH(EVBUFFER_OUTPUT(clt->clt_bev)) > (size_t) 935 SERVER_MAX_PREFETCH * clt->clt_sndbufsiz) { 936 bufferevent_disable(clt->clt_srvbev, EV_READ); 937 clt->clt_srvbev_throttled = 1; 938 } 939 940 return; 941 done: 942 (*bev->errorcb)(bev, EVBUFFER_READ, bev->cbarg); 943 return; 944 fail: 945 server_close(clt, strerror(errno)); 946 } 947 948 void 949 server_error(struct bufferevent *bev, short error, void *arg) 950 { 951 struct client *clt = arg; 952 struct evbuffer *dst; 953 954 if (error & EVBUFFER_TIMEOUT) { 955 server_abort_http(clt, 408, "timeout"); 956 return; 957 } 958 if (error & EVBUFFER_ERROR) { 959 if (errno == EFBIG) { 960 bufferevent_enable(bev, EV_READ); 961 return; 962 } 963 server_close(clt, "buffer event error"); 964 return; 965 } 966 if (error & EVBUFFER_EOF) { 967 server_close(clt, "closed"); 968 return; 969 } 970 if (error & (EVBUFFER_READ|EVBUFFER_WRITE)) { 971 bufferevent_disable(bev, EV_READ|EV_WRITE); 972 973 clt->clt_done = 1; 974 975 dst = EVBUFFER_OUTPUT(clt->clt_bev); 976 if (EVBUFFER_LENGTH(dst)) { 977 /* Finish writing all data first */ 978 bufferevent_enable(clt->clt_bev, EV_WRITE); 979 return; 980 } 981 982 server_close(clt, "done"); 983 return; 984 } 985 server_close(clt, "unknown event error"); 986 return; 987 } 988 989 void 990 server_accept(int fd, short event, void *arg) 991 { 992 struct server *srv = arg; 993 struct client *clt = NULL; 994 socklen_t slen; 995 struct sockaddr_storage ss; 996 int s = -1; 997 998 event_add(&srv->srv_ev, NULL); 999 if ((event & EV_TIMEOUT)) 1000 return; 1001 1002 slen = sizeof(ss); 1003 if ((s = accept_reserve(fd, (struct sockaddr *)&ss, 1004 &slen, FD_RESERVE, &server_inflight)) == -1) { 1005 /* 1006 * Pause accept if we are out of file descriptors, or 1007 * libevent will haunt us here too. 1008 */ 1009 if (errno == ENFILE || errno == EMFILE) { 1010 struct timeval evtpause = { 1, 0 }; 1011 1012 event_del(&srv->srv_ev); 1013 evtimer_add(&srv->srv_evt, &evtpause); 1014 log_debug("%s: deferring connections", __func__); 1015 } 1016 return; 1017 } 1018 if (server_clients >= SERVER_MAX_CLIENTS) 1019 goto err; 1020 1021 if ((clt = calloc(1, sizeof(*clt))) == NULL) 1022 goto err; 1023 1024 /* Pre-allocate log buffer */ 1025 clt->clt_log = evbuffer_new(); 1026 if (clt->clt_log == NULL) 1027 goto err; 1028 1029 clt->clt_s = s; 1030 clt->clt_fd = -1; 1031 clt->clt_toread = TOREAD_UNLIMITED; 1032 clt->clt_srv = srv; 1033 clt->clt_srv_conf = &srv->srv_conf; 1034 clt->clt_id = ++server_cltid; 1035 clt->clt_srv_id = srv->srv_conf.id; 1036 clt->clt_pid = getpid(); 1037 clt->clt_inflight = 1; 1038 1039 /* get local address */ 1040 slen = sizeof(clt->clt_srv_ss); 1041 if (getsockname(s, (struct sockaddr *)&clt->clt_srv_ss, 1042 &slen) == -1) { 1043 server_close(clt, "listen address lookup failed"); 1044 return; 1045 } 1046 1047 /* get client address */ 1048 memcpy(&clt->clt_ss, &ss, sizeof(clt->clt_ss)); 1049 1050 /* get ports */ 1051 switch (ss.ss_family) { 1052 case AF_INET: 1053 clt->clt_port = ((struct sockaddr_in *)&ss)->sin_port; 1054 break; 1055 case AF_INET6: 1056 clt->clt_port = ((struct sockaddr_in6 *)&ss)->sin6_port; 1057 break; 1058 } 1059 1060 getmonotime(&clt->clt_tv_start); 1061 memcpy(&clt->clt_tv_last, &clt->clt_tv_start, sizeof(clt->clt_tv_last)); 1062 1063 server_clients++; 1064 SPLAY_INSERT(client_tree, &srv->srv_clients, clt); 1065 1066 /* Pre-allocate output buffer */ 1067 clt->clt_output = evbuffer_new(); 1068 if (clt->clt_output == NULL) { 1069 server_close(clt, "failed to allocate output buffer"); 1070 return; 1071 } 1072 1073 if (srv->srv_conf.flags & SRVFLAG_TLS) { 1074 if (tls_accept_socket(srv->srv_tls_ctx, &clt->clt_tls_ctx, 1075 clt->clt_s) != 0) { 1076 server_close(clt, "failed to setup tls context"); 1077 return; 1078 } 1079 event_again(&clt->clt_ev, clt->clt_s, EV_TIMEOUT|EV_READ, 1080 server_tls_handshake, &clt->clt_tv_start, 1081 &srv->srv_conf.timeout, clt); 1082 return; 1083 } 1084 1085 server_input(clt); 1086 return; 1087 1088 err: 1089 if (s != -1) { 1090 close(s); 1091 free(clt); 1092 /* 1093 * the client struct was not completely set up, but still 1094 * counted as an inflight client. account for this. 1095 */ 1096 server_inflight_dec(NULL, __func__); 1097 } 1098 } 1099 1100 void 1101 server_tls_handshake(int fd, short event, void *arg) 1102 { 1103 struct client *clt = (struct client *)arg; 1104 struct server *srv = (struct server *)clt->clt_srv; 1105 int ret; 1106 1107 if (event == EV_TIMEOUT) { 1108 server_close(clt, "tls handshake timeout"); 1109 return; 1110 } 1111 1112 if (srv->srv_tls_ctx == NULL || clt->clt_tls_ctx == NULL) 1113 fatalx("NULL tls context"); 1114 1115 ret = tls_handshake(clt->clt_tls_ctx); 1116 if (ret == 0) { 1117 server_input(clt); 1118 } else if (ret == TLS_WANT_POLLIN) { 1119 event_again(&clt->clt_ev, clt->clt_s, EV_TIMEOUT|EV_READ, 1120 server_tls_handshake, &clt->clt_tv_start, 1121 &srv->srv_conf.timeout, clt); 1122 } else if (ret == TLS_WANT_POLLOUT) { 1123 event_again(&clt->clt_ev, clt->clt_s, EV_TIMEOUT|EV_WRITE, 1124 server_tls_handshake, &clt->clt_tv_start, 1125 &srv->srv_conf.timeout, clt); 1126 } else { 1127 log_debug("%s: tls handshake failed - %s", __func__, 1128 tls_error(clt->clt_tls_ctx)); 1129 server_close(clt, "tls handshake failed"); 1130 } 1131 } 1132 1133 void 1134 server_inflight_dec(struct client *clt, const char *why) 1135 { 1136 if (clt != NULL) { 1137 /* the flight already left inflight mode. */ 1138 if (clt->clt_inflight == 0) 1139 return; 1140 clt->clt_inflight = 0; 1141 } 1142 1143 /* the file was never opened, thus this was an inflight client. */ 1144 server_inflight--; 1145 DPRINTF("%s: inflight decremented, now %d, %s", 1146 __func__, server_inflight, why); 1147 } 1148 1149 void 1150 server_sendlog(struct server_config *srv_conf, int cmd, const char *emsg, ...) 1151 { 1152 va_list ap; 1153 char *msg; 1154 int ret; 1155 struct iovec iov[2]; 1156 1157 if (srv_conf->flags & SRVFLAG_SYSLOG) { 1158 va_start(ap, emsg); 1159 if (cmd == IMSG_LOG_ACCESS) 1160 vlog(LOG_INFO, emsg, ap); 1161 else 1162 vlog(LOG_DEBUG, emsg, ap); 1163 va_end(ap); 1164 return; 1165 } 1166 1167 va_start(ap, emsg); 1168 ret = vasprintf(&msg, emsg, ap); 1169 va_end(ap); 1170 if (ret == -1) { 1171 log_warn("%s: vasprintf", __func__); 1172 return; 1173 } 1174 1175 iov[0].iov_base = &srv_conf->id; 1176 iov[0].iov_len = sizeof(srv_conf->id); 1177 iov[1].iov_base = msg; 1178 iov[1].iov_len = strlen(msg) + 1; 1179 1180 if (proc_composev(httpd_env->sc_ps, PROC_LOGGER, cmd, iov, 2) != 0) { 1181 log_warn("%s: failed to compose imsg", __func__); 1182 return; 1183 } 1184 } 1185 1186 void 1187 server_log(struct client *clt, const char *msg) 1188 { 1189 char ibuf[HOST_NAME_MAX+1], obuf[HOST_NAME_MAX+1]; 1190 struct server_config *srv_conf = clt->clt_srv_conf; 1191 char *ptr = NULL, *vmsg = NULL; 1192 int debug_cmd = -1; 1193 1194 switch (srv_conf->logformat) { 1195 case LOG_FORMAT_CONNECTION: 1196 debug_cmd = IMSG_LOG_ACCESS; 1197 break; 1198 default: 1199 if (log_getverbose() > 1) 1200 debug_cmd = IMSG_LOG_ERROR; 1201 if (EVBUFFER_LENGTH(clt->clt_log)) { 1202 while ((ptr = 1203 evbuffer_readline(clt->clt_log)) != NULL) { 1204 server_sendlog(srv_conf, 1205 IMSG_LOG_ACCESS, "%s", ptr); 1206 free(ptr); 1207 } 1208 } 1209 break; 1210 } 1211 1212 if (debug_cmd != -1 && msg != NULL) { 1213 memset(ibuf, 0, sizeof(ibuf)); 1214 memset(obuf, 0, sizeof(obuf)); 1215 (void)print_host(&clt->clt_ss, ibuf, sizeof(ibuf)); 1216 (void)server_http_host(&clt->clt_srv_ss, obuf, sizeof(obuf)); 1217 if (EVBUFFER_LENGTH(clt->clt_log) && 1218 evbuffer_add_printf(clt->clt_log, "\n") != -1) 1219 ptr = evbuffer_readline(clt->clt_log); 1220 (void)stravis(&vmsg, msg, HTTPD_LOGVIS); 1221 server_sendlog(srv_conf, debug_cmd, "server %s, " 1222 "client %d (%d active), %s:%u -> %s, " 1223 "%s%s%s", srv_conf->name, clt->clt_id, server_clients, 1224 ibuf, ntohs(clt->clt_port), obuf, vmsg == NULL ? "" : vmsg, 1225 ptr == NULL ? "" : ",", ptr == NULL ? "" : ptr); 1226 free(vmsg); 1227 free(ptr); 1228 } 1229 } 1230 1231 void 1232 server_close(struct client *clt, const char *msg) 1233 { 1234 struct server *srv = clt->clt_srv; 1235 1236 SPLAY_REMOVE(client_tree, &srv->srv_clients, clt); 1237 1238 /* free the HTTP descriptors incl. headers */ 1239 server_close_http(clt); 1240 1241 event_del(&clt->clt_ev); 1242 if (clt->clt_bev != NULL) 1243 bufferevent_disable(clt->clt_bev, EV_READ|EV_WRITE); 1244 if (clt->clt_srvbev != NULL) 1245 bufferevent_disable(clt->clt_srvbev, EV_READ|EV_WRITE); 1246 1247 server_log(clt, msg); 1248 1249 if (clt->clt_bev != NULL) 1250 bufferevent_free(clt->clt_bev); 1251 if (clt->clt_output != NULL) 1252 evbuffer_free(clt->clt_output); 1253 if (clt->clt_srvevb != NULL) 1254 evbuffer_free(clt->clt_srvevb); 1255 1256 if (clt->clt_srvbev != NULL) 1257 bufferevent_free(clt->clt_srvbev); 1258 if (clt->clt_fd != -1) 1259 close(clt->clt_fd); 1260 if (clt->clt_s != -1) 1261 close(clt->clt_s); 1262 1263 if (clt->clt_tls_ctx != NULL) 1264 tls_close(clt->clt_tls_ctx); 1265 tls_free(clt->clt_tls_ctx); 1266 1267 server_inflight_dec(clt, __func__); 1268 1269 if (clt->clt_log != NULL) 1270 evbuffer_free(clt->clt_log); 1271 1272 free(clt); 1273 server_clients--; 1274 } 1275 1276 int 1277 server_dispatch_parent(int fd, struct privsep_proc *p, struct imsg *imsg) 1278 { 1279 struct server *srv; 1280 struct server_tls_ticket key; 1281 1282 switch (imsg->hdr.type) { 1283 case IMSG_CFG_MEDIA: 1284 config_getmedia(httpd_env, imsg); 1285 break; 1286 case IMSG_CFG_AUTH: 1287 config_getauth(httpd_env, imsg); 1288 break; 1289 case IMSG_CFG_SERVER: 1290 config_getserver(httpd_env, imsg); 1291 break; 1292 case IMSG_CFG_TLS: 1293 config_getserver_tls(httpd_env, imsg); 1294 break; 1295 case IMSG_CFG_DONE: 1296 config_getcfg(httpd_env, imsg); 1297 break; 1298 case IMSG_CTL_START: 1299 server_launch(); 1300 break; 1301 case IMSG_CTL_RESET: 1302 config_getreset(httpd_env, imsg); 1303 break; 1304 case IMSG_TLSTICKET_REKEY: 1305 IMSG_SIZE_CHECK(imsg, (&key)); 1306 memcpy(&key, imsg->data, sizeof(key)); 1307 /* apply to the right server */ 1308 srv = server_byid(key.tt_id); 1309 if (srv) { 1310 tls_config_add_ticket_key(srv->srv_tls_config, 1311 key.tt_keyrev, key.tt_key, sizeof(key.tt_key)); 1312 } 1313 break; 1314 default: 1315 return (-1); 1316 } 1317 1318 return (0); 1319 } 1320 1321 int 1322 server_dispatch_logger(int fd, struct privsep_proc *p, struct imsg *imsg) 1323 { 1324 switch (imsg->hdr.type) { 1325 default: 1326 return (-1); 1327 } 1328 1329 return (0); 1330 } 1331 1332 int 1333 server_bufferevent_add(struct event *ev, int timeout) 1334 { 1335 struct timeval tv, *ptv = NULL; 1336 1337 if (timeout) { 1338 timerclear(&tv); 1339 tv.tv_sec = timeout; 1340 ptv = &tv; 1341 } 1342 1343 return (event_add(ev, ptv)); 1344 } 1345 1346 int 1347 server_bufferevent_printf(struct client *clt, const char *fmt, ...) 1348 { 1349 int ret; 1350 va_list ap; 1351 char *str; 1352 1353 va_start(ap, fmt); 1354 ret = vasprintf(&str, fmt, ap); 1355 va_end(ap); 1356 1357 if (ret == -1) 1358 return (ret); 1359 1360 ret = server_bufferevent_print(clt, str); 1361 free(str); 1362 1363 return (ret); 1364 } 1365 1366 int 1367 server_bufferevent_print(struct client *clt, const char *str) 1368 { 1369 if (clt->clt_bev == NULL) 1370 return (evbuffer_add(clt->clt_output, str, strlen(str))); 1371 return (bufferevent_write(clt->clt_bev, str, strlen(str))); 1372 } 1373 1374 int 1375 server_bufferevent_write_buffer(struct client *clt, struct evbuffer *buf) 1376 { 1377 if (clt->clt_bev == NULL) 1378 return (evbuffer_add_buffer(clt->clt_output, buf)); 1379 return (bufferevent_write_buffer(clt->clt_bev, buf)); 1380 } 1381 1382 int 1383 server_bufferevent_write_chunk(struct client *clt, 1384 struct evbuffer *buf, size_t size) 1385 { 1386 int ret; 1387 ret = server_bufferevent_write(clt, buf->buffer, size); 1388 if (ret != -1) 1389 evbuffer_drain(buf, size); 1390 return (ret); 1391 } 1392 1393 int 1394 server_bufferevent_write(struct client *clt, void *data, size_t size) 1395 { 1396 if (clt->clt_bev == NULL) 1397 return (evbuffer_add(clt->clt_output, data, size)); 1398 return (bufferevent_write(clt->clt_bev, data, size)); 1399 } 1400 1401 int 1402 server_client_cmp(struct client *a, struct client *b) 1403 { 1404 return ((int)a->clt_id - b->clt_id); 1405 } 1406 1407 SPLAY_GENERATE(client_tree, client, clt_nodes, server_client_cmp); 1408