1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. All rights reserved. 3 * Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #if defined(__FreeBSD__) 10 #include <sys/event.h> 11 #define SPDK_KEVENT 12 #else 13 #include <sys/epoll.h> 14 #define SPDK_EPOLL 15 #endif 16 17 #if defined(__linux__) 18 #include <linux/errqueue.h> 19 #endif 20 21 #include "spdk/env.h" 22 #include "spdk/log.h" 23 #include "spdk/pipe.h" 24 #include "spdk/sock.h" 25 #include "spdk/util.h" 26 #include "spdk/string.h" 27 #include "spdk_internal/sock.h" 28 #include "../sock_kernel.h" 29 30 #include "openssl/crypto.h" 31 #include "openssl/err.h" 32 #include "openssl/ssl.h" 33 34 #define MAX_TMPBUF 1024 35 #define PORTNUMLEN 32 36 37 #if defined(SO_ZEROCOPY) && defined(MSG_ZEROCOPY) 38 #define SPDK_ZEROCOPY 39 #endif 40 41 struct spdk_posix_sock { 42 struct spdk_sock base; 43 int fd; 44 45 uint32_t sendmsg_idx; 46 47 struct spdk_pipe *recv_pipe; 48 void *recv_buf; 49 int recv_buf_sz; 50 bool pipe_has_data; 51 bool socket_has_data; 52 bool zcopy; 53 54 int placement_id; 55 56 SSL_CTX *ctx; 57 SSL *ssl; 58 59 TAILQ_ENTRY(spdk_posix_sock) link; 60 }; 61 62 TAILQ_HEAD(spdk_has_data_list, spdk_posix_sock); 63 64 struct spdk_posix_sock_group_impl { 65 struct spdk_sock_group_impl base; 66 int fd; 67 struct spdk_has_data_list socks_with_data; 68 int placement_id; 69 }; 70 71 static struct spdk_sock_impl_opts g_spdk_posix_sock_impl_opts = { 72 .recv_buf_size = MIN_SO_RCVBUF_SIZE, 73 .send_buf_size = MIN_SO_SNDBUF_SIZE, 74 .enable_recv_pipe = true, 75 .enable_quickack = false, 76 .enable_placement_id = PLACEMENT_NONE, 77 .enable_zerocopy_send_server = true, 78 .enable_zerocopy_send_client = false, 79 .zerocopy_threshold = 0, 80 .tls_version = 0, 81 .enable_ktls = false, 82 .psk_key = NULL, 83 .psk_identity = NULL 84 }; 85 86 static struct spdk_sock_map g_map = { 87 .entries = STAILQ_HEAD_INITIALIZER(g_map.entries), 88 .mtx = PTHREAD_MUTEX_INITIALIZER 89 }; 90 91 __attribute((destructor)) static void 92 posix_sock_map_cleanup(void) 93 { 94 spdk_sock_map_cleanup(&g_map); 95 } 96 97 #define __posix_sock(sock) (struct spdk_posix_sock *)sock 98 #define __posix_group_impl(group) (struct spdk_posix_sock_group_impl *)group 99 100 static void 101 posix_sock_copy_impl_opts(struct spdk_sock_impl_opts *dest, const struct spdk_sock_impl_opts *src, 102 size_t len) 103 { 104 #define FIELD_OK(field) \ 105 offsetof(struct spdk_sock_impl_opts, field) + sizeof(src->field) <= len 106 107 #define SET_FIELD(field) \ 108 if (FIELD_OK(field)) { \ 109 dest->field = src->field; \ 110 } 111 112 SET_FIELD(recv_buf_size); 113 SET_FIELD(send_buf_size); 114 SET_FIELD(enable_recv_pipe); 115 SET_FIELD(enable_zerocopy_send); 116 SET_FIELD(enable_quickack); 117 SET_FIELD(enable_placement_id); 118 SET_FIELD(enable_zerocopy_send_server); 119 SET_FIELD(enable_zerocopy_send_client); 120 SET_FIELD(zerocopy_threshold); 121 SET_FIELD(tls_version); 122 SET_FIELD(enable_ktls); 123 SET_FIELD(psk_key); 124 SET_FIELD(psk_identity); 125 126 #undef SET_FIELD 127 #undef FIELD_OK 128 } 129 130 static int 131 posix_sock_impl_get_opts(struct spdk_sock_impl_opts *opts, size_t *len) 132 { 133 if (!opts || !len) { 134 errno = EINVAL; 135 return -1; 136 } 137 138 assert(sizeof(*opts) >= *len); 139 memset(opts, 0, *len); 140 141 posix_sock_copy_impl_opts(opts, &g_spdk_posix_sock_impl_opts, *len); 142 *len = spdk_min(*len, sizeof(g_spdk_posix_sock_impl_opts)); 143 144 return 0; 145 } 146 147 static int 148 posix_sock_impl_set_opts(const struct spdk_sock_impl_opts *opts, size_t len) 149 { 150 if (!opts) { 151 errno = EINVAL; 152 return -1; 153 } 154 155 assert(sizeof(*opts) >= len); 156 posix_sock_copy_impl_opts(&g_spdk_posix_sock_impl_opts, opts, len); 157 158 return 0; 159 } 160 161 static void 162 posix_opts_get_impl_opts(const struct spdk_sock_opts *opts, struct spdk_sock_impl_opts *dest) 163 { 164 /* Copy the default impl_opts first to cover cases when user's impl_opts is smaller */ 165 memcpy(dest, &g_spdk_posix_sock_impl_opts, sizeof(*dest)); 166 167 if (opts->impl_opts != NULL) { 168 assert(sizeof(*dest) >= opts->impl_opts_size); 169 posix_sock_copy_impl_opts(dest, opts->impl_opts, opts->impl_opts_size); 170 } 171 } 172 173 static int 174 posix_sock_getaddr(struct spdk_sock *_sock, char *saddr, int slen, uint16_t *sport, 175 char *caddr, int clen, uint16_t *cport) 176 { 177 struct spdk_posix_sock *sock = __posix_sock(_sock); 178 struct sockaddr_storage sa; 179 socklen_t salen; 180 int rc; 181 182 assert(sock != NULL); 183 184 memset(&sa, 0, sizeof sa); 185 salen = sizeof sa; 186 rc = getsockname(sock->fd, (struct sockaddr *) &sa, &salen); 187 if (rc != 0) { 188 SPDK_ERRLOG("getsockname() failed (errno=%d)\n", errno); 189 return -1; 190 } 191 192 switch (sa.ss_family) { 193 case AF_UNIX: 194 /* Acceptable connection types that don't have IPs */ 195 return 0; 196 case AF_INET: 197 case AF_INET6: 198 /* Code below will get IP addresses */ 199 break; 200 default: 201 /* Unsupported socket family */ 202 return -1; 203 } 204 205 rc = get_addr_str((struct sockaddr *)&sa, saddr, slen); 206 if (rc != 0) { 207 SPDK_ERRLOG("getnameinfo() failed (errno=%d)\n", errno); 208 return -1; 209 } 210 211 if (sport) { 212 if (sa.ss_family == AF_INET) { 213 *sport = ntohs(((struct sockaddr_in *) &sa)->sin_port); 214 } else if (sa.ss_family == AF_INET6) { 215 *sport = ntohs(((struct sockaddr_in6 *) &sa)->sin6_port); 216 } 217 } 218 219 memset(&sa, 0, sizeof sa); 220 salen = sizeof sa; 221 rc = getpeername(sock->fd, (struct sockaddr *) &sa, &salen); 222 if (rc != 0) { 223 SPDK_ERRLOG("getpeername() failed (errno=%d)\n", errno); 224 return -1; 225 } 226 227 rc = get_addr_str((struct sockaddr *)&sa, caddr, clen); 228 if (rc != 0) { 229 SPDK_ERRLOG("getnameinfo() failed (errno=%d)\n", errno); 230 return -1; 231 } 232 233 if (cport) { 234 if (sa.ss_family == AF_INET) { 235 *cport = ntohs(((struct sockaddr_in *) &sa)->sin_port); 236 } else if (sa.ss_family == AF_INET6) { 237 *cport = ntohs(((struct sockaddr_in6 *) &sa)->sin6_port); 238 } 239 } 240 241 return 0; 242 } 243 244 enum posix_sock_create_type { 245 SPDK_SOCK_CREATE_LISTEN, 246 SPDK_SOCK_CREATE_CONNECT, 247 }; 248 249 static int 250 posix_sock_alloc_pipe(struct spdk_posix_sock *sock, int sz) 251 { 252 uint8_t *new_buf; 253 struct spdk_pipe *new_pipe; 254 struct iovec siov[2]; 255 struct iovec diov[2]; 256 int sbytes; 257 ssize_t bytes; 258 259 if (sock->recv_buf_sz == sz) { 260 return 0; 261 } 262 263 /* If the new size is 0, just free the pipe */ 264 if (sz == 0) { 265 spdk_pipe_destroy(sock->recv_pipe); 266 free(sock->recv_buf); 267 sock->recv_pipe = NULL; 268 sock->recv_buf = NULL; 269 return 0; 270 } else if (sz < MIN_SOCK_PIPE_SIZE) { 271 SPDK_ERRLOG("The size of the pipe must be larger than %d\n", MIN_SOCK_PIPE_SIZE); 272 return -1; 273 } 274 275 /* Round up to next 64 byte multiple */ 276 new_buf = calloc(SPDK_ALIGN_CEIL(sz + 1, 64), sizeof(uint8_t)); 277 if (!new_buf) { 278 SPDK_ERRLOG("socket recv buf allocation failed\n"); 279 return -ENOMEM; 280 } 281 282 new_pipe = spdk_pipe_create(new_buf, sz + 1); 283 if (new_pipe == NULL) { 284 SPDK_ERRLOG("socket pipe allocation failed\n"); 285 free(new_buf); 286 return -ENOMEM; 287 } 288 289 if (sock->recv_pipe != NULL) { 290 /* Pull all of the data out of the old pipe */ 291 sbytes = spdk_pipe_reader_get_buffer(sock->recv_pipe, sock->recv_buf_sz, siov); 292 if (sbytes > sz) { 293 /* Too much data to fit into the new pipe size */ 294 spdk_pipe_destroy(new_pipe); 295 free(new_buf); 296 return -EINVAL; 297 } 298 299 sbytes = spdk_pipe_writer_get_buffer(new_pipe, sz, diov); 300 assert(sbytes == sz); 301 302 bytes = spdk_iovcpy(siov, 2, diov, 2); 303 spdk_pipe_writer_advance(new_pipe, bytes); 304 305 spdk_pipe_destroy(sock->recv_pipe); 306 free(sock->recv_buf); 307 } 308 309 sock->recv_buf_sz = sz; 310 sock->recv_buf = new_buf; 311 sock->recv_pipe = new_pipe; 312 313 return 0; 314 } 315 316 static int 317 posix_sock_set_recvbuf(struct spdk_sock *_sock, int sz) 318 { 319 struct spdk_posix_sock *sock = __posix_sock(_sock); 320 int rc; 321 322 assert(sock != NULL); 323 324 if (_sock->impl_opts.enable_recv_pipe) { 325 rc = posix_sock_alloc_pipe(sock, sz); 326 if (rc) { 327 return rc; 328 } 329 } 330 331 /* Set kernel buffer size to be at least MIN_SO_RCVBUF_SIZE */ 332 if (sz < MIN_SO_RCVBUF_SIZE) { 333 sz = MIN_SO_RCVBUF_SIZE; 334 } 335 336 rc = setsockopt(sock->fd, SOL_SOCKET, SO_RCVBUF, &sz, sizeof(sz)); 337 if (rc < 0) { 338 return rc; 339 } 340 341 _sock->impl_opts.recv_buf_size = sz; 342 343 return 0; 344 } 345 346 static int 347 posix_sock_set_sendbuf(struct spdk_sock *_sock, int sz) 348 { 349 struct spdk_posix_sock *sock = __posix_sock(_sock); 350 int rc; 351 352 assert(sock != NULL); 353 354 if (sz < MIN_SO_SNDBUF_SIZE) { 355 sz = MIN_SO_SNDBUF_SIZE; 356 } 357 358 rc = setsockopt(sock->fd, SOL_SOCKET, SO_SNDBUF, &sz, sizeof(sz)); 359 if (rc < 0) { 360 return rc; 361 } 362 363 _sock->impl_opts.send_buf_size = sz; 364 365 return 0; 366 } 367 368 static void 369 posix_sock_init(struct spdk_posix_sock *sock, bool enable_zero_copy) 370 { 371 #if defined(SPDK_ZEROCOPY) || defined(__linux__) 372 int flag; 373 int rc; 374 #endif 375 376 #if defined(SPDK_ZEROCOPY) 377 flag = 1; 378 379 if (enable_zero_copy) { 380 /* Try to turn on zero copy sends */ 381 rc = setsockopt(sock->fd, SOL_SOCKET, SO_ZEROCOPY, &flag, sizeof(flag)); 382 if (rc == 0) { 383 sock->zcopy = true; 384 } 385 } 386 #endif 387 388 #if defined(__linux__) 389 flag = 1; 390 391 if (sock->base.impl_opts.enable_quickack) { 392 rc = setsockopt(sock->fd, IPPROTO_TCP, TCP_QUICKACK, &flag, sizeof(flag)); 393 if (rc != 0) { 394 SPDK_ERRLOG("quickack was failed to set\n"); 395 } 396 } 397 398 spdk_sock_get_placement_id(sock->fd, sock->base.impl_opts.enable_placement_id, 399 &sock->placement_id); 400 401 if (sock->base.impl_opts.enable_placement_id == PLACEMENT_MARK) { 402 /* Save placement_id */ 403 spdk_sock_map_insert(&g_map, sock->placement_id, NULL); 404 } 405 #endif 406 } 407 408 static struct spdk_posix_sock * 409 posix_sock_alloc(int fd, struct spdk_sock_impl_opts *impl_opts, bool enable_zero_copy) 410 { 411 struct spdk_posix_sock *sock; 412 413 sock = calloc(1, sizeof(*sock)); 414 if (sock == NULL) { 415 SPDK_ERRLOG("sock allocation failed\n"); 416 return NULL; 417 } 418 419 sock->fd = fd; 420 memcpy(&sock->base.impl_opts, impl_opts, sizeof(*impl_opts)); 421 posix_sock_init(sock, enable_zero_copy); 422 423 return sock; 424 } 425 426 static int 427 posix_fd_create(struct addrinfo *res, struct spdk_sock_opts *opts, 428 struct spdk_sock_impl_opts *impl_opts) 429 { 430 int fd; 431 int val = 1; 432 int rc, sz; 433 #if defined(__linux__) 434 int to; 435 #endif 436 437 fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol); 438 if (fd < 0) { 439 /* error */ 440 return -1; 441 } 442 443 sz = impl_opts->recv_buf_size; 444 rc = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &sz, sizeof(sz)); 445 if (rc) { 446 /* Not fatal */ 447 } 448 449 sz = impl_opts->send_buf_size; 450 rc = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &sz, sizeof(sz)); 451 if (rc) { 452 /* Not fatal */ 453 } 454 455 rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof val); 456 if (rc != 0) { 457 close(fd); 458 /* error */ 459 return -1; 460 } 461 rc = setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof val); 462 if (rc != 0) { 463 close(fd); 464 /* error */ 465 return -1; 466 } 467 468 #if defined(SO_PRIORITY) 469 if (opts->priority) { 470 rc = setsockopt(fd, SOL_SOCKET, SO_PRIORITY, &opts->priority, sizeof val); 471 if (rc != 0) { 472 close(fd); 473 /* error */ 474 return -1; 475 } 476 } 477 #endif 478 479 if (res->ai_family == AF_INET6) { 480 rc = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &val, sizeof val); 481 if (rc != 0) { 482 close(fd); 483 /* error */ 484 return -1; 485 } 486 } 487 488 if (opts->ack_timeout) { 489 #if defined(__linux__) 490 to = opts->ack_timeout; 491 rc = setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, &to, sizeof(to)); 492 if (rc != 0) { 493 close(fd); 494 /* error */ 495 return -1; 496 } 497 #else 498 SPDK_WARNLOG("TCP_USER_TIMEOUT is not supported.\n"); 499 #endif 500 } 501 502 return fd; 503 } 504 505 static unsigned int 506 posix_sock_tls_psk_server_cb(SSL *ssl, 507 const char *id, 508 unsigned char *psk, 509 unsigned int max_psk_len) 510 { 511 long key_len; 512 unsigned char *default_psk; 513 struct spdk_sock_impl_opts *impl_opts; 514 515 impl_opts = SSL_get_app_data(ssl); 516 517 if (impl_opts->psk_key == NULL) { 518 SPDK_ERRLOG("PSK is not set\n"); 519 goto err; 520 } 521 SPDK_DEBUGLOG(sock_posix, "Length of Client's PSK ID %lu\n", strlen(impl_opts->psk_identity)); 522 if (id == NULL) { 523 SPDK_ERRLOG("Received empty PSK ID\n"); 524 goto err; 525 } 526 SPDK_DEBUGLOG(sock_posix, "Received PSK ID '%s'\n", id); 527 if (strcmp(impl_opts->psk_identity, id) != 0) { 528 SPDK_ERRLOG("Unknown Client's PSK ID\n"); 529 goto err; 530 } 531 532 SPDK_DEBUGLOG(sock_posix, "Length of Client's PSK KEY %u\n", max_psk_len); 533 default_psk = OPENSSL_hexstr2buf(impl_opts->psk_key, &key_len); 534 if (default_psk == NULL) { 535 SPDK_ERRLOG("Could not unhexlify PSK\n"); 536 goto err; 537 } 538 if (key_len > max_psk_len) { 539 SPDK_ERRLOG("Insufficient buffer size to copy PSK\n"); 540 goto err; 541 } 542 543 memcpy(psk, default_psk, key_len); 544 545 return key_len; 546 547 err: 548 return 0; 549 } 550 551 static unsigned int 552 posix_sock_tls_psk_client_cb(SSL *ssl, const char *hint, 553 char *identity, 554 unsigned int max_identity_len, 555 unsigned char *psk, 556 unsigned int max_psk_len) 557 { 558 long key_len; 559 unsigned char *default_psk; 560 struct spdk_sock_impl_opts *impl_opts; 561 562 impl_opts = SSL_get_app_data(ssl); 563 564 if (hint) { 565 SPDK_DEBUGLOG(sock_posix, "Received PSK identity hint '%s'\n", hint); 566 } 567 568 if (impl_opts->psk_key == NULL) { 569 SPDK_ERRLOG("PSK is not set\n"); 570 goto err; 571 } 572 default_psk = OPENSSL_hexstr2buf(impl_opts->psk_key, &key_len); 573 if (default_psk == NULL) { 574 SPDK_ERRLOG("Could not unhexlify PSK\n"); 575 goto err; 576 } 577 if ((strlen(impl_opts->psk_identity) + 1 > max_identity_len) 578 || (key_len > max_psk_len)) { 579 SPDK_ERRLOG("PSK ID or Key buffer is not sufficient\n"); 580 goto err; 581 } 582 spdk_strcpy_pad(identity, impl_opts->psk_identity, strlen(impl_opts->psk_identity), 0); 583 SPDK_DEBUGLOG(sock_posix, "Sending PSK identity '%s'\n", identity); 584 585 memcpy(psk, default_psk, key_len); 586 SPDK_DEBUGLOG(sock_posix, "Provided out-of-band (OOB) PSK for TLS1.3 client\n"); 587 588 return key_len; 589 590 err: 591 return 0; 592 } 593 594 static SSL_CTX * 595 posix_sock_create_ssl_context(const SSL_METHOD *method, struct spdk_sock_opts *opts, 596 struct spdk_sock_impl_opts *impl_opts) 597 { 598 SSL_CTX *ctx; 599 int tls_version = 0; 600 bool ktls_enabled = false; 601 #ifdef SSL_OP_ENABLE_KTLS 602 long options; 603 #endif 604 605 SSL_library_init(); 606 OpenSSL_add_all_algorithms(); 607 SSL_load_error_strings(); 608 /* Produce a SSL CTX in SSL V2 and V3 standards compliant way */ 609 ctx = SSL_CTX_new(method); 610 if (!ctx) { 611 SPDK_ERRLOG("SSL_CTX_new() failed, msg = %s\n", ERR_error_string(ERR_peek_last_error(), NULL)); 612 return NULL; 613 } 614 SPDK_DEBUGLOG(sock_posix, "SSL context created\n"); 615 616 switch (impl_opts->tls_version) { 617 case 0: 618 /* auto-negotioation */ 619 break; 620 case SPDK_TLS_VERSION_1_1: 621 tls_version = TLS1_1_VERSION; 622 break; 623 case SPDK_TLS_VERSION_1_2: 624 tls_version = TLS1_2_VERSION; 625 break; 626 case SPDK_TLS_VERSION_1_3: 627 tls_version = TLS1_3_VERSION; 628 break; 629 default: 630 SPDK_ERRLOG("Incorrect TLS version provided: %d\n", impl_opts->tls_version); 631 goto err; 632 } 633 634 if (tls_version) { 635 SPDK_DEBUGLOG(sock_posix, "Hardening TLS version to '%d'='0x%X'\n", impl_opts->tls_version, 636 tls_version); 637 if (!SSL_CTX_set_min_proto_version(ctx, tls_version)) { 638 SPDK_ERRLOG("Unable to set Min TLS version to '%d'='0x%X\n", impl_opts->tls_version, tls_version); 639 goto err; 640 } 641 if (!SSL_CTX_set_max_proto_version(ctx, tls_version)) { 642 SPDK_ERRLOG("Unable to set Max TLS version to '%d'='0x%X\n", impl_opts->tls_version, tls_version); 643 goto err; 644 } 645 } 646 if (impl_opts->enable_ktls) { 647 SPDK_DEBUGLOG(sock_posix, "Enabling kTLS offload\n"); 648 #ifdef SSL_OP_ENABLE_KTLS 649 options = SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS); 650 ktls_enabled = options & SSL_OP_ENABLE_KTLS; 651 #else 652 ktls_enabled = false; 653 #endif 654 if (!ktls_enabled) { 655 SPDK_ERRLOG("Unable to set kTLS offload via SSL_CTX_set_options(). Configure openssl with 'enable-ktls'\n"); 656 goto err; 657 } 658 } 659 660 return ctx; 661 662 err: 663 SSL_CTX_free(ctx); 664 return NULL; 665 } 666 667 static SSL * 668 ssl_sock_connect_loop(SSL_CTX *ctx, int fd, struct spdk_sock_impl_opts *impl_opts) 669 { 670 int rc; 671 SSL *ssl; 672 int ssl_get_error; 673 674 ssl = SSL_new(ctx); 675 if (!ssl) { 676 SPDK_ERRLOG("SSL_new() failed, msg = %s\n", ERR_error_string(ERR_peek_last_error(), NULL)); 677 return NULL; 678 } 679 SSL_set_fd(ssl, fd); 680 SSL_set_app_data(ssl, impl_opts); 681 SSL_set_psk_client_callback(ssl, posix_sock_tls_psk_client_cb); 682 SPDK_DEBUGLOG(sock_posix, "SSL object creation finished: %p\n", ssl); 683 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 684 while ((rc = SSL_connect(ssl)) != 1) { 685 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 686 ssl_get_error = SSL_get_error(ssl, rc); 687 SPDK_DEBUGLOG(sock_posix, "SSL_connect failed %d = SSL_connect(%p), %d = SSL_get_error(%p, %d)\n", 688 rc, ssl, ssl_get_error, ssl, rc); 689 switch (ssl_get_error) { 690 case SSL_ERROR_WANT_READ: 691 case SSL_ERROR_WANT_WRITE: 692 continue; 693 default: 694 break; 695 } 696 SPDK_ERRLOG("SSL_connect() failed, errno = %d\n", errno); 697 SSL_free(ssl); 698 return NULL; 699 } 700 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 701 SPDK_DEBUGLOG(sock_posix, "Negotiated Cipher suite:%s\n", 702 SSL_CIPHER_get_name(SSL_get_current_cipher(ssl))); 703 return ssl; 704 } 705 706 static SSL * 707 ssl_sock_accept_loop(SSL_CTX *ctx, int fd, struct spdk_sock_impl_opts *impl_opts) 708 { 709 int rc; 710 SSL *ssl; 711 int ssl_get_error; 712 713 ssl = SSL_new(ctx); 714 if (!ssl) { 715 SPDK_ERRLOG("SSL_new() failed, msg = %s\n", ERR_error_string(ERR_peek_last_error(), NULL)); 716 return NULL; 717 } 718 SSL_set_fd(ssl, fd); 719 SSL_set_app_data(ssl, impl_opts); 720 SSL_set_psk_server_callback(ssl, posix_sock_tls_psk_server_cb); 721 SPDK_DEBUGLOG(sock_posix, "SSL object creation finished: %p\n", ssl); 722 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 723 while ((rc = SSL_accept(ssl)) != 1) { 724 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 725 ssl_get_error = SSL_get_error(ssl, rc); 726 SPDK_DEBUGLOG(sock_posix, "SSL_accept failed %d = SSL_accept(%p), %d = SSL_get_error(%p, %d)\n", rc, 727 ssl, ssl_get_error, ssl, rc); 728 switch (ssl_get_error) { 729 case SSL_ERROR_WANT_READ: 730 case SSL_ERROR_WANT_WRITE: 731 continue; 732 default: 733 break; 734 } 735 SPDK_ERRLOG("SSL_accept() failed, errno = %d\n", errno); 736 SSL_free(ssl); 737 return NULL; 738 } 739 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 740 SPDK_DEBUGLOG(sock_posix, "Negotiated Cipher suite:%s\n", 741 SSL_CIPHER_get_name(SSL_get_current_cipher(ssl))); 742 return ssl; 743 } 744 745 static ssize_t 746 SSL_readv(SSL *ssl, const struct iovec *iov, int iovcnt) 747 { 748 int i, rc = 0; 749 ssize_t total = 0; 750 751 for (i = 0; i < iovcnt; i++) { 752 rc = SSL_read(ssl, iov[i].iov_base, iov[i].iov_len); 753 754 if (rc > 0) { 755 total += rc; 756 } 757 if (rc != (int)iov[i].iov_len) { 758 break; 759 } 760 } 761 if (total > 0) { 762 errno = 0; 763 return total; 764 } 765 switch (SSL_get_error(ssl, rc)) { 766 case SSL_ERROR_ZERO_RETURN: 767 errno = ENOTCONN; 768 return 0; 769 case SSL_ERROR_WANT_READ: 770 case SSL_ERROR_WANT_WRITE: 771 case SSL_ERROR_WANT_CONNECT: 772 case SSL_ERROR_WANT_ACCEPT: 773 case SSL_ERROR_WANT_X509_LOOKUP: 774 case SSL_ERROR_WANT_ASYNC: 775 case SSL_ERROR_WANT_ASYNC_JOB: 776 case SSL_ERROR_WANT_CLIENT_HELLO_CB: 777 errno = EAGAIN; 778 return -1; 779 case SSL_ERROR_SYSCALL: 780 case SSL_ERROR_SSL: 781 errno = ENOTCONN; 782 return -1; 783 default: 784 errno = ENOTCONN; 785 return -1; 786 } 787 } 788 789 static ssize_t 790 SSL_writev(SSL *ssl, struct iovec *iov, int iovcnt) 791 { 792 int i, rc = 0; 793 ssize_t total = 0; 794 795 for (i = 0; i < iovcnt; i++) { 796 rc = SSL_write(ssl, iov[i].iov_base, iov[i].iov_len); 797 798 if (rc > 0) { 799 total += rc; 800 } 801 if (rc != (int)iov[i].iov_len) { 802 break; 803 } 804 } 805 if (total > 0) { 806 errno = 0; 807 return total; 808 } 809 switch (SSL_get_error(ssl, rc)) { 810 case SSL_ERROR_ZERO_RETURN: 811 errno = ENOTCONN; 812 return 0; 813 case SSL_ERROR_WANT_READ: 814 case SSL_ERROR_WANT_WRITE: 815 case SSL_ERROR_WANT_CONNECT: 816 case SSL_ERROR_WANT_ACCEPT: 817 case SSL_ERROR_WANT_X509_LOOKUP: 818 case SSL_ERROR_WANT_ASYNC: 819 case SSL_ERROR_WANT_ASYNC_JOB: 820 case SSL_ERROR_WANT_CLIENT_HELLO_CB: 821 errno = EAGAIN; 822 return -1; 823 case SSL_ERROR_SYSCALL: 824 case SSL_ERROR_SSL: 825 errno = ENOTCONN; 826 return -1; 827 default: 828 errno = ENOTCONN; 829 return -1; 830 } 831 } 832 833 static struct spdk_sock * 834 posix_sock_create(const char *ip, int port, 835 enum posix_sock_create_type type, 836 struct spdk_sock_opts *opts, 837 bool enable_ssl) 838 { 839 struct spdk_posix_sock *sock; 840 struct spdk_sock_impl_opts impl_opts; 841 char buf[MAX_TMPBUF]; 842 char portnum[PORTNUMLEN]; 843 char *p; 844 struct addrinfo hints, *res, *res0; 845 int fd, flag; 846 int rc; 847 bool enable_zcopy_user_opts = true; 848 bool enable_zcopy_impl_opts = true; 849 SSL_CTX *ctx = 0; 850 SSL *ssl = 0; 851 852 assert(opts != NULL); 853 posix_opts_get_impl_opts(opts, &impl_opts); 854 855 if (ip == NULL) { 856 return NULL; 857 } 858 if (ip[0] == '[') { 859 snprintf(buf, sizeof(buf), "%s", ip + 1); 860 p = strchr(buf, ']'); 861 if (p != NULL) { 862 *p = '\0'; 863 } 864 ip = (const char *) &buf[0]; 865 } 866 867 snprintf(portnum, sizeof portnum, "%d", port); 868 memset(&hints, 0, sizeof hints); 869 hints.ai_family = PF_UNSPEC; 870 hints.ai_socktype = SOCK_STREAM; 871 hints.ai_flags = AI_NUMERICSERV; 872 hints.ai_flags |= AI_PASSIVE; 873 hints.ai_flags |= AI_NUMERICHOST; 874 rc = getaddrinfo(ip, portnum, &hints, &res0); 875 if (rc != 0) { 876 SPDK_ERRLOG("getaddrinfo() failed %s (%d)\n", gai_strerror(rc), rc); 877 return NULL; 878 } 879 880 /* try listen */ 881 fd = -1; 882 for (res = res0; res != NULL; res = res->ai_next) { 883 retry: 884 fd = posix_fd_create(res, opts, &impl_opts); 885 if (fd < 0) { 886 continue; 887 } 888 if (type == SPDK_SOCK_CREATE_LISTEN) { 889 if (enable_ssl) { 890 ctx = posix_sock_create_ssl_context(TLS_server_method(), opts, &impl_opts); 891 if (!ctx) { 892 SPDK_ERRLOG("posix_sock_create_ssl_context() failed, errno = %d\n", errno); 893 close(fd); 894 fd = -1; 895 break; 896 } 897 } 898 rc = bind(fd, res->ai_addr, res->ai_addrlen); 899 if (rc != 0) { 900 SPDK_ERRLOG("bind() failed at port %d, errno = %d\n", port, errno); 901 switch (errno) { 902 case EINTR: 903 /* interrupted? */ 904 close(fd); 905 goto retry; 906 case EADDRNOTAVAIL: 907 SPDK_ERRLOG("IP address %s not available. " 908 "Verify IP address in config file " 909 "and make sure setup script is " 910 "run before starting spdk app.\n", ip); 911 /* FALLTHROUGH */ 912 default: 913 /* try next family */ 914 close(fd); 915 fd = -1; 916 continue; 917 } 918 } 919 /* bind OK */ 920 rc = listen(fd, 512); 921 if (rc != 0) { 922 SPDK_ERRLOG("listen() failed, errno = %d\n", errno); 923 close(fd); 924 fd = -1; 925 break; 926 } 927 enable_zcopy_impl_opts = impl_opts.enable_zerocopy_send_server; 928 } else if (type == SPDK_SOCK_CREATE_CONNECT) { 929 rc = connect(fd, res->ai_addr, res->ai_addrlen); 930 if (rc != 0) { 931 SPDK_ERRLOG("connect() failed, errno = %d\n", errno); 932 /* try next family */ 933 close(fd); 934 fd = -1; 935 continue; 936 } 937 enable_zcopy_impl_opts = impl_opts.enable_zerocopy_send_client; 938 if (enable_ssl) { 939 ctx = posix_sock_create_ssl_context(TLS_client_method(), opts, &impl_opts); 940 if (!ctx) { 941 SPDK_ERRLOG("posix_sock_create_ssl_context() failed, errno = %d\n", errno); 942 close(fd); 943 fd = -1; 944 break; 945 } 946 ssl = ssl_sock_connect_loop(ctx, fd, &impl_opts); 947 if (!ssl) { 948 SPDK_ERRLOG("ssl_sock_connect_loop() failed, errno = %d\n", errno); 949 close(fd); 950 fd = -1; 951 SSL_CTX_free(ctx); 952 break; 953 } 954 } 955 } 956 957 flag = fcntl(fd, F_GETFL); 958 if (fcntl(fd, F_SETFL, flag | O_NONBLOCK) < 0) { 959 SPDK_ERRLOG("fcntl can't set nonblocking mode for socket, fd: %d (%d)\n", fd, errno); 960 close(fd); 961 fd = -1; 962 break; 963 } 964 break; 965 } 966 freeaddrinfo(res0); 967 968 if (fd < 0) { 969 return NULL; 970 } 971 972 /* Only enable zero copy for non-loopback and non-ssl sockets. */ 973 enable_zcopy_user_opts = opts->zcopy && !sock_is_loopback(fd) && !enable_ssl; 974 975 sock = posix_sock_alloc(fd, &impl_opts, enable_zcopy_user_opts && enable_zcopy_impl_opts); 976 if (sock == NULL) { 977 SPDK_ERRLOG("sock allocation failed\n"); 978 close(fd); 979 return NULL; 980 } 981 982 if (ctx) { 983 sock->ctx = ctx; 984 } 985 986 if (ssl) { 987 sock->ssl = ssl; 988 } 989 990 return &sock->base; 991 } 992 993 static struct spdk_sock * 994 posix_sock_listen(const char *ip, int port, struct spdk_sock_opts *opts) 995 { 996 return posix_sock_create(ip, port, SPDK_SOCK_CREATE_LISTEN, opts, false); 997 } 998 999 static struct spdk_sock * 1000 posix_sock_connect(const char *ip, int port, struct spdk_sock_opts *opts) 1001 { 1002 return posix_sock_create(ip, port, SPDK_SOCK_CREATE_CONNECT, opts, false); 1003 } 1004 1005 static struct spdk_sock * 1006 posix_sock_accept(struct spdk_sock *_sock) 1007 { 1008 struct spdk_posix_sock *sock = __posix_sock(_sock); 1009 struct sockaddr_storage sa; 1010 socklen_t salen; 1011 int rc, fd; 1012 struct spdk_posix_sock *new_sock; 1013 int flag; 1014 SSL *ssl = 0; 1015 1016 memset(&sa, 0, sizeof(sa)); 1017 salen = sizeof(sa); 1018 1019 assert(sock != NULL); 1020 1021 rc = accept(sock->fd, (struct sockaddr *)&sa, &salen); 1022 1023 if (rc == -1) { 1024 return NULL; 1025 } 1026 1027 fd = rc; 1028 1029 flag = fcntl(fd, F_GETFL); 1030 if ((!(flag & O_NONBLOCK)) && (fcntl(fd, F_SETFL, flag | O_NONBLOCK) < 0)) { 1031 SPDK_ERRLOG("fcntl can't set nonblocking mode for socket, fd: %d (%d)\n", fd, errno); 1032 close(fd); 1033 return NULL; 1034 } 1035 1036 #if defined(SO_PRIORITY) 1037 /* The priority is not inherited, so call this function again */ 1038 if (sock->base.opts.priority) { 1039 rc = setsockopt(fd, SOL_SOCKET, SO_PRIORITY, &sock->base.opts.priority, sizeof(int)); 1040 if (rc != 0) { 1041 close(fd); 1042 return NULL; 1043 } 1044 } 1045 #endif 1046 1047 /* Establish SSL connection */ 1048 if (sock->ctx) { 1049 ssl = ssl_sock_accept_loop(sock->ctx, fd, &sock->base.impl_opts); 1050 if (!ssl) { 1051 SPDK_ERRLOG("ssl_sock_accept_loop() failed, errno = %d\n", errno); 1052 close(fd); 1053 return NULL; 1054 } 1055 } 1056 1057 /* Inherit the zero copy feature from the listen socket */ 1058 new_sock = posix_sock_alloc(fd, &sock->base.impl_opts, sock->zcopy); 1059 if (new_sock == NULL) { 1060 close(fd); 1061 return NULL; 1062 } 1063 1064 if (sock->ctx) { 1065 new_sock->ctx = sock->ctx; 1066 } 1067 1068 if (ssl) { 1069 new_sock->ssl = ssl; 1070 } 1071 1072 return &new_sock->base; 1073 } 1074 1075 static int 1076 posix_sock_close(struct spdk_sock *_sock) 1077 { 1078 struct spdk_posix_sock *sock = __posix_sock(_sock); 1079 1080 assert(TAILQ_EMPTY(&_sock->pending_reqs)); 1081 1082 /* If the socket fails to close, the best choice is to 1083 * leak the fd but continue to free the rest of the sock 1084 * memory. */ 1085 close(sock->fd); 1086 1087 spdk_pipe_destroy(sock->recv_pipe); 1088 free(sock->recv_buf); 1089 free(sock); 1090 1091 return 0; 1092 } 1093 1094 #ifdef SPDK_ZEROCOPY 1095 static int 1096 _sock_check_zcopy(struct spdk_sock *sock) 1097 { 1098 struct spdk_posix_sock *psock = __posix_sock(sock); 1099 struct msghdr msgh = {}; 1100 uint8_t buf[sizeof(struct cmsghdr) + sizeof(struct sock_extended_err)]; 1101 ssize_t rc; 1102 struct sock_extended_err *serr; 1103 struct cmsghdr *cm; 1104 uint32_t idx; 1105 struct spdk_sock_request *req, *treq; 1106 bool found; 1107 1108 msgh.msg_control = buf; 1109 msgh.msg_controllen = sizeof(buf); 1110 1111 while (true) { 1112 rc = recvmsg(psock->fd, &msgh, MSG_ERRQUEUE); 1113 1114 if (rc < 0) { 1115 if (errno == EWOULDBLOCK || errno == EAGAIN) { 1116 return 0; 1117 } 1118 1119 if (!TAILQ_EMPTY(&sock->pending_reqs)) { 1120 SPDK_ERRLOG("Attempting to receive from ERRQUEUE yielded error, but pending list still has orphaned entries\n"); 1121 } else { 1122 SPDK_WARNLOG("Recvmsg yielded an error!\n"); 1123 } 1124 return 0; 1125 } 1126 1127 cm = CMSG_FIRSTHDR(&msgh); 1128 if (!(cm && 1129 ((cm->cmsg_level == SOL_IP && cm->cmsg_type == IP_RECVERR) || 1130 (cm->cmsg_level == SOL_IPV6 && cm->cmsg_type == IPV6_RECVERR)))) { 1131 SPDK_WARNLOG("Unexpected cmsg level or type!\n"); 1132 return 0; 1133 } 1134 1135 serr = (struct sock_extended_err *)CMSG_DATA(cm); 1136 if (serr->ee_errno != 0 || serr->ee_origin != SO_EE_ORIGIN_ZEROCOPY) { 1137 SPDK_WARNLOG("Unexpected extended error origin\n"); 1138 return 0; 1139 } 1140 1141 /* Most of the time, the pending_reqs array is in the exact 1142 * order we need such that all of the requests to complete are 1143 * in order, in the front. It is guaranteed that all requests 1144 * belonging to the same sendmsg call are sequential, so once 1145 * we encounter one match we can stop looping as soon as a 1146 * non-match is found. 1147 */ 1148 for (idx = serr->ee_info; idx <= serr->ee_data; idx++) { 1149 found = false; 1150 TAILQ_FOREACH_SAFE(req, &sock->pending_reqs, internal.link, treq) { 1151 if (!req->internal.is_zcopy) { 1152 /* This wasn't a zcopy request. It was just waiting in line to complete */ 1153 rc = spdk_sock_request_put(sock, req, 0); 1154 if (rc < 0) { 1155 return rc; 1156 } 1157 } else if (req->internal.offset == idx) { 1158 found = true; 1159 rc = spdk_sock_request_put(sock, req, 0); 1160 if (rc < 0) { 1161 return rc; 1162 } 1163 } else if (found) { 1164 break; 1165 } 1166 } 1167 } 1168 } 1169 1170 return 0; 1171 } 1172 #endif 1173 1174 static int 1175 _sock_flush(struct spdk_sock *sock) 1176 { 1177 struct spdk_posix_sock *psock = __posix_sock(sock); 1178 struct msghdr msg = {}; 1179 int flags; 1180 struct iovec iovs[IOV_BATCH_SIZE]; 1181 int iovcnt; 1182 int retval; 1183 struct spdk_sock_request *req; 1184 int i; 1185 ssize_t rc; 1186 unsigned int offset; 1187 size_t len; 1188 bool is_zcopy = false; 1189 1190 /* Can't flush from within a callback or we end up with recursive calls */ 1191 if (sock->cb_cnt > 0) { 1192 return 0; 1193 } 1194 1195 #ifdef SPDK_ZEROCOPY 1196 if (psock->zcopy) { 1197 flags = MSG_ZEROCOPY | MSG_NOSIGNAL; 1198 } else 1199 #endif 1200 { 1201 flags = MSG_NOSIGNAL; 1202 } 1203 1204 iovcnt = spdk_sock_prep_reqs(sock, iovs, 0, NULL, &flags); 1205 if (iovcnt == 0) { 1206 return 0; 1207 } 1208 1209 #ifdef SPDK_ZEROCOPY 1210 is_zcopy = flags & MSG_ZEROCOPY; 1211 #endif 1212 1213 /* Perform the vectored write */ 1214 msg.msg_iov = iovs; 1215 msg.msg_iovlen = iovcnt; 1216 1217 if (psock->ssl) { 1218 rc = SSL_writev(psock->ssl, iovs, iovcnt); 1219 } else { 1220 rc = sendmsg(psock->fd, &msg, flags); 1221 } 1222 if (rc <= 0) { 1223 if (errno == EAGAIN || errno == EWOULDBLOCK || (errno == ENOBUFS && psock->zcopy)) { 1224 return 0; 1225 } 1226 return rc; 1227 } 1228 1229 if (is_zcopy) { 1230 /* Handling overflow case, because we use psock->sendmsg_idx - 1 for the 1231 * req->internal.offset, so sendmsg_idx should not be zero */ 1232 if (spdk_unlikely(psock->sendmsg_idx == UINT32_MAX)) { 1233 psock->sendmsg_idx = 1; 1234 } else { 1235 psock->sendmsg_idx++; 1236 } 1237 } 1238 1239 /* Consume the requests that were actually written */ 1240 req = TAILQ_FIRST(&sock->queued_reqs); 1241 while (req) { 1242 offset = req->internal.offset; 1243 1244 /* req->internal.is_zcopy is true when the whole req or part of it is sent with zerocopy */ 1245 req->internal.is_zcopy = is_zcopy; 1246 1247 for (i = 0; i < req->iovcnt; i++) { 1248 /* Advance by the offset first */ 1249 if (offset >= SPDK_SOCK_REQUEST_IOV(req, i)->iov_len) { 1250 offset -= SPDK_SOCK_REQUEST_IOV(req, i)->iov_len; 1251 continue; 1252 } 1253 1254 /* Calculate the remaining length of this element */ 1255 len = SPDK_SOCK_REQUEST_IOV(req, i)->iov_len - offset; 1256 1257 if (len > (size_t)rc) { 1258 /* This element was partially sent. */ 1259 req->internal.offset += rc; 1260 return 0; 1261 } 1262 1263 offset = 0; 1264 req->internal.offset += len; 1265 rc -= len; 1266 } 1267 1268 /* Handled a full request. */ 1269 spdk_sock_request_pend(sock, req); 1270 1271 if (!req->internal.is_zcopy && req == TAILQ_FIRST(&sock->pending_reqs)) { 1272 /* The sendmsg syscall above isn't currently asynchronous, 1273 * so it's already done. */ 1274 retval = spdk_sock_request_put(sock, req, 0); 1275 if (retval) { 1276 break; 1277 } 1278 } else { 1279 /* Re-use the offset field to hold the sendmsg call index. The 1280 * index is 0 based, so subtract one here because we've already 1281 * incremented above. */ 1282 req->internal.offset = psock->sendmsg_idx - 1; 1283 } 1284 1285 if (rc == 0) { 1286 break; 1287 } 1288 1289 req = TAILQ_FIRST(&sock->queued_reqs); 1290 } 1291 1292 return 0; 1293 } 1294 1295 static int 1296 posix_sock_flush(struct spdk_sock *sock) 1297 { 1298 #ifdef SPDK_ZEROCOPY 1299 struct spdk_posix_sock *psock = __posix_sock(sock); 1300 1301 if (psock->zcopy && !TAILQ_EMPTY(&sock->pending_reqs)) { 1302 _sock_check_zcopy(sock); 1303 } 1304 #endif 1305 1306 return _sock_flush(sock); 1307 } 1308 1309 static ssize_t 1310 posix_sock_recv_from_pipe(struct spdk_posix_sock *sock, struct iovec *diov, int diovcnt) 1311 { 1312 struct iovec siov[2]; 1313 int sbytes; 1314 ssize_t bytes; 1315 struct spdk_posix_sock_group_impl *group; 1316 1317 sbytes = spdk_pipe_reader_get_buffer(sock->recv_pipe, sock->recv_buf_sz, siov); 1318 if (sbytes < 0) { 1319 errno = EINVAL; 1320 return -1; 1321 } else if (sbytes == 0) { 1322 errno = EAGAIN; 1323 return -1; 1324 } 1325 1326 bytes = spdk_iovcpy(siov, 2, diov, diovcnt); 1327 1328 if (bytes == 0) { 1329 /* The only way this happens is if diov is 0 length */ 1330 errno = EINVAL; 1331 return -1; 1332 } 1333 1334 spdk_pipe_reader_advance(sock->recv_pipe, bytes); 1335 1336 /* If we drained the pipe, mark it appropriately */ 1337 if (spdk_pipe_reader_bytes_available(sock->recv_pipe) == 0) { 1338 assert(sock->pipe_has_data == true); 1339 1340 group = __posix_group_impl(sock->base.group_impl); 1341 if (group && !sock->socket_has_data) { 1342 TAILQ_REMOVE(&group->socks_with_data, sock, link); 1343 } 1344 1345 sock->pipe_has_data = false; 1346 } 1347 1348 return bytes; 1349 } 1350 1351 static inline ssize_t 1352 posix_sock_read(struct spdk_posix_sock *sock) 1353 { 1354 struct iovec iov[2]; 1355 int bytes_avail, bytes_recvd; 1356 struct spdk_posix_sock_group_impl *group; 1357 1358 bytes_avail = spdk_pipe_writer_get_buffer(sock->recv_pipe, sock->recv_buf_sz, iov); 1359 1360 if (bytes_avail <= 0) { 1361 return bytes_avail; 1362 } 1363 1364 if (sock->ssl) { 1365 bytes_recvd = SSL_readv(sock->ssl, iov, 2); 1366 } else { 1367 bytes_recvd = readv(sock->fd, iov, 2); 1368 } 1369 1370 assert(sock->pipe_has_data == false); 1371 1372 if (bytes_recvd <= 0) { 1373 /* Errors count as draining the socket data */ 1374 if (sock->base.group_impl && sock->socket_has_data) { 1375 group = __posix_group_impl(sock->base.group_impl); 1376 TAILQ_REMOVE(&group->socks_with_data, sock, link); 1377 } 1378 1379 sock->socket_has_data = false; 1380 1381 return bytes_recvd; 1382 } 1383 1384 spdk_pipe_writer_advance(sock->recv_pipe, bytes_recvd); 1385 1386 #if DEBUG 1387 if (sock->base.group_impl) { 1388 assert(sock->socket_has_data == true); 1389 } 1390 #endif 1391 1392 sock->pipe_has_data = true; 1393 if (bytes_recvd < bytes_avail) { 1394 /* We drained the kernel socket entirely. */ 1395 sock->socket_has_data = false; 1396 } 1397 1398 return bytes_recvd; 1399 } 1400 1401 static ssize_t 1402 posix_sock_readv(struct spdk_sock *_sock, struct iovec *iov, int iovcnt) 1403 { 1404 struct spdk_posix_sock *sock = __posix_sock(_sock); 1405 struct spdk_posix_sock_group_impl *group = __posix_group_impl(sock->base.group_impl); 1406 int rc, i; 1407 size_t len; 1408 1409 if (sock->recv_pipe == NULL) { 1410 assert(sock->pipe_has_data == false); 1411 if (group && sock->socket_has_data) { 1412 sock->socket_has_data = false; 1413 TAILQ_REMOVE(&group->socks_with_data, sock, link); 1414 } 1415 if (sock->ssl) { 1416 return SSL_readv(sock->ssl, iov, iovcnt); 1417 } else { 1418 return readv(sock->fd, iov, iovcnt); 1419 } 1420 } 1421 1422 /* If the socket is not in a group, we must assume it always has 1423 * data waiting for us because it is not epolled */ 1424 if (!sock->pipe_has_data && (group == NULL || sock->socket_has_data)) { 1425 /* If the user is receiving a sufficiently large amount of data, 1426 * receive directly to their buffers. */ 1427 len = 0; 1428 for (i = 0; i < iovcnt; i++) { 1429 len += iov[i].iov_len; 1430 } 1431 1432 if (len >= MIN_SOCK_PIPE_SIZE) { 1433 /* TODO: Should this detect if kernel socket is drained? */ 1434 if (sock->ssl) { 1435 return SSL_readv(sock->ssl, iov, iovcnt); 1436 } else { 1437 return readv(sock->fd, iov, iovcnt); 1438 } 1439 } 1440 1441 /* Otherwise, do a big read into our pipe */ 1442 rc = posix_sock_read(sock); 1443 if (rc <= 0) { 1444 return rc; 1445 } 1446 } 1447 1448 return posix_sock_recv_from_pipe(sock, iov, iovcnt); 1449 } 1450 1451 static ssize_t 1452 posix_sock_recv(struct spdk_sock *sock, void *buf, size_t len) 1453 { 1454 struct iovec iov[1]; 1455 1456 iov[0].iov_base = buf; 1457 iov[0].iov_len = len; 1458 1459 return posix_sock_readv(sock, iov, 1); 1460 } 1461 1462 static void 1463 posix_sock_readv_async(struct spdk_sock *sock, struct spdk_sock_request *req) 1464 { 1465 req->cb_fn(req->cb_arg, -ENOTSUP); 1466 } 1467 1468 static ssize_t 1469 posix_sock_writev(struct spdk_sock *_sock, struct iovec *iov, int iovcnt) 1470 { 1471 struct spdk_posix_sock *sock = __posix_sock(_sock); 1472 int rc; 1473 1474 /* In order to process a writev, we need to flush any asynchronous writes 1475 * first. */ 1476 rc = _sock_flush(_sock); 1477 if (rc < 0) { 1478 return rc; 1479 } 1480 1481 if (!TAILQ_EMPTY(&_sock->queued_reqs)) { 1482 /* We weren't able to flush all requests */ 1483 errno = EAGAIN; 1484 return -1; 1485 } 1486 1487 if (sock->ssl) { 1488 return SSL_writev(sock->ssl, iov, iovcnt); 1489 } else { 1490 return writev(sock->fd, iov, iovcnt); 1491 } 1492 } 1493 1494 static void 1495 posix_sock_writev_async(struct spdk_sock *sock, struct spdk_sock_request *req) 1496 { 1497 int rc; 1498 1499 spdk_sock_request_queue(sock, req); 1500 1501 /* If there are a sufficient number queued, just flush them out immediately. */ 1502 if (sock->queued_iovcnt >= IOV_BATCH_SIZE) { 1503 rc = _sock_flush(sock); 1504 if (rc) { 1505 spdk_sock_abort_requests(sock); 1506 } 1507 } 1508 } 1509 1510 static int 1511 posix_sock_set_recvlowat(struct spdk_sock *_sock, int nbytes) 1512 { 1513 struct spdk_posix_sock *sock = __posix_sock(_sock); 1514 int val; 1515 int rc; 1516 1517 assert(sock != NULL); 1518 1519 val = nbytes; 1520 rc = setsockopt(sock->fd, SOL_SOCKET, SO_RCVLOWAT, &val, sizeof val); 1521 if (rc != 0) { 1522 return -1; 1523 } 1524 return 0; 1525 } 1526 1527 static bool 1528 posix_sock_is_ipv6(struct spdk_sock *_sock) 1529 { 1530 struct spdk_posix_sock *sock = __posix_sock(_sock); 1531 struct sockaddr_storage sa; 1532 socklen_t salen; 1533 int rc; 1534 1535 assert(sock != NULL); 1536 1537 memset(&sa, 0, sizeof sa); 1538 salen = sizeof sa; 1539 rc = getsockname(sock->fd, (struct sockaddr *) &sa, &salen); 1540 if (rc != 0) { 1541 SPDK_ERRLOG("getsockname() failed (errno=%d)\n", errno); 1542 return false; 1543 } 1544 1545 return (sa.ss_family == AF_INET6); 1546 } 1547 1548 static bool 1549 posix_sock_is_ipv4(struct spdk_sock *_sock) 1550 { 1551 struct spdk_posix_sock *sock = __posix_sock(_sock); 1552 struct sockaddr_storage sa; 1553 socklen_t salen; 1554 int rc; 1555 1556 assert(sock != NULL); 1557 1558 memset(&sa, 0, sizeof sa); 1559 salen = sizeof sa; 1560 rc = getsockname(sock->fd, (struct sockaddr *) &sa, &salen); 1561 if (rc != 0) { 1562 SPDK_ERRLOG("getsockname() failed (errno=%d)\n", errno); 1563 return false; 1564 } 1565 1566 return (sa.ss_family == AF_INET); 1567 } 1568 1569 static bool 1570 posix_sock_is_connected(struct spdk_sock *_sock) 1571 { 1572 struct spdk_posix_sock *sock = __posix_sock(_sock); 1573 uint8_t byte; 1574 int rc; 1575 1576 rc = recv(sock->fd, &byte, 1, MSG_PEEK); 1577 if (rc == 0) { 1578 return false; 1579 } 1580 1581 if (rc < 0) { 1582 if (errno == EAGAIN || errno == EWOULDBLOCK) { 1583 return true; 1584 } 1585 1586 return false; 1587 } 1588 1589 return true; 1590 } 1591 1592 static struct spdk_sock_group_impl * 1593 posix_sock_group_impl_get_optimal(struct spdk_sock *_sock, struct spdk_sock_group_impl *hint) 1594 { 1595 struct spdk_posix_sock *sock = __posix_sock(_sock); 1596 struct spdk_sock_group_impl *group_impl; 1597 1598 if (sock->placement_id != -1) { 1599 spdk_sock_map_lookup(&g_map, sock->placement_id, &group_impl, hint); 1600 return group_impl; 1601 } 1602 1603 return NULL; 1604 } 1605 1606 static struct spdk_sock_group_impl * 1607 posix_sock_group_impl_create(void) 1608 { 1609 struct spdk_posix_sock_group_impl *group_impl; 1610 int fd; 1611 1612 #if defined(SPDK_EPOLL) 1613 fd = epoll_create1(0); 1614 #elif defined(SPDK_KEVENT) 1615 fd = kqueue(); 1616 #endif 1617 if (fd == -1) { 1618 return NULL; 1619 } 1620 1621 group_impl = calloc(1, sizeof(*group_impl)); 1622 if (group_impl == NULL) { 1623 SPDK_ERRLOG("group_impl allocation failed\n"); 1624 close(fd); 1625 return NULL; 1626 } 1627 1628 group_impl->fd = fd; 1629 TAILQ_INIT(&group_impl->socks_with_data); 1630 group_impl->placement_id = -1; 1631 1632 if (g_spdk_posix_sock_impl_opts.enable_placement_id == PLACEMENT_CPU) { 1633 spdk_sock_map_insert(&g_map, spdk_env_get_current_core(), &group_impl->base); 1634 group_impl->placement_id = spdk_env_get_current_core(); 1635 } 1636 1637 return &group_impl->base; 1638 } 1639 1640 static void 1641 posix_sock_mark(struct spdk_posix_sock_group_impl *group, struct spdk_posix_sock *sock, 1642 int placement_id) 1643 { 1644 #if defined(SO_MARK) 1645 int rc; 1646 1647 rc = setsockopt(sock->fd, SOL_SOCKET, SO_MARK, 1648 &placement_id, sizeof(placement_id)); 1649 if (rc != 0) { 1650 /* Not fatal */ 1651 SPDK_ERRLOG("Error setting SO_MARK\n"); 1652 return; 1653 } 1654 1655 rc = spdk_sock_map_insert(&g_map, placement_id, &group->base); 1656 if (rc != 0) { 1657 /* Not fatal */ 1658 SPDK_ERRLOG("Failed to insert sock group into map: %d\n", rc); 1659 return; 1660 } 1661 1662 sock->placement_id = placement_id; 1663 #endif 1664 } 1665 1666 static void 1667 posix_sock_update_mark(struct spdk_sock_group_impl *_group, struct spdk_sock *_sock) 1668 { 1669 struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group); 1670 1671 if (group->placement_id == -1) { 1672 group->placement_id = spdk_sock_map_find_free(&g_map); 1673 1674 /* If a free placement id is found, update existing sockets in this group */ 1675 if (group->placement_id != -1) { 1676 struct spdk_sock *sock, *tmp; 1677 1678 TAILQ_FOREACH_SAFE(sock, &_group->socks, link, tmp) { 1679 posix_sock_mark(group, __posix_sock(sock), group->placement_id); 1680 } 1681 } 1682 } 1683 1684 if (group->placement_id != -1) { 1685 /* 1686 * group placement id is already determined for this poll group. 1687 * Mark socket with group's placement id. 1688 */ 1689 posix_sock_mark(group, __posix_sock(_sock), group->placement_id); 1690 } 1691 } 1692 1693 static int 1694 posix_sock_group_impl_add_sock(struct spdk_sock_group_impl *_group, struct spdk_sock *_sock) 1695 { 1696 struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group); 1697 struct spdk_posix_sock *sock = __posix_sock(_sock); 1698 int rc; 1699 1700 #if defined(SPDK_EPOLL) 1701 struct epoll_event event; 1702 1703 memset(&event, 0, sizeof(event)); 1704 /* EPOLLERR is always on even if we don't set it, but be explicit for clarity */ 1705 event.events = EPOLLIN | EPOLLERR; 1706 event.data.ptr = sock; 1707 1708 rc = epoll_ctl(group->fd, EPOLL_CTL_ADD, sock->fd, &event); 1709 #elif defined(SPDK_KEVENT) 1710 struct kevent event; 1711 struct timespec ts = {0}; 1712 1713 EV_SET(&event, sock->fd, EVFILT_READ, EV_ADD, 0, 0, sock); 1714 1715 rc = kevent(group->fd, &event, 1, NULL, 0, &ts); 1716 #endif 1717 1718 if (rc != 0) { 1719 return rc; 1720 } 1721 1722 /* switched from another polling group due to scheduling */ 1723 if (spdk_unlikely(sock->recv_pipe != NULL && 1724 (spdk_pipe_reader_bytes_available(sock->recv_pipe) > 0))) { 1725 sock->pipe_has_data = true; 1726 sock->socket_has_data = false; 1727 TAILQ_INSERT_TAIL(&group->socks_with_data, sock, link); 1728 } 1729 1730 if (g_spdk_posix_sock_impl_opts.enable_placement_id == PLACEMENT_MARK) { 1731 posix_sock_update_mark(_group, _sock); 1732 } else if (sock->placement_id != -1) { 1733 rc = spdk_sock_map_insert(&g_map, sock->placement_id, &group->base); 1734 if (rc != 0) { 1735 SPDK_ERRLOG("Failed to insert sock group into map: %d\n", rc); 1736 /* Do not treat this as an error. The system will continue running. */ 1737 } 1738 } 1739 1740 return rc; 1741 } 1742 1743 static int 1744 posix_sock_group_impl_remove_sock(struct spdk_sock_group_impl *_group, struct spdk_sock *_sock) 1745 { 1746 struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group); 1747 struct spdk_posix_sock *sock = __posix_sock(_sock); 1748 int rc; 1749 1750 if (sock->pipe_has_data || sock->socket_has_data) { 1751 TAILQ_REMOVE(&group->socks_with_data, sock, link); 1752 sock->pipe_has_data = false; 1753 sock->socket_has_data = false; 1754 } 1755 1756 if (sock->placement_id != -1) { 1757 spdk_sock_map_release(&g_map, sock->placement_id); 1758 } 1759 1760 #if defined(SPDK_EPOLL) 1761 struct epoll_event event; 1762 1763 /* Event parameter is ignored but some old kernel version still require it. */ 1764 rc = epoll_ctl(group->fd, EPOLL_CTL_DEL, sock->fd, &event); 1765 #elif defined(SPDK_KEVENT) 1766 struct kevent event; 1767 struct timespec ts = {0}; 1768 1769 EV_SET(&event, sock->fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); 1770 1771 rc = kevent(group->fd, &event, 1, NULL, 0, &ts); 1772 if (rc == 0 && event.flags & EV_ERROR) { 1773 rc = -1; 1774 errno = event.data; 1775 } 1776 #endif 1777 1778 spdk_sock_abort_requests(_sock); 1779 1780 return rc; 1781 } 1782 1783 static int 1784 posix_sock_group_impl_poll(struct spdk_sock_group_impl *_group, int max_events, 1785 struct spdk_sock **socks) 1786 { 1787 struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group); 1788 struct spdk_sock *sock, *tmp; 1789 int num_events, i, rc; 1790 struct spdk_posix_sock *psock, *ptmp; 1791 #if defined(SPDK_EPOLL) 1792 struct epoll_event events[MAX_EVENTS_PER_POLL]; 1793 #elif defined(SPDK_KEVENT) 1794 struct kevent events[MAX_EVENTS_PER_POLL]; 1795 struct timespec ts = {0}; 1796 #endif 1797 1798 #ifdef SPDK_ZEROCOPY 1799 /* When all of the following conditions are met 1800 * - non-blocking socket 1801 * - zero copy is enabled 1802 * - interrupts suppressed (i.e. busy polling) 1803 * - the NIC tx queue is full at the time sendmsg() is called 1804 * - epoll_wait determines there is an EPOLLIN event for the socket 1805 * then we can get into a situation where data we've sent is queued 1806 * up in the kernel network stack, but interrupts have been suppressed 1807 * because other traffic is flowing so the kernel misses the signal 1808 * to flush the software tx queue. If there wasn't incoming data 1809 * pending on the socket, then epoll_wait would have been sufficient 1810 * to kick off the send operation, but since there is a pending event 1811 * epoll_wait does not trigger the necessary operation. 1812 * 1813 * We deal with this by checking for all of the above conditions and 1814 * additionally looking for EPOLLIN events that were not consumed from 1815 * the last poll loop. We take this to mean that the upper layer is 1816 * unable to consume them because it is blocked waiting for resources 1817 * to free up, and those resources are most likely freed in response 1818 * to a pending asynchronous write completing. 1819 * 1820 * Additionally, sockets that have the same placement_id actually share 1821 * an underlying hardware queue. That means polling one of them is 1822 * equivalent to polling all of them. As a quick mechanism to avoid 1823 * making extra poll() calls, stash the last placement_id during the loop 1824 * and only poll if it's not the same. The overwhelmingly common case 1825 * is that all sockets in this list have the same placement_id because 1826 * SPDK is intentionally grouping sockets by that value, so even 1827 * though this won't stop all extra calls to poll(), it's very fast 1828 * and will catch all of them in practice. 1829 */ 1830 int last_placement_id = -1; 1831 1832 TAILQ_FOREACH(psock, &group->socks_with_data, link) { 1833 if (psock->zcopy && psock->placement_id >= 0 && 1834 psock->placement_id != last_placement_id) { 1835 struct pollfd pfd = {psock->fd, POLLIN | POLLERR, 0}; 1836 1837 poll(&pfd, 1, 0); 1838 last_placement_id = psock->placement_id; 1839 } 1840 } 1841 #endif 1842 1843 /* This must be a TAILQ_FOREACH_SAFE because while flushing, 1844 * a completion callback could remove the sock from the 1845 * group. */ 1846 TAILQ_FOREACH_SAFE(sock, &_group->socks, link, tmp) { 1847 rc = _sock_flush(sock); 1848 if (rc) { 1849 spdk_sock_abort_requests(sock); 1850 } 1851 } 1852 1853 assert(max_events > 0); 1854 1855 #if defined(SPDK_EPOLL) 1856 num_events = epoll_wait(group->fd, events, max_events, 0); 1857 #elif defined(SPDK_KEVENT) 1858 num_events = kevent(group->fd, NULL, 0, events, max_events, &ts); 1859 #endif 1860 1861 if (num_events == -1) { 1862 return -1; 1863 } else if (num_events == 0 && !TAILQ_EMPTY(&_group->socks)) { 1864 sock = TAILQ_FIRST(&_group->socks); 1865 psock = __posix_sock(sock); 1866 /* poll() is called here to busy poll the queue associated with 1867 * first socket in list and potentially reap incoming data. 1868 */ 1869 if (sock->opts.priority) { 1870 struct pollfd pfd = {0, 0, 0}; 1871 1872 pfd.fd = psock->fd; 1873 pfd.events = POLLIN | POLLERR; 1874 poll(&pfd, 1, 0); 1875 } 1876 } 1877 1878 for (i = 0; i < num_events; i++) { 1879 #if defined(SPDK_EPOLL) 1880 sock = events[i].data.ptr; 1881 psock = __posix_sock(sock); 1882 1883 #ifdef SPDK_ZEROCOPY 1884 if (events[i].events & EPOLLERR) { 1885 rc = _sock_check_zcopy(sock); 1886 /* If the socket was closed or removed from 1887 * the group in response to a send ack, don't 1888 * add it to the array here. */ 1889 if (rc || sock->cb_fn == NULL) { 1890 continue; 1891 } 1892 } 1893 #endif 1894 if ((events[i].events & EPOLLIN) == 0) { 1895 continue; 1896 } 1897 1898 #elif defined(SPDK_KEVENT) 1899 sock = events[i].udata; 1900 psock = __posix_sock(sock); 1901 #endif 1902 1903 /* If the socket is not already in the list, add it now */ 1904 if (!psock->socket_has_data && !psock->pipe_has_data) { 1905 TAILQ_INSERT_TAIL(&group->socks_with_data, psock, link); 1906 } 1907 psock->socket_has_data = true; 1908 } 1909 1910 num_events = 0; 1911 1912 TAILQ_FOREACH_SAFE(psock, &group->socks_with_data, link, ptmp) { 1913 if (num_events == max_events) { 1914 break; 1915 } 1916 1917 /* If the socket's cb_fn is NULL, just remove it from the 1918 * list and do not add it to socks array */ 1919 if (spdk_unlikely(psock->base.cb_fn == NULL)) { 1920 psock->socket_has_data = false; 1921 psock->pipe_has_data = false; 1922 TAILQ_REMOVE(&group->socks_with_data, psock, link); 1923 continue; 1924 } 1925 1926 socks[num_events++] = &psock->base; 1927 } 1928 1929 /* Cycle the has_data list so that each time we poll things aren't 1930 * in the same order. Say we have 6 sockets in the list, named as follows: 1931 * A B C D E F 1932 * And all 6 sockets had epoll events, but max_events is only 3. That means 1933 * psock currently points at D. We want to rearrange the list to the following: 1934 * D E F A B C 1935 * 1936 * The variables below are named according to this example to make it easier to 1937 * follow the swaps. 1938 */ 1939 if (psock != NULL) { 1940 struct spdk_posix_sock *pa, *pc, *pd, *pf; 1941 1942 /* Capture pointers to the elements we need */ 1943 pd = psock; 1944 pc = TAILQ_PREV(pd, spdk_has_data_list, link); 1945 pa = TAILQ_FIRST(&group->socks_with_data); 1946 pf = TAILQ_LAST(&group->socks_with_data, spdk_has_data_list); 1947 1948 /* Break the link between C and D */ 1949 pc->link.tqe_next = NULL; 1950 1951 /* Connect F to A */ 1952 pf->link.tqe_next = pa; 1953 pa->link.tqe_prev = &pf->link.tqe_next; 1954 1955 /* Fix up the list first/last pointers */ 1956 group->socks_with_data.tqh_first = pd; 1957 group->socks_with_data.tqh_last = &pc->link.tqe_next; 1958 1959 /* D is in front of the list, make tqe prev pointer point to the head of list */ 1960 pd->link.tqe_prev = &group->socks_with_data.tqh_first; 1961 } 1962 1963 return num_events; 1964 } 1965 1966 static int 1967 posix_sock_group_impl_close(struct spdk_sock_group_impl *_group) 1968 { 1969 struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group); 1970 int rc; 1971 1972 if (g_spdk_posix_sock_impl_opts.enable_placement_id == PLACEMENT_CPU) { 1973 spdk_sock_map_release(&g_map, spdk_env_get_current_core()); 1974 } 1975 1976 rc = close(group->fd); 1977 free(group); 1978 return rc; 1979 } 1980 1981 static struct spdk_net_impl g_posix_net_impl = { 1982 .name = "posix", 1983 .getaddr = posix_sock_getaddr, 1984 .connect = posix_sock_connect, 1985 .listen = posix_sock_listen, 1986 .accept = posix_sock_accept, 1987 .close = posix_sock_close, 1988 .recv = posix_sock_recv, 1989 .readv = posix_sock_readv, 1990 .readv_async = posix_sock_readv_async, 1991 .writev = posix_sock_writev, 1992 .writev_async = posix_sock_writev_async, 1993 .flush = posix_sock_flush, 1994 .set_recvlowat = posix_sock_set_recvlowat, 1995 .set_recvbuf = posix_sock_set_recvbuf, 1996 .set_sendbuf = posix_sock_set_sendbuf, 1997 .is_ipv6 = posix_sock_is_ipv6, 1998 .is_ipv4 = posix_sock_is_ipv4, 1999 .is_connected = posix_sock_is_connected, 2000 .group_impl_get_optimal = posix_sock_group_impl_get_optimal, 2001 .group_impl_create = posix_sock_group_impl_create, 2002 .group_impl_add_sock = posix_sock_group_impl_add_sock, 2003 .group_impl_remove_sock = posix_sock_group_impl_remove_sock, 2004 .group_impl_poll = posix_sock_group_impl_poll, 2005 .group_impl_close = posix_sock_group_impl_close, 2006 .get_opts = posix_sock_impl_get_opts, 2007 .set_opts = posix_sock_impl_set_opts, 2008 }; 2009 2010 SPDK_NET_IMPL_REGISTER(posix, &g_posix_net_impl, DEFAULT_SOCK_PRIORITY + 1); 2011 2012 static struct spdk_sock * 2013 ssl_sock_listen(const char *ip, int port, struct spdk_sock_opts *opts) 2014 { 2015 return posix_sock_create(ip, port, SPDK_SOCK_CREATE_LISTEN, opts, true); 2016 } 2017 2018 static struct spdk_sock * 2019 ssl_sock_connect(const char *ip, int port, struct spdk_sock_opts *opts) 2020 { 2021 return posix_sock_create(ip, port, SPDK_SOCK_CREATE_CONNECT, opts, true); 2022 } 2023 2024 static struct spdk_net_impl g_ssl_net_impl = { 2025 .name = "ssl", 2026 .getaddr = posix_sock_getaddr, 2027 .connect = ssl_sock_connect, 2028 .listen = ssl_sock_listen, 2029 .accept = posix_sock_accept, 2030 .close = posix_sock_close, 2031 .recv = posix_sock_recv, 2032 .readv = posix_sock_readv, 2033 .writev = posix_sock_writev, 2034 .writev_async = posix_sock_writev_async, 2035 .flush = posix_sock_flush, 2036 .set_recvlowat = posix_sock_set_recvlowat, 2037 .set_recvbuf = posix_sock_set_recvbuf, 2038 .set_sendbuf = posix_sock_set_sendbuf, 2039 .is_ipv6 = posix_sock_is_ipv6, 2040 .is_ipv4 = posix_sock_is_ipv4, 2041 .is_connected = posix_sock_is_connected, 2042 .group_impl_get_optimal = posix_sock_group_impl_get_optimal, 2043 .group_impl_create = posix_sock_group_impl_create, 2044 .group_impl_add_sock = posix_sock_group_impl_add_sock, 2045 .group_impl_remove_sock = posix_sock_group_impl_remove_sock, 2046 .group_impl_poll = posix_sock_group_impl_poll, 2047 .group_impl_close = posix_sock_group_impl_close, 2048 .get_opts = posix_sock_impl_get_opts, 2049 .set_opts = posix_sock_impl_set_opts, 2050 }; 2051 2052 SPDK_NET_IMPL_REGISTER(ssl, &g_ssl_net_impl, DEFAULT_SOCK_PRIORITY); 2053 SPDK_LOG_REGISTER_COMPONENT(sock_posix) 2054