1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. All rights reserved. 3 * Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #if defined(__FreeBSD__) 10 #include <sys/event.h> 11 #define SPDK_KEVENT 12 #else 13 #include <sys/epoll.h> 14 #define SPDK_EPOLL 15 #endif 16 17 #if defined(__linux__) 18 #include <linux/errqueue.h> 19 #endif 20 21 #include "spdk/env.h" 22 #include "spdk/log.h" 23 #include "spdk/pipe.h" 24 #include "spdk/sock.h" 25 #include "spdk/util.h" 26 #include "spdk/string.h" 27 #include "spdk_internal/sock.h" 28 #include "../sock_kernel.h" 29 30 #include "openssl/crypto.h" 31 #include "openssl/err.h" 32 #include "openssl/ssl.h" 33 34 #define MAX_TMPBUF 1024 35 #define PORTNUMLEN 32 36 37 #if defined(SO_ZEROCOPY) && defined(MSG_ZEROCOPY) 38 #define SPDK_ZEROCOPY 39 #endif 40 41 struct spdk_posix_sock { 42 struct spdk_sock base; 43 int fd; 44 45 uint32_t sendmsg_idx; 46 47 struct spdk_pipe *recv_pipe; 48 void *recv_buf; 49 int recv_buf_sz; 50 bool pipe_has_data; 51 bool socket_has_data; 52 bool zcopy; 53 54 int placement_id; 55 56 SSL_CTX *ctx; 57 SSL *ssl; 58 59 TAILQ_ENTRY(spdk_posix_sock) link; 60 }; 61 62 TAILQ_HEAD(spdk_has_data_list, spdk_posix_sock); 63 64 struct spdk_posix_sock_group_impl { 65 struct spdk_sock_group_impl base; 66 int fd; 67 struct spdk_has_data_list socks_with_data; 68 int placement_id; 69 }; 70 71 static struct spdk_sock_impl_opts g_spdk_posix_sock_impl_opts = { 72 .recv_buf_size = MIN_SO_RCVBUF_SIZE, 73 .send_buf_size = MIN_SO_SNDBUF_SIZE, 74 .enable_recv_pipe = true, 75 .enable_quickack = false, 76 .enable_placement_id = PLACEMENT_NONE, 77 .enable_zerocopy_send_server = true, 78 .enable_zerocopy_send_client = false, 79 .zerocopy_threshold = 0, 80 .tls_version = 0, 81 .enable_ktls = false, 82 .psk_key = NULL, 83 .psk_identity = NULL 84 }; 85 86 static struct spdk_sock_map g_map = { 87 .entries = STAILQ_HEAD_INITIALIZER(g_map.entries), 88 .mtx = PTHREAD_MUTEX_INITIALIZER 89 }; 90 91 __attribute((destructor)) static void 92 posix_sock_map_cleanup(void) 93 { 94 spdk_sock_map_cleanup(&g_map); 95 } 96 97 #define __posix_sock(sock) (struct spdk_posix_sock *)sock 98 #define __posix_group_impl(group) (struct spdk_posix_sock_group_impl *)group 99 100 static void 101 posix_sock_copy_impl_opts(struct spdk_sock_impl_opts *dest, const struct spdk_sock_impl_opts *src, 102 size_t len) 103 { 104 #define FIELD_OK(field) \ 105 offsetof(struct spdk_sock_impl_opts, field) + sizeof(src->field) <= len 106 107 #define SET_FIELD(field) \ 108 if (FIELD_OK(field)) { \ 109 dest->field = src->field; \ 110 } 111 112 SET_FIELD(recv_buf_size); 113 SET_FIELD(send_buf_size); 114 SET_FIELD(enable_recv_pipe); 115 SET_FIELD(enable_zerocopy_send); 116 SET_FIELD(enable_quickack); 117 SET_FIELD(enable_placement_id); 118 SET_FIELD(enable_zerocopy_send_server); 119 SET_FIELD(enable_zerocopy_send_client); 120 SET_FIELD(zerocopy_threshold); 121 SET_FIELD(tls_version); 122 SET_FIELD(enable_ktls); 123 SET_FIELD(psk_key); 124 SET_FIELD(psk_identity); 125 126 #undef SET_FIELD 127 #undef FIELD_OK 128 } 129 130 static int 131 posix_sock_impl_get_opts(struct spdk_sock_impl_opts *opts, size_t *len) 132 { 133 if (!opts || !len) { 134 errno = EINVAL; 135 return -1; 136 } 137 138 assert(sizeof(*opts) >= *len); 139 memset(opts, 0, *len); 140 141 posix_sock_copy_impl_opts(opts, &g_spdk_posix_sock_impl_opts, *len); 142 *len = spdk_min(*len, sizeof(g_spdk_posix_sock_impl_opts)); 143 144 return 0; 145 } 146 147 static int 148 posix_sock_impl_set_opts(const struct spdk_sock_impl_opts *opts, size_t len) 149 { 150 if (!opts) { 151 errno = EINVAL; 152 return -1; 153 } 154 155 assert(sizeof(*opts) >= len); 156 posix_sock_copy_impl_opts(&g_spdk_posix_sock_impl_opts, opts, len); 157 158 return 0; 159 } 160 161 static void 162 posix_opts_get_impl_opts(const struct spdk_sock_opts *opts, struct spdk_sock_impl_opts *dest) 163 { 164 /* Copy the default impl_opts first to cover cases when user's impl_opts is smaller */ 165 memcpy(dest, &g_spdk_posix_sock_impl_opts, sizeof(*dest)); 166 167 if (opts->impl_opts != NULL) { 168 assert(sizeof(*dest) >= opts->impl_opts_size); 169 posix_sock_copy_impl_opts(dest, opts->impl_opts, opts->impl_opts_size); 170 } 171 } 172 173 static int 174 posix_sock_getaddr(struct spdk_sock *_sock, char *saddr, int slen, uint16_t *sport, 175 char *caddr, int clen, uint16_t *cport) 176 { 177 struct spdk_posix_sock *sock = __posix_sock(_sock); 178 struct sockaddr_storage sa; 179 socklen_t salen; 180 int rc; 181 182 assert(sock != NULL); 183 184 memset(&sa, 0, sizeof sa); 185 salen = sizeof sa; 186 rc = getsockname(sock->fd, (struct sockaddr *) &sa, &salen); 187 if (rc != 0) { 188 SPDK_ERRLOG("getsockname() failed (errno=%d)\n", errno); 189 return -1; 190 } 191 192 switch (sa.ss_family) { 193 case AF_UNIX: 194 /* Acceptable connection types that don't have IPs */ 195 return 0; 196 case AF_INET: 197 case AF_INET6: 198 /* Code below will get IP addresses */ 199 break; 200 default: 201 /* Unsupported socket family */ 202 return -1; 203 } 204 205 rc = get_addr_str((struct sockaddr *)&sa, saddr, slen); 206 if (rc != 0) { 207 SPDK_ERRLOG("getnameinfo() failed (errno=%d)\n", errno); 208 return -1; 209 } 210 211 if (sport) { 212 if (sa.ss_family == AF_INET) { 213 *sport = ntohs(((struct sockaddr_in *) &sa)->sin_port); 214 } else if (sa.ss_family == AF_INET6) { 215 *sport = ntohs(((struct sockaddr_in6 *) &sa)->sin6_port); 216 } 217 } 218 219 memset(&sa, 0, sizeof sa); 220 salen = sizeof sa; 221 rc = getpeername(sock->fd, (struct sockaddr *) &sa, &salen); 222 if (rc != 0) { 223 SPDK_ERRLOG("getpeername() failed (errno=%d)\n", errno); 224 return -1; 225 } 226 227 rc = get_addr_str((struct sockaddr *)&sa, caddr, clen); 228 if (rc != 0) { 229 SPDK_ERRLOG("getnameinfo() failed (errno=%d)\n", errno); 230 return -1; 231 } 232 233 if (cport) { 234 if (sa.ss_family == AF_INET) { 235 *cport = ntohs(((struct sockaddr_in *) &sa)->sin_port); 236 } else if (sa.ss_family == AF_INET6) { 237 *cport = ntohs(((struct sockaddr_in6 *) &sa)->sin6_port); 238 } 239 } 240 241 return 0; 242 } 243 244 enum posix_sock_create_type { 245 SPDK_SOCK_CREATE_LISTEN, 246 SPDK_SOCK_CREATE_CONNECT, 247 }; 248 249 static int 250 posix_sock_alloc_pipe(struct spdk_posix_sock *sock, int sz) 251 { 252 uint8_t *new_buf; 253 struct spdk_pipe *new_pipe; 254 struct iovec siov[2]; 255 struct iovec diov[2]; 256 int sbytes; 257 ssize_t bytes; 258 259 if (sock->recv_buf_sz == sz) { 260 return 0; 261 } 262 263 /* If the new size is 0, just free the pipe */ 264 if (sz == 0) { 265 spdk_pipe_destroy(sock->recv_pipe); 266 free(sock->recv_buf); 267 sock->recv_pipe = NULL; 268 sock->recv_buf = NULL; 269 return 0; 270 } else if (sz < MIN_SOCK_PIPE_SIZE) { 271 SPDK_ERRLOG("The size of the pipe must be larger than %d\n", MIN_SOCK_PIPE_SIZE); 272 return -1; 273 } 274 275 /* Round up to next 64 byte multiple */ 276 new_buf = calloc(SPDK_ALIGN_CEIL(sz + 1, 64), sizeof(uint8_t)); 277 if (!new_buf) { 278 SPDK_ERRLOG("socket recv buf allocation failed\n"); 279 return -ENOMEM; 280 } 281 282 new_pipe = spdk_pipe_create(new_buf, sz + 1); 283 if (new_pipe == NULL) { 284 SPDK_ERRLOG("socket pipe allocation failed\n"); 285 free(new_buf); 286 return -ENOMEM; 287 } 288 289 if (sock->recv_pipe != NULL) { 290 /* Pull all of the data out of the old pipe */ 291 sbytes = spdk_pipe_reader_get_buffer(sock->recv_pipe, sock->recv_buf_sz, siov); 292 if (sbytes > sz) { 293 /* Too much data to fit into the new pipe size */ 294 spdk_pipe_destroy(new_pipe); 295 free(new_buf); 296 return -EINVAL; 297 } 298 299 sbytes = spdk_pipe_writer_get_buffer(new_pipe, sz, diov); 300 assert(sbytes == sz); 301 302 bytes = spdk_iovcpy(siov, 2, diov, 2); 303 spdk_pipe_writer_advance(new_pipe, bytes); 304 305 spdk_pipe_destroy(sock->recv_pipe); 306 free(sock->recv_buf); 307 } 308 309 sock->recv_buf_sz = sz; 310 sock->recv_buf = new_buf; 311 sock->recv_pipe = new_pipe; 312 313 return 0; 314 } 315 316 static int 317 posix_sock_set_recvbuf(struct spdk_sock *_sock, int sz) 318 { 319 struct spdk_posix_sock *sock = __posix_sock(_sock); 320 int rc; 321 322 assert(sock != NULL); 323 324 if (_sock->impl_opts.enable_recv_pipe) { 325 rc = posix_sock_alloc_pipe(sock, sz); 326 if (rc) { 327 return rc; 328 } 329 } 330 331 /* Set kernel buffer size to be at least MIN_SO_RCVBUF_SIZE */ 332 if (sz < MIN_SO_RCVBUF_SIZE) { 333 sz = MIN_SO_RCVBUF_SIZE; 334 } 335 336 rc = setsockopt(sock->fd, SOL_SOCKET, SO_RCVBUF, &sz, sizeof(sz)); 337 if (rc < 0) { 338 return rc; 339 } 340 341 _sock->impl_opts.recv_buf_size = sz; 342 343 return 0; 344 } 345 346 static int 347 posix_sock_set_sendbuf(struct spdk_sock *_sock, int sz) 348 { 349 struct spdk_posix_sock *sock = __posix_sock(_sock); 350 int rc; 351 352 assert(sock != NULL); 353 354 if (sz < MIN_SO_SNDBUF_SIZE) { 355 sz = MIN_SO_SNDBUF_SIZE; 356 } 357 358 rc = setsockopt(sock->fd, SOL_SOCKET, SO_SNDBUF, &sz, sizeof(sz)); 359 if (rc < 0) { 360 return rc; 361 } 362 363 _sock->impl_opts.send_buf_size = sz; 364 365 return 0; 366 } 367 368 static void 369 posix_sock_init(struct spdk_posix_sock *sock, bool enable_zero_copy) 370 { 371 #if defined(SPDK_ZEROCOPY) || defined(__linux__) 372 int flag; 373 int rc; 374 #endif 375 376 #if defined(SPDK_ZEROCOPY) 377 flag = 1; 378 379 if (enable_zero_copy) { 380 /* Try to turn on zero copy sends */ 381 rc = setsockopt(sock->fd, SOL_SOCKET, SO_ZEROCOPY, &flag, sizeof(flag)); 382 if (rc == 0) { 383 sock->zcopy = true; 384 } 385 } 386 #endif 387 388 #if defined(__linux__) 389 flag = 1; 390 391 if (sock->base.impl_opts.enable_quickack) { 392 rc = setsockopt(sock->fd, IPPROTO_TCP, TCP_QUICKACK, &flag, sizeof(flag)); 393 if (rc != 0) { 394 SPDK_ERRLOG("quickack was failed to set\n"); 395 } 396 } 397 398 spdk_sock_get_placement_id(sock->fd, sock->base.impl_opts.enable_placement_id, 399 &sock->placement_id); 400 401 if (sock->base.impl_opts.enable_placement_id == PLACEMENT_MARK) { 402 /* Save placement_id */ 403 spdk_sock_map_insert(&g_map, sock->placement_id, NULL); 404 } 405 #endif 406 } 407 408 static struct spdk_posix_sock * 409 posix_sock_alloc(int fd, struct spdk_sock_impl_opts *impl_opts, bool enable_zero_copy) 410 { 411 struct spdk_posix_sock *sock; 412 413 sock = calloc(1, sizeof(*sock)); 414 if (sock == NULL) { 415 SPDK_ERRLOG("sock allocation failed\n"); 416 return NULL; 417 } 418 419 sock->fd = fd; 420 memcpy(&sock->base.impl_opts, impl_opts, sizeof(*impl_opts)); 421 posix_sock_init(sock, enable_zero_copy); 422 423 return sock; 424 } 425 426 static int 427 posix_fd_create(struct addrinfo *res, struct spdk_sock_opts *opts, 428 struct spdk_sock_impl_opts *impl_opts) 429 { 430 int fd; 431 int val = 1; 432 int rc, sz; 433 #if defined(__linux__) 434 int to; 435 #endif 436 437 fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol); 438 if (fd < 0) { 439 /* error */ 440 return -1; 441 } 442 443 sz = impl_opts->recv_buf_size; 444 rc = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &sz, sizeof(sz)); 445 if (rc) { 446 /* Not fatal */ 447 } 448 449 sz = impl_opts->send_buf_size; 450 rc = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &sz, sizeof(sz)); 451 if (rc) { 452 /* Not fatal */ 453 } 454 455 rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof val); 456 if (rc != 0) { 457 close(fd); 458 /* error */ 459 return -1; 460 } 461 rc = setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof val); 462 if (rc != 0) { 463 close(fd); 464 /* error */ 465 return -1; 466 } 467 468 #if defined(SO_PRIORITY) 469 if (opts->priority) { 470 rc = setsockopt(fd, SOL_SOCKET, SO_PRIORITY, &opts->priority, sizeof val); 471 if (rc != 0) { 472 close(fd); 473 /* error */ 474 return -1; 475 } 476 } 477 #endif 478 479 if (res->ai_family == AF_INET6) { 480 rc = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &val, sizeof val); 481 if (rc != 0) { 482 close(fd); 483 /* error */ 484 return -1; 485 } 486 } 487 488 if (opts->ack_timeout) { 489 #if defined(__linux__) 490 to = opts->ack_timeout; 491 rc = setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, &to, sizeof(to)); 492 if (rc != 0) { 493 close(fd); 494 /* error */ 495 return -1; 496 } 497 #else 498 SPDK_WARNLOG("TCP_USER_TIMEOUT is not supported.\n"); 499 #endif 500 } 501 502 return fd; 503 } 504 505 static unsigned int 506 posix_sock_tls_psk_server_cb(SSL *ssl, 507 const char *id, 508 unsigned char *psk, 509 unsigned int max_psk_len) 510 { 511 long key_len; 512 unsigned char *default_psk; 513 struct spdk_sock_impl_opts *impl_opts; 514 515 impl_opts = SSL_get_app_data(ssl); 516 517 if (impl_opts->psk_key == NULL) { 518 SPDK_ERRLOG("PSK is not set\n"); 519 goto err; 520 } 521 SPDK_DEBUGLOG(sock_posix, "Length of Client's PSK ID %lu\n", strlen(impl_opts->psk_identity)); 522 if (id == NULL) { 523 SPDK_ERRLOG("Received empty PSK ID\n"); 524 goto err; 525 } 526 SPDK_DEBUGLOG(sock_posix, "Received PSK ID '%s'\n", id); 527 if (strcmp(impl_opts->psk_identity, id) != 0) { 528 SPDK_ERRLOG("Unknown Client's PSK ID\n"); 529 goto err; 530 } 531 532 SPDK_DEBUGLOG(sock_posix, "Length of Client's PSK KEY %u\n", max_psk_len); 533 default_psk = OPENSSL_hexstr2buf(impl_opts->psk_key, &key_len); 534 if (default_psk == NULL) { 535 SPDK_ERRLOG("Could not unhexlify PSK\n"); 536 goto err; 537 } 538 if (key_len > max_psk_len) { 539 SPDK_ERRLOG("Insufficient buffer size to copy PSK\n"); 540 goto err; 541 } 542 543 memcpy(psk, default_psk, key_len); 544 545 return key_len; 546 547 err: 548 return 0; 549 } 550 551 static unsigned int 552 posix_sock_tls_psk_client_cb(SSL *ssl, const char *hint, 553 char *identity, 554 unsigned int max_identity_len, 555 unsigned char *psk, 556 unsigned int max_psk_len) 557 { 558 long key_len; 559 unsigned char *default_psk; 560 struct spdk_sock_impl_opts *impl_opts; 561 562 impl_opts = SSL_get_app_data(ssl); 563 564 if (hint) { 565 SPDK_DEBUGLOG(sock_posix, "Received PSK identity hint '%s'\n", hint); 566 } 567 568 if (impl_opts->psk_key == NULL) { 569 SPDK_ERRLOG("PSK is not set\n"); 570 goto err; 571 } 572 default_psk = OPENSSL_hexstr2buf(impl_opts->psk_key, &key_len); 573 if (default_psk == NULL) { 574 SPDK_ERRLOG("Could not unhexlify PSK\n"); 575 goto err; 576 } 577 if ((strlen(impl_opts->psk_identity) + 1 > max_identity_len) 578 || (key_len > max_psk_len)) { 579 SPDK_ERRLOG("PSK ID or Key buffer is not sufficient\n"); 580 goto err; 581 } 582 spdk_strcpy_pad(identity, impl_opts->psk_identity, strlen(impl_opts->psk_identity), 0); 583 SPDK_DEBUGLOG(sock_posix, "Sending PSK identity '%s'\n", identity); 584 585 memcpy(psk, default_psk, key_len); 586 SPDK_DEBUGLOG(sock_posix, "Provided out-of-band (OOB) PSK for TLS1.3 client\n"); 587 588 return key_len; 589 590 err: 591 return 0; 592 } 593 594 static SSL_CTX * 595 posix_sock_create_ssl_context(const SSL_METHOD *method, struct spdk_sock_opts *opts, 596 struct spdk_sock_impl_opts *impl_opts) 597 { 598 SSL_CTX *ctx; 599 int tls_version = 0; 600 bool ktls_enabled = false; 601 #ifdef SSL_OP_ENABLE_KTLS 602 long options; 603 #endif 604 605 SSL_library_init(); 606 OpenSSL_add_all_algorithms(); 607 SSL_load_error_strings(); 608 /* Produce a SSL CTX in SSL V2 and V3 standards compliant way */ 609 ctx = SSL_CTX_new(method); 610 if (!ctx) { 611 SPDK_ERRLOG("SSL_CTX_new() failed, msg = %s\n", ERR_error_string(ERR_peek_last_error(), NULL)); 612 return NULL; 613 } 614 SPDK_DEBUGLOG(sock_posix, "SSL context created\n"); 615 616 switch (impl_opts->tls_version) { 617 case 0: 618 /* auto-negotioation */ 619 break; 620 case SPDK_TLS_VERSION_1_1: 621 tls_version = TLS1_1_VERSION; 622 break; 623 case SPDK_TLS_VERSION_1_2: 624 tls_version = TLS1_2_VERSION; 625 break; 626 case SPDK_TLS_VERSION_1_3: 627 tls_version = TLS1_3_VERSION; 628 break; 629 default: 630 SPDK_ERRLOG("Incorrect TLS version provided: %d\n", impl_opts->tls_version); 631 goto err; 632 } 633 634 if (tls_version) { 635 SPDK_DEBUGLOG(sock_posix, "Hardening TLS version to '%d'='0x%X'\n", impl_opts->tls_version, 636 tls_version); 637 if (!SSL_CTX_set_min_proto_version(ctx, tls_version)) { 638 SPDK_ERRLOG("Unable to set Min TLS version to '%d'='0x%X\n", impl_opts->tls_version, tls_version); 639 goto err; 640 } 641 if (!SSL_CTX_set_max_proto_version(ctx, tls_version)) { 642 SPDK_ERRLOG("Unable to set Max TLS version to '%d'='0x%X\n", impl_opts->tls_version, tls_version); 643 goto err; 644 } 645 } 646 if (impl_opts->enable_ktls) { 647 SPDK_DEBUGLOG(sock_posix, "Enabling kTLS offload\n"); 648 #ifdef SSL_OP_ENABLE_KTLS 649 options = SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS); 650 ktls_enabled = options & SSL_OP_ENABLE_KTLS; 651 #else 652 ktls_enabled = false; 653 #endif 654 if (!ktls_enabled) { 655 SPDK_ERRLOG("Unable to set kTLS offload via SSL_CTX_set_options(). Configure openssl with 'enable-ktls'\n"); 656 goto err; 657 } 658 } 659 660 return ctx; 661 662 err: 663 SSL_CTX_free(ctx); 664 return NULL; 665 } 666 667 static SSL * 668 ssl_sock_connect_loop(SSL_CTX *ctx, int fd, struct spdk_sock_impl_opts *impl_opts) 669 { 670 int rc; 671 SSL *ssl; 672 int ssl_get_error; 673 674 ssl = SSL_new(ctx); 675 if (!ssl) { 676 SPDK_ERRLOG("SSL_new() failed, msg = %s\n", ERR_error_string(ERR_peek_last_error(), NULL)); 677 return NULL; 678 } 679 SSL_set_fd(ssl, fd); 680 SSL_set_app_data(ssl, impl_opts); 681 SSL_set_psk_client_callback(ssl, posix_sock_tls_psk_client_cb); 682 SPDK_DEBUGLOG(sock_posix, "SSL object creation finished: %p\n", ssl); 683 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 684 while ((rc = SSL_connect(ssl)) != 1) { 685 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 686 ssl_get_error = SSL_get_error(ssl, rc); 687 SPDK_DEBUGLOG(sock_posix, "SSL_connect failed %d = SSL_connect(%p), %d = SSL_get_error(%p, %d)\n", 688 rc, ssl, ssl_get_error, ssl, rc); 689 switch (ssl_get_error) { 690 case SSL_ERROR_WANT_READ: 691 case SSL_ERROR_WANT_WRITE: 692 continue; 693 default: 694 break; 695 } 696 SPDK_ERRLOG("SSL_connect() failed, errno = %d\n", errno); 697 SSL_free(ssl); 698 return NULL; 699 } 700 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 701 SPDK_DEBUGLOG(sock_posix, "Negotiated Cipher suite:%s\n", 702 SSL_CIPHER_get_name(SSL_get_current_cipher(ssl))); 703 return ssl; 704 } 705 706 static SSL * 707 ssl_sock_accept_loop(SSL_CTX *ctx, int fd, struct spdk_sock_impl_opts *impl_opts) 708 { 709 int rc; 710 SSL *ssl; 711 int ssl_get_error; 712 713 ssl = SSL_new(ctx); 714 if (!ssl) { 715 SPDK_ERRLOG("SSL_new() failed, msg = %s\n", ERR_error_string(ERR_peek_last_error(), NULL)); 716 return NULL; 717 } 718 SSL_set_fd(ssl, fd); 719 SSL_set_app_data(ssl, impl_opts); 720 SSL_set_psk_server_callback(ssl, posix_sock_tls_psk_server_cb); 721 SPDK_DEBUGLOG(sock_posix, "SSL object creation finished: %p\n", ssl); 722 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 723 while ((rc = SSL_accept(ssl)) != 1) { 724 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 725 ssl_get_error = SSL_get_error(ssl, rc); 726 SPDK_DEBUGLOG(sock_posix, "SSL_accept failed %d = SSL_accept(%p), %d = SSL_get_error(%p, %d)\n", rc, 727 ssl, ssl_get_error, ssl, rc); 728 switch (ssl_get_error) { 729 case SSL_ERROR_WANT_READ: 730 case SSL_ERROR_WANT_WRITE: 731 continue; 732 default: 733 break; 734 } 735 SPDK_ERRLOG("SSL_accept() failed, errno = %d\n", errno); 736 SSL_free(ssl); 737 return NULL; 738 } 739 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 740 SPDK_DEBUGLOG(sock_posix, "Negotiated Cipher suite:%s\n", 741 SSL_CIPHER_get_name(SSL_get_current_cipher(ssl))); 742 return ssl; 743 } 744 745 static ssize_t 746 SSL_readv(SSL *ssl, const struct iovec *iov, int iovcnt) 747 { 748 int i, rc = 0; 749 ssize_t total = 0; 750 751 for (i = 0; i < iovcnt; i++) { 752 rc = SSL_read(ssl, iov[i].iov_base, iov[i].iov_len); 753 754 if (rc > 0) { 755 total += rc; 756 } 757 if (rc != (int)iov[i].iov_len) { 758 break; 759 } 760 } 761 if (total > 0) { 762 errno = 0; 763 return total; 764 } 765 switch (SSL_get_error(ssl, rc)) { 766 case SSL_ERROR_ZERO_RETURN: 767 errno = ENOTCONN; 768 return 0; 769 case SSL_ERROR_WANT_READ: 770 case SSL_ERROR_WANT_WRITE: 771 case SSL_ERROR_WANT_CONNECT: 772 case SSL_ERROR_WANT_ACCEPT: 773 case SSL_ERROR_WANT_X509_LOOKUP: 774 case SSL_ERROR_WANT_ASYNC: 775 case SSL_ERROR_WANT_ASYNC_JOB: 776 case SSL_ERROR_WANT_CLIENT_HELLO_CB: 777 errno = EAGAIN; 778 return -1; 779 case SSL_ERROR_SYSCALL: 780 case SSL_ERROR_SSL: 781 errno = ENOTCONN; 782 return -1; 783 default: 784 errno = ENOTCONN; 785 return -1; 786 } 787 } 788 789 static ssize_t 790 SSL_writev(SSL *ssl, struct iovec *iov, int iovcnt) 791 { 792 int i, rc = 0; 793 ssize_t total = 0; 794 795 for (i = 0; i < iovcnt; i++) { 796 rc = SSL_write(ssl, iov[i].iov_base, iov[i].iov_len); 797 798 if (rc > 0) { 799 total += rc; 800 } 801 if (rc != (int)iov[i].iov_len) { 802 break; 803 } 804 } 805 if (total > 0) { 806 errno = 0; 807 return total; 808 } 809 switch (SSL_get_error(ssl, rc)) { 810 case SSL_ERROR_ZERO_RETURN: 811 errno = ENOTCONN; 812 return 0; 813 case SSL_ERROR_WANT_READ: 814 case SSL_ERROR_WANT_WRITE: 815 case SSL_ERROR_WANT_CONNECT: 816 case SSL_ERROR_WANT_ACCEPT: 817 case SSL_ERROR_WANT_X509_LOOKUP: 818 case SSL_ERROR_WANT_ASYNC: 819 case SSL_ERROR_WANT_ASYNC_JOB: 820 case SSL_ERROR_WANT_CLIENT_HELLO_CB: 821 errno = EAGAIN; 822 return -1; 823 case SSL_ERROR_SYSCALL: 824 case SSL_ERROR_SSL: 825 errno = ENOTCONN; 826 return -1; 827 default: 828 errno = ENOTCONN; 829 return -1; 830 } 831 } 832 833 static struct spdk_sock * 834 posix_sock_create(const char *ip, int port, 835 enum posix_sock_create_type type, 836 struct spdk_sock_opts *opts, 837 bool enable_ssl) 838 { 839 struct spdk_posix_sock *sock; 840 struct spdk_sock_impl_opts impl_opts; 841 char buf[MAX_TMPBUF]; 842 char portnum[PORTNUMLEN]; 843 char *p; 844 struct addrinfo hints, *res, *res0; 845 int fd, flag; 846 int rc; 847 bool enable_zcopy_user_opts = true; 848 bool enable_zcopy_impl_opts = true; 849 SSL_CTX *ctx = 0; 850 SSL *ssl = 0; 851 852 assert(opts != NULL); 853 posix_opts_get_impl_opts(opts, &impl_opts); 854 855 if (ip == NULL) { 856 return NULL; 857 } 858 if (ip[0] == '[') { 859 snprintf(buf, sizeof(buf), "%s", ip + 1); 860 p = strchr(buf, ']'); 861 if (p != NULL) { 862 *p = '\0'; 863 } 864 ip = (const char *) &buf[0]; 865 } 866 867 snprintf(portnum, sizeof portnum, "%d", port); 868 memset(&hints, 0, sizeof hints); 869 hints.ai_family = PF_UNSPEC; 870 hints.ai_socktype = SOCK_STREAM; 871 hints.ai_flags = AI_NUMERICSERV; 872 hints.ai_flags |= AI_PASSIVE; 873 hints.ai_flags |= AI_NUMERICHOST; 874 rc = getaddrinfo(ip, portnum, &hints, &res0); 875 if (rc != 0) { 876 SPDK_ERRLOG("getaddrinfo() failed %s (%d)\n", gai_strerror(rc), rc); 877 return NULL; 878 } 879 880 /* try listen */ 881 fd = -1; 882 for (res = res0; res != NULL; res = res->ai_next) { 883 retry: 884 fd = posix_fd_create(res, opts, &impl_opts); 885 if (fd < 0) { 886 continue; 887 } 888 if (type == SPDK_SOCK_CREATE_LISTEN) { 889 rc = bind(fd, res->ai_addr, res->ai_addrlen); 890 if (rc != 0) { 891 SPDK_ERRLOG("bind() failed at port %d, errno = %d\n", port, errno); 892 switch (errno) { 893 case EINTR: 894 /* interrupted? */ 895 close(fd); 896 goto retry; 897 case EADDRNOTAVAIL: 898 SPDK_ERRLOG("IP address %s not available. " 899 "Verify IP address in config file " 900 "and make sure setup script is " 901 "run before starting spdk app.\n", ip); 902 /* FALLTHROUGH */ 903 default: 904 /* try next family */ 905 close(fd); 906 fd = -1; 907 continue; 908 } 909 } 910 /* bind OK */ 911 rc = listen(fd, 512); 912 if (rc != 0) { 913 SPDK_ERRLOG("listen() failed, errno = %d\n", errno); 914 close(fd); 915 fd = -1; 916 break; 917 } 918 enable_zcopy_impl_opts = impl_opts.enable_zerocopy_send_server; 919 } else if (type == SPDK_SOCK_CREATE_CONNECT) { 920 rc = connect(fd, res->ai_addr, res->ai_addrlen); 921 if (rc != 0) { 922 SPDK_ERRLOG("connect() failed, errno = %d\n", errno); 923 /* try next family */ 924 close(fd); 925 fd = -1; 926 continue; 927 } 928 enable_zcopy_impl_opts = impl_opts.enable_zerocopy_send_client; 929 if (enable_ssl) { 930 ctx = posix_sock_create_ssl_context(TLS_client_method(), opts, &impl_opts); 931 if (!ctx) { 932 SPDK_ERRLOG("posix_sock_create_ssl_context() failed, errno = %d\n", errno); 933 close(fd); 934 fd = -1; 935 break; 936 } 937 ssl = ssl_sock_connect_loop(ctx, fd, &impl_opts); 938 if (!ssl) { 939 SPDK_ERRLOG("ssl_sock_connect_loop() failed, errno = %d\n", errno); 940 close(fd); 941 fd = -1; 942 SSL_CTX_free(ctx); 943 break; 944 } 945 } 946 } 947 948 flag = fcntl(fd, F_GETFL); 949 if (fcntl(fd, F_SETFL, flag | O_NONBLOCK) < 0) { 950 SPDK_ERRLOG("fcntl can't set nonblocking mode for socket, fd: %d (%d)\n", fd, errno); 951 SSL_free(ssl); 952 SSL_CTX_free(ctx); 953 close(fd); 954 fd = -1; 955 break; 956 } 957 break; 958 } 959 freeaddrinfo(res0); 960 961 if (fd < 0) { 962 return NULL; 963 } 964 965 /* Only enable zero copy for non-loopback and non-ssl sockets. */ 966 enable_zcopy_user_opts = opts->zcopy && !sock_is_loopback(fd) && !enable_ssl; 967 968 sock = posix_sock_alloc(fd, &impl_opts, enable_zcopy_user_opts && enable_zcopy_impl_opts); 969 if (sock == NULL) { 970 SPDK_ERRLOG("sock allocation failed\n"); 971 SSL_free(ssl); 972 SSL_CTX_free(ctx); 973 close(fd); 974 return NULL; 975 } 976 977 if (ctx) { 978 sock->ctx = ctx; 979 } 980 981 if (ssl) { 982 sock->ssl = ssl; 983 } 984 985 return &sock->base; 986 } 987 988 static struct spdk_sock * 989 posix_sock_listen(const char *ip, int port, struct spdk_sock_opts *opts) 990 { 991 return posix_sock_create(ip, port, SPDK_SOCK_CREATE_LISTEN, opts, false); 992 } 993 994 static struct spdk_sock * 995 posix_sock_connect(const char *ip, int port, struct spdk_sock_opts *opts) 996 { 997 return posix_sock_create(ip, port, SPDK_SOCK_CREATE_CONNECT, opts, false); 998 } 999 1000 static struct spdk_sock * 1001 _posix_sock_accept(struct spdk_sock *_sock, bool enable_ssl) 1002 { 1003 struct spdk_posix_sock *sock = __posix_sock(_sock); 1004 struct sockaddr_storage sa; 1005 socklen_t salen; 1006 int rc, fd; 1007 struct spdk_posix_sock *new_sock; 1008 int flag; 1009 SSL_CTX *ctx = 0; 1010 SSL *ssl = 0; 1011 1012 memset(&sa, 0, sizeof(sa)); 1013 salen = sizeof(sa); 1014 1015 assert(sock != NULL); 1016 1017 rc = accept(sock->fd, (struct sockaddr *)&sa, &salen); 1018 1019 if (rc == -1) { 1020 return NULL; 1021 } 1022 1023 fd = rc; 1024 1025 flag = fcntl(fd, F_GETFL); 1026 if ((!(flag & O_NONBLOCK)) && (fcntl(fd, F_SETFL, flag | O_NONBLOCK) < 0)) { 1027 SPDK_ERRLOG("fcntl can't set nonblocking mode for socket, fd: %d (%d)\n", fd, errno); 1028 close(fd); 1029 return NULL; 1030 } 1031 1032 #if defined(SO_PRIORITY) 1033 /* The priority is not inherited, so call this function again */ 1034 if (sock->base.opts.priority) { 1035 rc = setsockopt(fd, SOL_SOCKET, SO_PRIORITY, &sock->base.opts.priority, sizeof(int)); 1036 if (rc != 0) { 1037 close(fd); 1038 return NULL; 1039 } 1040 } 1041 #endif 1042 1043 /* Establish SSL connection */ 1044 if (enable_ssl) { 1045 ctx = posix_sock_create_ssl_context(TLS_server_method(), &sock->base.opts, &sock->base.impl_opts); 1046 if (!ctx) { 1047 SPDK_ERRLOG("posix_sock_create_ssl_context() failed, errno = %d\n", errno); 1048 close(fd); 1049 return NULL; 1050 } 1051 ssl = ssl_sock_accept_loop(ctx, fd, &sock->base.impl_opts); 1052 if (!ssl) { 1053 SPDK_ERRLOG("ssl_sock_accept_loop() failed, errno = %d\n", errno); 1054 close(fd); 1055 SSL_CTX_free(ctx); 1056 return NULL; 1057 } 1058 } 1059 1060 /* Inherit the zero copy feature from the listen socket */ 1061 new_sock = posix_sock_alloc(fd, &sock->base.impl_opts, sock->zcopy); 1062 if (new_sock == NULL) { 1063 close(fd); 1064 SSL_free(ssl); 1065 SSL_CTX_free(ctx); 1066 return NULL; 1067 } 1068 1069 if (ctx) { 1070 new_sock->ctx = ctx; 1071 } 1072 1073 if (ssl) { 1074 new_sock->ssl = ssl; 1075 } 1076 1077 return &new_sock->base; 1078 } 1079 1080 static struct spdk_sock * 1081 posix_sock_accept(struct spdk_sock *_sock) 1082 { 1083 return _posix_sock_accept(_sock, false); 1084 } 1085 1086 static int 1087 posix_sock_close(struct spdk_sock *_sock) 1088 { 1089 struct spdk_posix_sock *sock = __posix_sock(_sock); 1090 1091 assert(TAILQ_EMPTY(&_sock->pending_reqs)); 1092 1093 /* If the socket fails to close, the best choice is to 1094 * leak the fd but continue to free the rest of the sock 1095 * memory. */ 1096 close(sock->fd); 1097 1098 SSL_free(sock->ssl); 1099 SSL_CTX_free(sock->ctx); 1100 1101 spdk_pipe_destroy(sock->recv_pipe); 1102 free(sock->recv_buf); 1103 free(sock); 1104 1105 return 0; 1106 } 1107 1108 #ifdef SPDK_ZEROCOPY 1109 static int 1110 _sock_check_zcopy(struct spdk_sock *sock) 1111 { 1112 struct spdk_posix_sock *psock = __posix_sock(sock); 1113 struct msghdr msgh = {}; 1114 uint8_t buf[sizeof(struct cmsghdr) + sizeof(struct sock_extended_err)]; 1115 ssize_t rc; 1116 struct sock_extended_err *serr; 1117 struct cmsghdr *cm; 1118 uint32_t idx; 1119 struct spdk_sock_request *req, *treq; 1120 bool found; 1121 1122 msgh.msg_control = buf; 1123 msgh.msg_controllen = sizeof(buf); 1124 1125 while (true) { 1126 rc = recvmsg(psock->fd, &msgh, MSG_ERRQUEUE); 1127 1128 if (rc < 0) { 1129 if (errno == EWOULDBLOCK || errno == EAGAIN) { 1130 return 0; 1131 } 1132 1133 if (!TAILQ_EMPTY(&sock->pending_reqs)) { 1134 SPDK_ERRLOG("Attempting to receive from ERRQUEUE yielded error, but pending list still has orphaned entries\n"); 1135 } else { 1136 SPDK_WARNLOG("Recvmsg yielded an error!\n"); 1137 } 1138 return 0; 1139 } 1140 1141 cm = CMSG_FIRSTHDR(&msgh); 1142 if (!(cm && 1143 ((cm->cmsg_level == SOL_IP && cm->cmsg_type == IP_RECVERR) || 1144 (cm->cmsg_level == SOL_IPV6 && cm->cmsg_type == IPV6_RECVERR)))) { 1145 SPDK_WARNLOG("Unexpected cmsg level or type!\n"); 1146 return 0; 1147 } 1148 1149 serr = (struct sock_extended_err *)CMSG_DATA(cm); 1150 if (serr->ee_errno != 0 || serr->ee_origin != SO_EE_ORIGIN_ZEROCOPY) { 1151 SPDK_WARNLOG("Unexpected extended error origin\n"); 1152 return 0; 1153 } 1154 1155 /* Most of the time, the pending_reqs array is in the exact 1156 * order we need such that all of the requests to complete are 1157 * in order, in the front. It is guaranteed that all requests 1158 * belonging to the same sendmsg call are sequential, so once 1159 * we encounter one match we can stop looping as soon as a 1160 * non-match is found. 1161 */ 1162 for (idx = serr->ee_info; idx <= serr->ee_data; idx++) { 1163 found = false; 1164 TAILQ_FOREACH_SAFE(req, &sock->pending_reqs, internal.link, treq) { 1165 if (!req->internal.is_zcopy) { 1166 /* This wasn't a zcopy request. It was just waiting in line to complete */ 1167 rc = spdk_sock_request_put(sock, req, 0); 1168 if (rc < 0) { 1169 return rc; 1170 } 1171 } else if (req->internal.offset == idx) { 1172 found = true; 1173 rc = spdk_sock_request_put(sock, req, 0); 1174 if (rc < 0) { 1175 return rc; 1176 } 1177 } else if (found) { 1178 break; 1179 } 1180 } 1181 } 1182 } 1183 1184 return 0; 1185 } 1186 #endif 1187 1188 static int 1189 _sock_flush(struct spdk_sock *sock) 1190 { 1191 struct spdk_posix_sock *psock = __posix_sock(sock); 1192 struct msghdr msg = {}; 1193 int flags; 1194 struct iovec iovs[IOV_BATCH_SIZE]; 1195 int iovcnt; 1196 int retval; 1197 struct spdk_sock_request *req; 1198 int i; 1199 ssize_t rc; 1200 unsigned int offset; 1201 size_t len; 1202 bool is_zcopy = false; 1203 1204 /* Can't flush from within a callback or we end up with recursive calls */ 1205 if (sock->cb_cnt > 0) { 1206 return 0; 1207 } 1208 1209 #ifdef SPDK_ZEROCOPY 1210 if (psock->zcopy) { 1211 flags = MSG_ZEROCOPY | MSG_NOSIGNAL; 1212 } else 1213 #endif 1214 { 1215 flags = MSG_NOSIGNAL; 1216 } 1217 1218 iovcnt = spdk_sock_prep_reqs(sock, iovs, 0, NULL, &flags); 1219 if (iovcnt == 0) { 1220 return 0; 1221 } 1222 1223 #ifdef SPDK_ZEROCOPY 1224 is_zcopy = flags & MSG_ZEROCOPY; 1225 #endif 1226 1227 /* Perform the vectored write */ 1228 msg.msg_iov = iovs; 1229 msg.msg_iovlen = iovcnt; 1230 1231 if (psock->ssl) { 1232 rc = SSL_writev(psock->ssl, iovs, iovcnt); 1233 } else { 1234 rc = sendmsg(psock->fd, &msg, flags); 1235 } 1236 if (rc <= 0) { 1237 if (errno == EAGAIN || errno == EWOULDBLOCK || (errno == ENOBUFS && psock->zcopy)) { 1238 return 0; 1239 } 1240 return rc; 1241 } 1242 1243 if (is_zcopy) { 1244 /* Handling overflow case, because we use psock->sendmsg_idx - 1 for the 1245 * req->internal.offset, so sendmsg_idx should not be zero */ 1246 if (spdk_unlikely(psock->sendmsg_idx == UINT32_MAX)) { 1247 psock->sendmsg_idx = 1; 1248 } else { 1249 psock->sendmsg_idx++; 1250 } 1251 } 1252 1253 /* Consume the requests that were actually written */ 1254 req = TAILQ_FIRST(&sock->queued_reqs); 1255 while (req) { 1256 offset = req->internal.offset; 1257 1258 /* req->internal.is_zcopy is true when the whole req or part of it is sent with zerocopy */ 1259 req->internal.is_zcopy = is_zcopy; 1260 1261 for (i = 0; i < req->iovcnt; i++) { 1262 /* Advance by the offset first */ 1263 if (offset >= SPDK_SOCK_REQUEST_IOV(req, i)->iov_len) { 1264 offset -= SPDK_SOCK_REQUEST_IOV(req, i)->iov_len; 1265 continue; 1266 } 1267 1268 /* Calculate the remaining length of this element */ 1269 len = SPDK_SOCK_REQUEST_IOV(req, i)->iov_len - offset; 1270 1271 if (len > (size_t)rc) { 1272 /* This element was partially sent. */ 1273 req->internal.offset += rc; 1274 return 0; 1275 } 1276 1277 offset = 0; 1278 req->internal.offset += len; 1279 rc -= len; 1280 } 1281 1282 /* Handled a full request. */ 1283 spdk_sock_request_pend(sock, req); 1284 1285 if (!req->internal.is_zcopy && req == TAILQ_FIRST(&sock->pending_reqs)) { 1286 /* The sendmsg syscall above isn't currently asynchronous, 1287 * so it's already done. */ 1288 retval = spdk_sock_request_put(sock, req, 0); 1289 if (retval) { 1290 break; 1291 } 1292 } else { 1293 /* Re-use the offset field to hold the sendmsg call index. The 1294 * index is 0 based, so subtract one here because we've already 1295 * incremented above. */ 1296 req->internal.offset = psock->sendmsg_idx - 1; 1297 } 1298 1299 if (rc == 0) { 1300 break; 1301 } 1302 1303 req = TAILQ_FIRST(&sock->queued_reqs); 1304 } 1305 1306 return 0; 1307 } 1308 1309 static int 1310 posix_sock_flush(struct spdk_sock *sock) 1311 { 1312 #ifdef SPDK_ZEROCOPY 1313 struct spdk_posix_sock *psock = __posix_sock(sock); 1314 1315 if (psock->zcopy && !TAILQ_EMPTY(&sock->pending_reqs)) { 1316 _sock_check_zcopy(sock); 1317 } 1318 #endif 1319 1320 return _sock_flush(sock); 1321 } 1322 1323 static ssize_t 1324 posix_sock_recv_from_pipe(struct spdk_posix_sock *sock, struct iovec *diov, int diovcnt) 1325 { 1326 struct iovec siov[2]; 1327 int sbytes; 1328 ssize_t bytes; 1329 struct spdk_posix_sock_group_impl *group; 1330 1331 sbytes = spdk_pipe_reader_get_buffer(sock->recv_pipe, sock->recv_buf_sz, siov); 1332 if (sbytes < 0) { 1333 errno = EINVAL; 1334 return -1; 1335 } else if (sbytes == 0) { 1336 errno = EAGAIN; 1337 return -1; 1338 } 1339 1340 bytes = spdk_iovcpy(siov, 2, diov, diovcnt); 1341 1342 if (bytes == 0) { 1343 /* The only way this happens is if diov is 0 length */ 1344 errno = EINVAL; 1345 return -1; 1346 } 1347 1348 spdk_pipe_reader_advance(sock->recv_pipe, bytes); 1349 1350 /* If we drained the pipe, mark it appropriately */ 1351 if (spdk_pipe_reader_bytes_available(sock->recv_pipe) == 0) { 1352 assert(sock->pipe_has_data == true); 1353 1354 group = __posix_group_impl(sock->base.group_impl); 1355 if (group && !sock->socket_has_data) { 1356 TAILQ_REMOVE(&group->socks_with_data, sock, link); 1357 } 1358 1359 sock->pipe_has_data = false; 1360 } 1361 1362 return bytes; 1363 } 1364 1365 static inline ssize_t 1366 posix_sock_read(struct spdk_posix_sock *sock) 1367 { 1368 struct iovec iov[2]; 1369 int bytes_avail, bytes_recvd; 1370 struct spdk_posix_sock_group_impl *group; 1371 1372 bytes_avail = spdk_pipe_writer_get_buffer(sock->recv_pipe, sock->recv_buf_sz, iov); 1373 1374 if (bytes_avail <= 0) { 1375 return bytes_avail; 1376 } 1377 1378 if (sock->ssl) { 1379 bytes_recvd = SSL_readv(sock->ssl, iov, 2); 1380 } else { 1381 bytes_recvd = readv(sock->fd, iov, 2); 1382 } 1383 1384 assert(sock->pipe_has_data == false); 1385 1386 if (bytes_recvd <= 0) { 1387 /* Errors count as draining the socket data */ 1388 if (sock->base.group_impl && sock->socket_has_data) { 1389 group = __posix_group_impl(sock->base.group_impl); 1390 TAILQ_REMOVE(&group->socks_with_data, sock, link); 1391 } 1392 1393 sock->socket_has_data = false; 1394 1395 return bytes_recvd; 1396 } 1397 1398 spdk_pipe_writer_advance(sock->recv_pipe, bytes_recvd); 1399 1400 #if DEBUG 1401 if (sock->base.group_impl) { 1402 assert(sock->socket_has_data == true); 1403 } 1404 #endif 1405 1406 sock->pipe_has_data = true; 1407 if (bytes_recvd < bytes_avail) { 1408 /* We drained the kernel socket entirely. */ 1409 sock->socket_has_data = false; 1410 } 1411 1412 return bytes_recvd; 1413 } 1414 1415 static ssize_t 1416 posix_sock_readv(struct spdk_sock *_sock, struct iovec *iov, int iovcnt) 1417 { 1418 struct spdk_posix_sock *sock = __posix_sock(_sock); 1419 struct spdk_posix_sock_group_impl *group = __posix_group_impl(sock->base.group_impl); 1420 int rc, i; 1421 size_t len; 1422 1423 if (sock->recv_pipe == NULL) { 1424 assert(sock->pipe_has_data == false); 1425 if (group && sock->socket_has_data) { 1426 sock->socket_has_data = false; 1427 TAILQ_REMOVE(&group->socks_with_data, sock, link); 1428 } 1429 if (sock->ssl) { 1430 return SSL_readv(sock->ssl, iov, iovcnt); 1431 } else { 1432 return readv(sock->fd, iov, iovcnt); 1433 } 1434 } 1435 1436 /* If the socket is not in a group, we must assume it always has 1437 * data waiting for us because it is not epolled */ 1438 if (!sock->pipe_has_data && (group == NULL || sock->socket_has_data)) { 1439 /* If the user is receiving a sufficiently large amount of data, 1440 * receive directly to their buffers. */ 1441 len = 0; 1442 for (i = 0; i < iovcnt; i++) { 1443 len += iov[i].iov_len; 1444 } 1445 1446 if (len >= MIN_SOCK_PIPE_SIZE) { 1447 /* TODO: Should this detect if kernel socket is drained? */ 1448 if (sock->ssl) { 1449 return SSL_readv(sock->ssl, iov, iovcnt); 1450 } else { 1451 return readv(sock->fd, iov, iovcnt); 1452 } 1453 } 1454 1455 /* Otherwise, do a big read into our pipe */ 1456 rc = posix_sock_read(sock); 1457 if (rc <= 0) { 1458 return rc; 1459 } 1460 } 1461 1462 return posix_sock_recv_from_pipe(sock, iov, iovcnt); 1463 } 1464 1465 static ssize_t 1466 posix_sock_recv(struct spdk_sock *sock, void *buf, size_t len) 1467 { 1468 struct iovec iov[1]; 1469 1470 iov[0].iov_base = buf; 1471 iov[0].iov_len = len; 1472 1473 return posix_sock_readv(sock, iov, 1); 1474 } 1475 1476 static void 1477 posix_sock_readv_async(struct spdk_sock *sock, struct spdk_sock_request *req) 1478 { 1479 req->cb_fn(req->cb_arg, -ENOTSUP); 1480 } 1481 1482 static ssize_t 1483 posix_sock_writev(struct spdk_sock *_sock, struct iovec *iov, int iovcnt) 1484 { 1485 struct spdk_posix_sock *sock = __posix_sock(_sock); 1486 int rc; 1487 1488 /* In order to process a writev, we need to flush any asynchronous writes 1489 * first. */ 1490 rc = _sock_flush(_sock); 1491 if (rc < 0) { 1492 return rc; 1493 } 1494 1495 if (!TAILQ_EMPTY(&_sock->queued_reqs)) { 1496 /* We weren't able to flush all requests */ 1497 errno = EAGAIN; 1498 return -1; 1499 } 1500 1501 if (sock->ssl) { 1502 return SSL_writev(sock->ssl, iov, iovcnt); 1503 } else { 1504 return writev(sock->fd, iov, iovcnt); 1505 } 1506 } 1507 1508 static void 1509 posix_sock_writev_async(struct spdk_sock *sock, struct spdk_sock_request *req) 1510 { 1511 int rc; 1512 1513 spdk_sock_request_queue(sock, req); 1514 1515 /* If there are a sufficient number queued, just flush them out immediately. */ 1516 if (sock->queued_iovcnt >= IOV_BATCH_SIZE) { 1517 rc = _sock_flush(sock); 1518 if (rc) { 1519 spdk_sock_abort_requests(sock); 1520 } 1521 } 1522 } 1523 1524 static int 1525 posix_sock_set_recvlowat(struct spdk_sock *_sock, int nbytes) 1526 { 1527 struct spdk_posix_sock *sock = __posix_sock(_sock); 1528 int val; 1529 int rc; 1530 1531 assert(sock != NULL); 1532 1533 val = nbytes; 1534 rc = setsockopt(sock->fd, SOL_SOCKET, SO_RCVLOWAT, &val, sizeof val); 1535 if (rc != 0) { 1536 return -1; 1537 } 1538 return 0; 1539 } 1540 1541 static bool 1542 posix_sock_is_ipv6(struct spdk_sock *_sock) 1543 { 1544 struct spdk_posix_sock *sock = __posix_sock(_sock); 1545 struct sockaddr_storage sa; 1546 socklen_t salen; 1547 int rc; 1548 1549 assert(sock != NULL); 1550 1551 memset(&sa, 0, sizeof sa); 1552 salen = sizeof sa; 1553 rc = getsockname(sock->fd, (struct sockaddr *) &sa, &salen); 1554 if (rc != 0) { 1555 SPDK_ERRLOG("getsockname() failed (errno=%d)\n", errno); 1556 return false; 1557 } 1558 1559 return (sa.ss_family == AF_INET6); 1560 } 1561 1562 static bool 1563 posix_sock_is_ipv4(struct spdk_sock *_sock) 1564 { 1565 struct spdk_posix_sock *sock = __posix_sock(_sock); 1566 struct sockaddr_storage sa; 1567 socklen_t salen; 1568 int rc; 1569 1570 assert(sock != NULL); 1571 1572 memset(&sa, 0, sizeof sa); 1573 salen = sizeof sa; 1574 rc = getsockname(sock->fd, (struct sockaddr *) &sa, &salen); 1575 if (rc != 0) { 1576 SPDK_ERRLOG("getsockname() failed (errno=%d)\n", errno); 1577 return false; 1578 } 1579 1580 return (sa.ss_family == AF_INET); 1581 } 1582 1583 static bool 1584 posix_sock_is_connected(struct spdk_sock *_sock) 1585 { 1586 struct spdk_posix_sock *sock = __posix_sock(_sock); 1587 uint8_t byte; 1588 int rc; 1589 1590 rc = recv(sock->fd, &byte, 1, MSG_PEEK); 1591 if (rc == 0) { 1592 return false; 1593 } 1594 1595 if (rc < 0) { 1596 if (errno == EAGAIN || errno == EWOULDBLOCK) { 1597 return true; 1598 } 1599 1600 return false; 1601 } 1602 1603 return true; 1604 } 1605 1606 static struct spdk_sock_group_impl * 1607 posix_sock_group_impl_get_optimal(struct spdk_sock *_sock, struct spdk_sock_group_impl *hint) 1608 { 1609 struct spdk_posix_sock *sock = __posix_sock(_sock); 1610 struct spdk_sock_group_impl *group_impl; 1611 1612 if (sock->placement_id != -1) { 1613 spdk_sock_map_lookup(&g_map, sock->placement_id, &group_impl, hint); 1614 return group_impl; 1615 } 1616 1617 return NULL; 1618 } 1619 1620 static struct spdk_sock_group_impl * 1621 posix_sock_group_impl_create(void) 1622 { 1623 struct spdk_posix_sock_group_impl *group_impl; 1624 int fd; 1625 1626 #if defined(SPDK_EPOLL) 1627 fd = epoll_create1(0); 1628 #elif defined(SPDK_KEVENT) 1629 fd = kqueue(); 1630 #endif 1631 if (fd == -1) { 1632 return NULL; 1633 } 1634 1635 group_impl = calloc(1, sizeof(*group_impl)); 1636 if (group_impl == NULL) { 1637 SPDK_ERRLOG("group_impl allocation failed\n"); 1638 close(fd); 1639 return NULL; 1640 } 1641 1642 group_impl->fd = fd; 1643 TAILQ_INIT(&group_impl->socks_with_data); 1644 group_impl->placement_id = -1; 1645 1646 if (g_spdk_posix_sock_impl_opts.enable_placement_id == PLACEMENT_CPU) { 1647 spdk_sock_map_insert(&g_map, spdk_env_get_current_core(), &group_impl->base); 1648 group_impl->placement_id = spdk_env_get_current_core(); 1649 } 1650 1651 return &group_impl->base; 1652 } 1653 1654 static void 1655 posix_sock_mark(struct spdk_posix_sock_group_impl *group, struct spdk_posix_sock *sock, 1656 int placement_id) 1657 { 1658 #if defined(SO_MARK) 1659 int rc; 1660 1661 rc = setsockopt(sock->fd, SOL_SOCKET, SO_MARK, 1662 &placement_id, sizeof(placement_id)); 1663 if (rc != 0) { 1664 /* Not fatal */ 1665 SPDK_ERRLOG("Error setting SO_MARK\n"); 1666 return; 1667 } 1668 1669 rc = spdk_sock_map_insert(&g_map, placement_id, &group->base); 1670 if (rc != 0) { 1671 /* Not fatal */ 1672 SPDK_ERRLOG("Failed to insert sock group into map: %d\n", rc); 1673 return; 1674 } 1675 1676 sock->placement_id = placement_id; 1677 #endif 1678 } 1679 1680 static void 1681 posix_sock_update_mark(struct spdk_sock_group_impl *_group, struct spdk_sock *_sock) 1682 { 1683 struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group); 1684 1685 if (group->placement_id == -1) { 1686 group->placement_id = spdk_sock_map_find_free(&g_map); 1687 1688 /* If a free placement id is found, update existing sockets in this group */ 1689 if (group->placement_id != -1) { 1690 struct spdk_sock *sock, *tmp; 1691 1692 TAILQ_FOREACH_SAFE(sock, &_group->socks, link, tmp) { 1693 posix_sock_mark(group, __posix_sock(sock), group->placement_id); 1694 } 1695 } 1696 } 1697 1698 if (group->placement_id != -1) { 1699 /* 1700 * group placement id is already determined for this poll group. 1701 * Mark socket with group's placement id. 1702 */ 1703 posix_sock_mark(group, __posix_sock(_sock), group->placement_id); 1704 } 1705 } 1706 1707 static int 1708 posix_sock_group_impl_add_sock(struct spdk_sock_group_impl *_group, struct spdk_sock *_sock) 1709 { 1710 struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group); 1711 struct spdk_posix_sock *sock = __posix_sock(_sock); 1712 int rc; 1713 1714 #if defined(SPDK_EPOLL) 1715 struct epoll_event event; 1716 1717 memset(&event, 0, sizeof(event)); 1718 /* EPOLLERR is always on even if we don't set it, but be explicit for clarity */ 1719 event.events = EPOLLIN | EPOLLERR; 1720 event.data.ptr = sock; 1721 1722 rc = epoll_ctl(group->fd, EPOLL_CTL_ADD, sock->fd, &event); 1723 #elif defined(SPDK_KEVENT) 1724 struct kevent event; 1725 struct timespec ts = {0}; 1726 1727 EV_SET(&event, sock->fd, EVFILT_READ, EV_ADD, 0, 0, sock); 1728 1729 rc = kevent(group->fd, &event, 1, NULL, 0, &ts); 1730 #endif 1731 1732 if (rc != 0) { 1733 return rc; 1734 } 1735 1736 /* switched from another polling group due to scheduling */ 1737 if (spdk_unlikely(sock->recv_pipe != NULL && 1738 (spdk_pipe_reader_bytes_available(sock->recv_pipe) > 0))) { 1739 sock->pipe_has_data = true; 1740 sock->socket_has_data = false; 1741 TAILQ_INSERT_TAIL(&group->socks_with_data, sock, link); 1742 } 1743 1744 if (g_spdk_posix_sock_impl_opts.enable_placement_id == PLACEMENT_MARK) { 1745 posix_sock_update_mark(_group, _sock); 1746 } else if (sock->placement_id != -1) { 1747 rc = spdk_sock_map_insert(&g_map, sock->placement_id, &group->base); 1748 if (rc != 0) { 1749 SPDK_ERRLOG("Failed to insert sock group into map: %d\n", rc); 1750 /* Do not treat this as an error. The system will continue running. */ 1751 } 1752 } 1753 1754 return rc; 1755 } 1756 1757 static int 1758 posix_sock_group_impl_remove_sock(struct spdk_sock_group_impl *_group, struct spdk_sock *_sock) 1759 { 1760 struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group); 1761 struct spdk_posix_sock *sock = __posix_sock(_sock); 1762 int rc; 1763 1764 if (sock->pipe_has_data || sock->socket_has_data) { 1765 TAILQ_REMOVE(&group->socks_with_data, sock, link); 1766 sock->pipe_has_data = false; 1767 sock->socket_has_data = false; 1768 } 1769 1770 if (sock->placement_id != -1) { 1771 spdk_sock_map_release(&g_map, sock->placement_id); 1772 } 1773 1774 #if defined(SPDK_EPOLL) 1775 struct epoll_event event; 1776 1777 /* Event parameter is ignored but some old kernel version still require it. */ 1778 rc = epoll_ctl(group->fd, EPOLL_CTL_DEL, sock->fd, &event); 1779 #elif defined(SPDK_KEVENT) 1780 struct kevent event; 1781 struct timespec ts = {0}; 1782 1783 EV_SET(&event, sock->fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); 1784 1785 rc = kevent(group->fd, &event, 1, NULL, 0, &ts); 1786 if (rc == 0 && event.flags & EV_ERROR) { 1787 rc = -1; 1788 errno = event.data; 1789 } 1790 #endif 1791 1792 spdk_sock_abort_requests(_sock); 1793 1794 return rc; 1795 } 1796 1797 static int 1798 posix_sock_group_impl_poll(struct spdk_sock_group_impl *_group, int max_events, 1799 struct spdk_sock **socks) 1800 { 1801 struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group); 1802 struct spdk_sock *sock, *tmp; 1803 int num_events, i, rc; 1804 struct spdk_posix_sock *psock, *ptmp; 1805 #if defined(SPDK_EPOLL) 1806 struct epoll_event events[MAX_EVENTS_PER_POLL]; 1807 #elif defined(SPDK_KEVENT) 1808 struct kevent events[MAX_EVENTS_PER_POLL]; 1809 struct timespec ts = {0}; 1810 #endif 1811 1812 #ifdef SPDK_ZEROCOPY 1813 /* When all of the following conditions are met 1814 * - non-blocking socket 1815 * - zero copy is enabled 1816 * - interrupts suppressed (i.e. busy polling) 1817 * - the NIC tx queue is full at the time sendmsg() is called 1818 * - epoll_wait determines there is an EPOLLIN event for the socket 1819 * then we can get into a situation where data we've sent is queued 1820 * up in the kernel network stack, but interrupts have been suppressed 1821 * because other traffic is flowing so the kernel misses the signal 1822 * to flush the software tx queue. If there wasn't incoming data 1823 * pending on the socket, then epoll_wait would have been sufficient 1824 * to kick off the send operation, but since there is a pending event 1825 * epoll_wait does not trigger the necessary operation. 1826 * 1827 * We deal with this by checking for all of the above conditions and 1828 * additionally looking for EPOLLIN events that were not consumed from 1829 * the last poll loop. We take this to mean that the upper layer is 1830 * unable to consume them because it is blocked waiting for resources 1831 * to free up, and those resources are most likely freed in response 1832 * to a pending asynchronous write completing. 1833 * 1834 * Additionally, sockets that have the same placement_id actually share 1835 * an underlying hardware queue. That means polling one of them is 1836 * equivalent to polling all of them. As a quick mechanism to avoid 1837 * making extra poll() calls, stash the last placement_id during the loop 1838 * and only poll if it's not the same. The overwhelmingly common case 1839 * is that all sockets in this list have the same placement_id because 1840 * SPDK is intentionally grouping sockets by that value, so even 1841 * though this won't stop all extra calls to poll(), it's very fast 1842 * and will catch all of them in practice. 1843 */ 1844 int last_placement_id = -1; 1845 1846 TAILQ_FOREACH(psock, &group->socks_with_data, link) { 1847 if (psock->zcopy && psock->placement_id >= 0 && 1848 psock->placement_id != last_placement_id) { 1849 struct pollfd pfd = {psock->fd, POLLIN | POLLERR, 0}; 1850 1851 poll(&pfd, 1, 0); 1852 last_placement_id = psock->placement_id; 1853 } 1854 } 1855 #endif 1856 1857 /* This must be a TAILQ_FOREACH_SAFE because while flushing, 1858 * a completion callback could remove the sock from the 1859 * group. */ 1860 TAILQ_FOREACH_SAFE(sock, &_group->socks, link, tmp) { 1861 rc = _sock_flush(sock); 1862 if (rc) { 1863 spdk_sock_abort_requests(sock); 1864 } 1865 } 1866 1867 assert(max_events > 0); 1868 1869 #if defined(SPDK_EPOLL) 1870 num_events = epoll_wait(group->fd, events, max_events, 0); 1871 #elif defined(SPDK_KEVENT) 1872 num_events = kevent(group->fd, NULL, 0, events, max_events, &ts); 1873 #endif 1874 1875 if (num_events == -1) { 1876 return -1; 1877 } else if (num_events == 0 && !TAILQ_EMPTY(&_group->socks)) { 1878 sock = TAILQ_FIRST(&_group->socks); 1879 psock = __posix_sock(sock); 1880 /* poll() is called here to busy poll the queue associated with 1881 * first socket in list and potentially reap incoming data. 1882 */ 1883 if (sock->opts.priority) { 1884 struct pollfd pfd = {0, 0, 0}; 1885 1886 pfd.fd = psock->fd; 1887 pfd.events = POLLIN | POLLERR; 1888 poll(&pfd, 1, 0); 1889 } 1890 } 1891 1892 for (i = 0; i < num_events; i++) { 1893 #if defined(SPDK_EPOLL) 1894 sock = events[i].data.ptr; 1895 psock = __posix_sock(sock); 1896 1897 #ifdef SPDK_ZEROCOPY 1898 if (events[i].events & EPOLLERR) { 1899 rc = _sock_check_zcopy(sock); 1900 /* If the socket was closed or removed from 1901 * the group in response to a send ack, don't 1902 * add it to the array here. */ 1903 if (rc || sock->cb_fn == NULL) { 1904 continue; 1905 } 1906 } 1907 #endif 1908 if ((events[i].events & EPOLLIN) == 0) { 1909 continue; 1910 } 1911 1912 #elif defined(SPDK_KEVENT) 1913 sock = events[i].udata; 1914 psock = __posix_sock(sock); 1915 #endif 1916 1917 /* If the socket is not already in the list, add it now */ 1918 if (!psock->socket_has_data && !psock->pipe_has_data) { 1919 TAILQ_INSERT_TAIL(&group->socks_with_data, psock, link); 1920 } 1921 psock->socket_has_data = true; 1922 } 1923 1924 num_events = 0; 1925 1926 TAILQ_FOREACH_SAFE(psock, &group->socks_with_data, link, ptmp) { 1927 if (num_events == max_events) { 1928 break; 1929 } 1930 1931 /* If the socket's cb_fn is NULL, just remove it from the 1932 * list and do not add it to socks array */ 1933 if (spdk_unlikely(psock->base.cb_fn == NULL)) { 1934 psock->socket_has_data = false; 1935 psock->pipe_has_data = false; 1936 TAILQ_REMOVE(&group->socks_with_data, psock, link); 1937 continue; 1938 } 1939 1940 socks[num_events++] = &psock->base; 1941 } 1942 1943 /* Cycle the has_data list so that each time we poll things aren't 1944 * in the same order. Say we have 6 sockets in the list, named as follows: 1945 * A B C D E F 1946 * And all 6 sockets had epoll events, but max_events is only 3. That means 1947 * psock currently points at D. We want to rearrange the list to the following: 1948 * D E F A B C 1949 * 1950 * The variables below are named according to this example to make it easier to 1951 * follow the swaps. 1952 */ 1953 if (psock != NULL) { 1954 struct spdk_posix_sock *pa, *pc, *pd, *pf; 1955 1956 /* Capture pointers to the elements we need */ 1957 pd = psock; 1958 pc = TAILQ_PREV(pd, spdk_has_data_list, link); 1959 pa = TAILQ_FIRST(&group->socks_with_data); 1960 pf = TAILQ_LAST(&group->socks_with_data, spdk_has_data_list); 1961 1962 /* Break the link between C and D */ 1963 pc->link.tqe_next = NULL; 1964 1965 /* Connect F to A */ 1966 pf->link.tqe_next = pa; 1967 pa->link.tqe_prev = &pf->link.tqe_next; 1968 1969 /* Fix up the list first/last pointers */ 1970 group->socks_with_data.tqh_first = pd; 1971 group->socks_with_data.tqh_last = &pc->link.tqe_next; 1972 1973 /* D is in front of the list, make tqe prev pointer point to the head of list */ 1974 pd->link.tqe_prev = &group->socks_with_data.tqh_first; 1975 } 1976 1977 return num_events; 1978 } 1979 1980 static int 1981 posix_sock_group_impl_close(struct spdk_sock_group_impl *_group) 1982 { 1983 struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group); 1984 int rc; 1985 1986 if (g_spdk_posix_sock_impl_opts.enable_placement_id == PLACEMENT_CPU) { 1987 spdk_sock_map_release(&g_map, spdk_env_get_current_core()); 1988 } 1989 1990 rc = close(group->fd); 1991 free(group); 1992 return rc; 1993 } 1994 1995 static struct spdk_net_impl g_posix_net_impl = { 1996 .name = "posix", 1997 .getaddr = posix_sock_getaddr, 1998 .connect = posix_sock_connect, 1999 .listen = posix_sock_listen, 2000 .accept = posix_sock_accept, 2001 .close = posix_sock_close, 2002 .recv = posix_sock_recv, 2003 .readv = posix_sock_readv, 2004 .readv_async = posix_sock_readv_async, 2005 .writev = posix_sock_writev, 2006 .writev_async = posix_sock_writev_async, 2007 .flush = posix_sock_flush, 2008 .set_recvlowat = posix_sock_set_recvlowat, 2009 .set_recvbuf = posix_sock_set_recvbuf, 2010 .set_sendbuf = posix_sock_set_sendbuf, 2011 .is_ipv6 = posix_sock_is_ipv6, 2012 .is_ipv4 = posix_sock_is_ipv4, 2013 .is_connected = posix_sock_is_connected, 2014 .group_impl_get_optimal = posix_sock_group_impl_get_optimal, 2015 .group_impl_create = posix_sock_group_impl_create, 2016 .group_impl_add_sock = posix_sock_group_impl_add_sock, 2017 .group_impl_remove_sock = posix_sock_group_impl_remove_sock, 2018 .group_impl_poll = posix_sock_group_impl_poll, 2019 .group_impl_close = posix_sock_group_impl_close, 2020 .get_opts = posix_sock_impl_get_opts, 2021 .set_opts = posix_sock_impl_set_opts, 2022 }; 2023 2024 SPDK_NET_IMPL_REGISTER(posix, &g_posix_net_impl, DEFAULT_SOCK_PRIORITY + 1); 2025 2026 static struct spdk_sock * 2027 ssl_sock_listen(const char *ip, int port, struct spdk_sock_opts *opts) 2028 { 2029 return posix_sock_create(ip, port, SPDK_SOCK_CREATE_LISTEN, opts, true); 2030 } 2031 2032 static struct spdk_sock * 2033 ssl_sock_connect(const char *ip, int port, struct spdk_sock_opts *opts) 2034 { 2035 return posix_sock_create(ip, port, SPDK_SOCK_CREATE_CONNECT, opts, true); 2036 } 2037 2038 static struct spdk_sock * 2039 ssl_sock_accept(struct spdk_sock *_sock) 2040 { 2041 return _posix_sock_accept(_sock, true); 2042 } 2043 2044 static struct spdk_net_impl g_ssl_net_impl = { 2045 .name = "ssl", 2046 .getaddr = posix_sock_getaddr, 2047 .connect = ssl_sock_connect, 2048 .listen = ssl_sock_listen, 2049 .accept = ssl_sock_accept, 2050 .close = posix_sock_close, 2051 .recv = posix_sock_recv, 2052 .readv = posix_sock_readv, 2053 .writev = posix_sock_writev, 2054 .writev_async = posix_sock_writev_async, 2055 .flush = posix_sock_flush, 2056 .set_recvlowat = posix_sock_set_recvlowat, 2057 .set_recvbuf = posix_sock_set_recvbuf, 2058 .set_sendbuf = posix_sock_set_sendbuf, 2059 .is_ipv6 = posix_sock_is_ipv6, 2060 .is_ipv4 = posix_sock_is_ipv4, 2061 .is_connected = posix_sock_is_connected, 2062 .group_impl_get_optimal = posix_sock_group_impl_get_optimal, 2063 .group_impl_create = posix_sock_group_impl_create, 2064 .group_impl_add_sock = posix_sock_group_impl_add_sock, 2065 .group_impl_remove_sock = posix_sock_group_impl_remove_sock, 2066 .group_impl_poll = posix_sock_group_impl_poll, 2067 .group_impl_close = posix_sock_group_impl_close, 2068 .get_opts = posix_sock_impl_get_opts, 2069 .set_opts = posix_sock_impl_set_opts, 2070 }; 2071 2072 SPDK_NET_IMPL_REGISTER(ssl, &g_ssl_net_impl, DEFAULT_SOCK_PRIORITY); 2073 SPDK_LOG_REGISTER_COMPONENT(sock_posix) 2074