1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2018 Intel Corporation. All rights reserved. 3 * Copyright (c) 2020, 2021 Mellanox Technologies LTD. All rights reserved. 4 * Copyright (c) 2021 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #if defined(__FreeBSD__) 10 #include <sys/event.h> 11 #define SPDK_KEVENT 12 #else 13 #include <sys/epoll.h> 14 #define SPDK_EPOLL 15 #endif 16 17 #if defined(__linux__) 18 #include <linux/errqueue.h> 19 #endif 20 21 #include "spdk/env.h" 22 #include "spdk/log.h" 23 #include "spdk/pipe.h" 24 #include "spdk/sock.h" 25 #include "spdk/util.h" 26 #include "spdk/string.h" 27 #include "spdk_internal/sock.h" 28 #include "../sock_kernel.h" 29 30 #include "openssl/crypto.h" 31 #include "openssl/err.h" 32 #include "openssl/ssl.h" 33 34 #define MAX_TMPBUF 1024 35 #define PORTNUMLEN 32 36 37 #if defined(SO_ZEROCOPY) && defined(MSG_ZEROCOPY) 38 #define SPDK_ZEROCOPY 39 #endif 40 41 struct spdk_posix_sock { 42 struct spdk_sock base; 43 int fd; 44 45 uint32_t sendmsg_idx; 46 47 struct spdk_pipe *recv_pipe; 48 void *recv_buf; 49 int recv_buf_sz; 50 bool pipe_has_data; 51 bool socket_has_data; 52 bool zcopy; 53 54 int placement_id; 55 56 SSL_CTX *ctx; 57 SSL *ssl; 58 59 TAILQ_ENTRY(spdk_posix_sock) link; 60 }; 61 62 TAILQ_HEAD(spdk_has_data_list, spdk_posix_sock); 63 64 struct spdk_posix_sock_group_impl { 65 struct spdk_sock_group_impl base; 66 int fd; 67 struct spdk_has_data_list socks_with_data; 68 int placement_id; 69 }; 70 71 static struct spdk_sock_impl_opts g_spdk_posix_sock_impl_opts = { 72 .recv_buf_size = DEFAULT_SO_RCVBUF_SIZE, 73 .send_buf_size = DEFAULT_SO_SNDBUF_SIZE, 74 .enable_recv_pipe = true, 75 .enable_quickack = false, 76 .enable_placement_id = PLACEMENT_NONE, 77 .enable_zerocopy_send_server = true, 78 .enable_zerocopy_send_client = false, 79 .zerocopy_threshold = 0, 80 .tls_version = 0, 81 .enable_ktls = false, 82 .psk_key = NULL, 83 .psk_identity = NULL 84 }; 85 86 static struct spdk_sock_map g_map = { 87 .entries = STAILQ_HEAD_INITIALIZER(g_map.entries), 88 .mtx = PTHREAD_MUTEX_INITIALIZER 89 }; 90 91 __attribute((destructor)) static void 92 posix_sock_map_cleanup(void) 93 { 94 spdk_sock_map_cleanup(&g_map); 95 } 96 97 #define __posix_sock(sock) (struct spdk_posix_sock *)sock 98 #define __posix_group_impl(group) (struct spdk_posix_sock_group_impl *)group 99 100 static void 101 posix_sock_copy_impl_opts(struct spdk_sock_impl_opts *dest, const struct spdk_sock_impl_opts *src, 102 size_t len) 103 { 104 #define FIELD_OK(field) \ 105 offsetof(struct spdk_sock_impl_opts, field) + sizeof(src->field) <= len 106 107 #define SET_FIELD(field) \ 108 if (FIELD_OK(field)) { \ 109 dest->field = src->field; \ 110 } 111 112 SET_FIELD(recv_buf_size); 113 SET_FIELD(send_buf_size); 114 SET_FIELD(enable_recv_pipe); 115 SET_FIELD(enable_zerocopy_send); 116 SET_FIELD(enable_quickack); 117 SET_FIELD(enable_placement_id); 118 SET_FIELD(enable_zerocopy_send_server); 119 SET_FIELD(enable_zerocopy_send_client); 120 SET_FIELD(zerocopy_threshold); 121 SET_FIELD(tls_version); 122 SET_FIELD(enable_ktls); 123 SET_FIELD(psk_key); 124 SET_FIELD(psk_identity); 125 126 #undef SET_FIELD 127 #undef FIELD_OK 128 } 129 130 static int 131 posix_sock_impl_get_opts(struct spdk_sock_impl_opts *opts, size_t *len) 132 { 133 if (!opts || !len) { 134 errno = EINVAL; 135 return -1; 136 } 137 138 assert(sizeof(*opts) >= *len); 139 memset(opts, 0, *len); 140 141 posix_sock_copy_impl_opts(opts, &g_spdk_posix_sock_impl_opts, *len); 142 *len = spdk_min(*len, sizeof(g_spdk_posix_sock_impl_opts)); 143 144 return 0; 145 } 146 147 static int 148 posix_sock_impl_set_opts(const struct spdk_sock_impl_opts *opts, size_t len) 149 { 150 if (!opts) { 151 errno = EINVAL; 152 return -1; 153 } 154 155 assert(sizeof(*opts) >= len); 156 posix_sock_copy_impl_opts(&g_spdk_posix_sock_impl_opts, opts, len); 157 158 return 0; 159 } 160 161 static void 162 posix_opts_get_impl_opts(const struct spdk_sock_opts *opts, struct spdk_sock_impl_opts *dest) 163 { 164 /* Copy the default impl_opts first to cover cases when user's impl_opts is smaller */ 165 memcpy(dest, &g_spdk_posix_sock_impl_opts, sizeof(*dest)); 166 167 if (opts->impl_opts != NULL) { 168 assert(sizeof(*dest) >= opts->impl_opts_size); 169 posix_sock_copy_impl_opts(dest, opts->impl_opts, opts->impl_opts_size); 170 } 171 } 172 173 static int 174 posix_sock_getaddr(struct spdk_sock *_sock, char *saddr, int slen, uint16_t *sport, 175 char *caddr, int clen, uint16_t *cport) 176 { 177 struct spdk_posix_sock *sock = __posix_sock(_sock); 178 struct sockaddr_storage sa; 179 socklen_t salen; 180 int rc; 181 182 assert(sock != NULL); 183 184 memset(&sa, 0, sizeof sa); 185 salen = sizeof sa; 186 rc = getsockname(sock->fd, (struct sockaddr *) &sa, &salen); 187 if (rc != 0) { 188 SPDK_ERRLOG("getsockname() failed (errno=%d)\n", errno); 189 return -1; 190 } 191 192 switch (sa.ss_family) { 193 case AF_UNIX: 194 /* Acceptable connection types that don't have IPs */ 195 return 0; 196 case AF_INET: 197 case AF_INET6: 198 /* Code below will get IP addresses */ 199 break; 200 default: 201 /* Unsupported socket family */ 202 return -1; 203 } 204 205 rc = get_addr_str((struct sockaddr *)&sa, saddr, slen); 206 if (rc != 0) { 207 SPDK_ERRLOG("getnameinfo() failed (errno=%d)\n", errno); 208 return -1; 209 } 210 211 if (sport) { 212 if (sa.ss_family == AF_INET) { 213 *sport = ntohs(((struct sockaddr_in *) &sa)->sin_port); 214 } else if (sa.ss_family == AF_INET6) { 215 *sport = ntohs(((struct sockaddr_in6 *) &sa)->sin6_port); 216 } 217 } 218 219 memset(&sa, 0, sizeof sa); 220 salen = sizeof sa; 221 rc = getpeername(sock->fd, (struct sockaddr *) &sa, &salen); 222 if (rc != 0) { 223 SPDK_ERRLOG("getpeername() failed (errno=%d)\n", errno); 224 return -1; 225 } 226 227 rc = get_addr_str((struct sockaddr *)&sa, caddr, clen); 228 if (rc != 0) { 229 SPDK_ERRLOG("getnameinfo() failed (errno=%d)\n", errno); 230 return -1; 231 } 232 233 if (cport) { 234 if (sa.ss_family == AF_INET) { 235 *cport = ntohs(((struct sockaddr_in *) &sa)->sin_port); 236 } else if (sa.ss_family == AF_INET6) { 237 *cport = ntohs(((struct sockaddr_in6 *) &sa)->sin6_port); 238 } 239 } 240 241 return 0; 242 } 243 244 enum posix_sock_create_type { 245 SPDK_SOCK_CREATE_LISTEN, 246 SPDK_SOCK_CREATE_CONNECT, 247 }; 248 249 static int 250 posix_sock_alloc_pipe(struct spdk_posix_sock *sock, int sz) 251 { 252 uint8_t *new_buf; 253 struct spdk_pipe *new_pipe; 254 struct iovec siov[2]; 255 struct iovec diov[2]; 256 int sbytes; 257 ssize_t bytes; 258 int rc; 259 260 if (sock->recv_buf_sz == sz) { 261 return 0; 262 } 263 264 /* If the new size is 0, just free the pipe */ 265 if (sz == 0) { 266 spdk_pipe_destroy(sock->recv_pipe); 267 free(sock->recv_buf); 268 sock->recv_pipe = NULL; 269 sock->recv_buf = NULL; 270 return 0; 271 } else if (sz < MIN_SOCK_PIPE_SIZE) { 272 SPDK_ERRLOG("The size of the pipe must be larger than %d\n", MIN_SOCK_PIPE_SIZE); 273 return -1; 274 } 275 276 /* Round up to next 64 byte multiple */ 277 rc = posix_memalign((void **)&new_buf, 64, sz); 278 if (rc != 0) { 279 SPDK_ERRLOG("socket recv buf allocation failed\n"); 280 return -ENOMEM; 281 } 282 memset(new_buf, 0, sz); 283 284 new_pipe = spdk_pipe_create(new_buf, sz); 285 if (new_pipe == NULL) { 286 SPDK_ERRLOG("socket pipe allocation failed\n"); 287 free(new_buf); 288 return -ENOMEM; 289 } 290 291 if (sock->recv_pipe != NULL) { 292 /* Pull all of the data out of the old pipe */ 293 sbytes = spdk_pipe_reader_get_buffer(sock->recv_pipe, sock->recv_buf_sz, siov); 294 if (sbytes > sz) { 295 /* Too much data to fit into the new pipe size */ 296 spdk_pipe_destroy(new_pipe); 297 free(new_buf); 298 return -EINVAL; 299 } 300 301 sbytes = spdk_pipe_writer_get_buffer(new_pipe, sz, diov); 302 assert(sbytes == sz); 303 304 bytes = spdk_iovcpy(siov, 2, diov, 2); 305 spdk_pipe_writer_advance(new_pipe, bytes); 306 307 spdk_pipe_destroy(sock->recv_pipe); 308 free(sock->recv_buf); 309 } 310 311 sock->recv_buf_sz = sz; 312 sock->recv_buf = new_buf; 313 sock->recv_pipe = new_pipe; 314 315 return 0; 316 } 317 318 static int 319 posix_sock_set_recvbuf(struct spdk_sock *_sock, int sz) 320 { 321 struct spdk_posix_sock *sock = __posix_sock(_sock); 322 int min_size; 323 int rc; 324 325 assert(sock != NULL); 326 327 if (_sock->impl_opts.enable_recv_pipe) { 328 rc = posix_sock_alloc_pipe(sock, sz); 329 if (rc) { 330 return rc; 331 } 332 } 333 334 /* Set kernel buffer size to be at least MIN_SO_RCVBUF_SIZE and 335 * g_spdk_posix_sock_impl_opts.recv_buf_size. */ 336 min_size = spdk_max(MIN_SO_RCVBUF_SIZE, g_spdk_posix_sock_impl_opts.recv_buf_size); 337 338 if (sz < min_size) { 339 sz = min_size; 340 } 341 342 rc = setsockopt(sock->fd, SOL_SOCKET, SO_RCVBUF, &sz, sizeof(sz)); 343 if (rc < 0) { 344 return rc; 345 } 346 347 _sock->impl_opts.recv_buf_size = sz; 348 349 return 0; 350 } 351 352 static int 353 posix_sock_set_sendbuf(struct spdk_sock *_sock, int sz) 354 { 355 struct spdk_posix_sock *sock = __posix_sock(_sock); 356 int min_size; 357 int rc; 358 359 assert(sock != NULL); 360 361 /* Set kernel buffer size to be at least MIN_SO_SNDBUF_SIZE and 362 * g_spdk_posix_sock_impl_opts.send_buf_size. */ 363 min_size = spdk_max(MIN_SO_SNDBUF_SIZE, g_spdk_posix_sock_impl_opts.send_buf_size); 364 365 if (sz < min_size) { 366 sz = min_size; 367 } 368 369 rc = setsockopt(sock->fd, SOL_SOCKET, SO_SNDBUF, &sz, sizeof(sz)); 370 if (rc < 0) { 371 return rc; 372 } 373 374 _sock->impl_opts.send_buf_size = sz; 375 376 return 0; 377 } 378 379 static void 380 posix_sock_init(struct spdk_posix_sock *sock, bool enable_zero_copy) 381 { 382 #if defined(SPDK_ZEROCOPY) || defined(__linux__) 383 int flag; 384 int rc; 385 #endif 386 387 #if defined(SPDK_ZEROCOPY) 388 flag = 1; 389 390 if (enable_zero_copy) { 391 /* Try to turn on zero copy sends */ 392 rc = setsockopt(sock->fd, SOL_SOCKET, SO_ZEROCOPY, &flag, sizeof(flag)); 393 if (rc == 0) { 394 sock->zcopy = true; 395 } 396 } 397 #endif 398 399 #if defined(__linux__) 400 flag = 1; 401 402 if (sock->base.impl_opts.enable_quickack) { 403 rc = setsockopt(sock->fd, IPPROTO_TCP, TCP_QUICKACK, &flag, sizeof(flag)); 404 if (rc != 0) { 405 SPDK_ERRLOG("quickack was failed to set\n"); 406 } 407 } 408 409 spdk_sock_get_placement_id(sock->fd, sock->base.impl_opts.enable_placement_id, 410 &sock->placement_id); 411 412 if (sock->base.impl_opts.enable_placement_id == PLACEMENT_MARK) { 413 /* Save placement_id */ 414 spdk_sock_map_insert(&g_map, sock->placement_id, NULL); 415 } 416 #endif 417 } 418 419 static struct spdk_posix_sock * 420 posix_sock_alloc(int fd, struct spdk_sock_impl_opts *impl_opts, bool enable_zero_copy) 421 { 422 struct spdk_posix_sock *sock; 423 424 sock = calloc(1, sizeof(*sock)); 425 if (sock == NULL) { 426 SPDK_ERRLOG("sock allocation failed\n"); 427 return NULL; 428 } 429 430 sock->fd = fd; 431 memcpy(&sock->base.impl_opts, impl_opts, sizeof(*impl_opts)); 432 posix_sock_init(sock, enable_zero_copy); 433 434 return sock; 435 } 436 437 static int 438 posix_fd_create(struct addrinfo *res, struct spdk_sock_opts *opts, 439 struct spdk_sock_impl_opts *impl_opts) 440 { 441 int fd; 442 int val = 1; 443 int rc, sz; 444 #if defined(__linux__) 445 int to; 446 #endif 447 448 fd = socket(res->ai_family, res->ai_socktype, res->ai_protocol); 449 if (fd < 0) { 450 /* error */ 451 return -1; 452 } 453 454 sz = impl_opts->recv_buf_size; 455 rc = setsockopt(fd, SOL_SOCKET, SO_RCVBUF, &sz, sizeof(sz)); 456 if (rc) { 457 /* Not fatal */ 458 } 459 460 sz = impl_opts->send_buf_size; 461 rc = setsockopt(fd, SOL_SOCKET, SO_SNDBUF, &sz, sizeof(sz)); 462 if (rc) { 463 /* Not fatal */ 464 } 465 466 rc = setsockopt(fd, SOL_SOCKET, SO_REUSEADDR, &val, sizeof val); 467 if (rc != 0) { 468 close(fd); 469 /* error */ 470 return -1; 471 } 472 rc = setsockopt(fd, IPPROTO_TCP, TCP_NODELAY, &val, sizeof val); 473 if (rc != 0) { 474 close(fd); 475 /* error */ 476 return -1; 477 } 478 479 #if defined(SO_PRIORITY) 480 if (opts->priority) { 481 rc = setsockopt(fd, SOL_SOCKET, SO_PRIORITY, &opts->priority, sizeof val); 482 if (rc != 0) { 483 close(fd); 484 /* error */ 485 return -1; 486 } 487 } 488 #endif 489 490 if (res->ai_family == AF_INET6) { 491 rc = setsockopt(fd, IPPROTO_IPV6, IPV6_V6ONLY, &val, sizeof val); 492 if (rc != 0) { 493 close(fd); 494 /* error */ 495 return -1; 496 } 497 } 498 499 if (opts->ack_timeout) { 500 #if defined(__linux__) 501 to = opts->ack_timeout; 502 rc = setsockopt(fd, IPPROTO_TCP, TCP_USER_TIMEOUT, &to, sizeof(to)); 503 if (rc != 0) { 504 close(fd); 505 /* error */ 506 return -1; 507 } 508 #else 509 SPDK_WARNLOG("TCP_USER_TIMEOUT is not supported.\n"); 510 #endif 511 } 512 513 return fd; 514 } 515 516 static unsigned int 517 posix_sock_tls_psk_server_cb(SSL *ssl, 518 const char *id, 519 unsigned char *psk, 520 unsigned int max_psk_len) 521 { 522 long key_len; 523 unsigned char *default_psk; 524 struct spdk_sock_impl_opts *impl_opts; 525 526 impl_opts = SSL_get_app_data(ssl); 527 528 if (impl_opts->psk_key == NULL) { 529 SPDK_ERRLOG("PSK is not set\n"); 530 goto err; 531 } 532 SPDK_DEBUGLOG(sock_posix, "Length of Client's PSK ID %lu\n", strlen(impl_opts->psk_identity)); 533 if (id == NULL) { 534 SPDK_ERRLOG("Received empty PSK ID\n"); 535 goto err; 536 } 537 SPDK_DEBUGLOG(sock_posix, "Received PSK ID '%s'\n", id); 538 if (strcmp(impl_opts->psk_identity, id) != 0) { 539 SPDK_ERRLOG("Unknown Client's PSK ID\n"); 540 goto err; 541 } 542 543 SPDK_DEBUGLOG(sock_posix, "Length of Client's PSK KEY %u\n", max_psk_len); 544 default_psk = OPENSSL_hexstr2buf(impl_opts->psk_key, &key_len); 545 if (default_psk == NULL) { 546 SPDK_ERRLOG("Could not unhexlify PSK\n"); 547 goto err; 548 } 549 if (key_len > max_psk_len) { 550 SPDK_ERRLOG("Insufficient buffer size to copy PSK\n"); 551 OPENSSL_free(default_psk); 552 goto err; 553 } 554 555 memcpy(psk, default_psk, key_len); 556 OPENSSL_free(default_psk); 557 558 return key_len; 559 560 err: 561 return 0; 562 } 563 564 static unsigned int 565 posix_sock_tls_psk_client_cb(SSL *ssl, const char *hint, 566 char *identity, 567 unsigned int max_identity_len, 568 unsigned char *psk, 569 unsigned int max_psk_len) 570 { 571 long key_len; 572 unsigned char *default_psk; 573 struct spdk_sock_impl_opts *impl_opts; 574 575 impl_opts = SSL_get_app_data(ssl); 576 577 if (hint) { 578 SPDK_DEBUGLOG(sock_posix, "Received PSK identity hint '%s'\n", hint); 579 } 580 581 if (impl_opts->psk_key == NULL) { 582 SPDK_ERRLOG("PSK is not set\n"); 583 goto err; 584 } 585 default_psk = OPENSSL_hexstr2buf(impl_opts->psk_key, &key_len); 586 if (default_psk == NULL) { 587 SPDK_ERRLOG("Could not unhexlify PSK\n"); 588 goto err; 589 } 590 if ((strlen(impl_opts->psk_identity) + 1 > max_identity_len) 591 || (key_len > max_psk_len)) { 592 OPENSSL_free(default_psk); 593 SPDK_ERRLOG("PSK ID or Key buffer is not sufficient\n"); 594 goto err; 595 } 596 spdk_strcpy_pad(identity, impl_opts->psk_identity, strlen(impl_opts->psk_identity), 0); 597 SPDK_DEBUGLOG(sock_posix, "Sending PSK identity '%s'\n", identity); 598 599 memcpy(psk, default_psk, key_len); 600 SPDK_DEBUGLOG(sock_posix, "Provided out-of-band (OOB) PSK for TLS1.3 client\n"); 601 OPENSSL_free(default_psk); 602 603 return key_len; 604 605 err: 606 return 0; 607 } 608 609 static SSL_CTX * 610 posix_sock_create_ssl_context(const SSL_METHOD *method, struct spdk_sock_opts *opts, 611 struct spdk_sock_impl_opts *impl_opts) 612 { 613 SSL_CTX *ctx; 614 int tls_version = 0; 615 bool ktls_enabled = false; 616 #ifdef SSL_OP_ENABLE_KTLS 617 long options; 618 #endif 619 620 SSL_library_init(); 621 OpenSSL_add_all_algorithms(); 622 SSL_load_error_strings(); 623 /* Produce a SSL CTX in SSL V2 and V3 standards compliant way */ 624 ctx = SSL_CTX_new(method); 625 if (!ctx) { 626 SPDK_ERRLOG("SSL_CTX_new() failed, msg = %s\n", ERR_error_string(ERR_peek_last_error(), NULL)); 627 return NULL; 628 } 629 SPDK_DEBUGLOG(sock_posix, "SSL context created\n"); 630 631 switch (impl_opts->tls_version) { 632 case 0: 633 /* auto-negotioation */ 634 break; 635 case SPDK_TLS_VERSION_1_1: 636 tls_version = TLS1_1_VERSION; 637 break; 638 case SPDK_TLS_VERSION_1_2: 639 tls_version = TLS1_2_VERSION; 640 break; 641 case SPDK_TLS_VERSION_1_3: 642 tls_version = TLS1_3_VERSION; 643 break; 644 default: 645 SPDK_ERRLOG("Incorrect TLS version provided: %d\n", impl_opts->tls_version); 646 goto err; 647 } 648 649 if (tls_version) { 650 SPDK_DEBUGLOG(sock_posix, "Hardening TLS version to '%d'='0x%X'\n", impl_opts->tls_version, 651 tls_version); 652 if (!SSL_CTX_set_min_proto_version(ctx, tls_version)) { 653 SPDK_ERRLOG("Unable to set Min TLS version to '%d'='0x%X\n", impl_opts->tls_version, tls_version); 654 goto err; 655 } 656 if (!SSL_CTX_set_max_proto_version(ctx, tls_version)) { 657 SPDK_ERRLOG("Unable to set Max TLS version to '%d'='0x%X\n", impl_opts->tls_version, tls_version); 658 goto err; 659 } 660 } 661 if (impl_opts->enable_ktls) { 662 SPDK_DEBUGLOG(sock_posix, "Enabling kTLS offload\n"); 663 #ifdef SSL_OP_ENABLE_KTLS 664 options = SSL_CTX_set_options(ctx, SSL_OP_ENABLE_KTLS); 665 ktls_enabled = options & SSL_OP_ENABLE_KTLS; 666 #else 667 ktls_enabled = false; 668 #endif 669 if (!ktls_enabled) { 670 SPDK_ERRLOG("Unable to set kTLS offload via SSL_CTX_set_options(). Configure openssl with 'enable-ktls'\n"); 671 goto err; 672 } 673 } 674 675 return ctx; 676 677 err: 678 SSL_CTX_free(ctx); 679 return NULL; 680 } 681 682 static SSL * 683 ssl_sock_connect_loop(SSL_CTX *ctx, int fd, struct spdk_sock_impl_opts *impl_opts) 684 { 685 int rc; 686 SSL *ssl; 687 int ssl_get_error; 688 689 ssl = SSL_new(ctx); 690 if (!ssl) { 691 SPDK_ERRLOG("SSL_new() failed, msg = %s\n", ERR_error_string(ERR_peek_last_error(), NULL)); 692 return NULL; 693 } 694 SSL_set_fd(ssl, fd); 695 SSL_set_app_data(ssl, impl_opts); 696 SSL_set_psk_client_callback(ssl, posix_sock_tls_psk_client_cb); 697 SPDK_DEBUGLOG(sock_posix, "SSL object creation finished: %p\n", ssl); 698 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 699 while ((rc = SSL_connect(ssl)) != 1) { 700 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 701 ssl_get_error = SSL_get_error(ssl, rc); 702 SPDK_DEBUGLOG(sock_posix, "SSL_connect failed %d = SSL_connect(%p), %d = SSL_get_error(%p, %d)\n", 703 rc, ssl, ssl_get_error, ssl, rc); 704 switch (ssl_get_error) { 705 case SSL_ERROR_WANT_READ: 706 case SSL_ERROR_WANT_WRITE: 707 continue; 708 default: 709 break; 710 } 711 SPDK_ERRLOG("SSL_connect() failed, errno = %d\n", errno); 712 SSL_free(ssl); 713 return NULL; 714 } 715 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 716 SPDK_DEBUGLOG(sock_posix, "Negotiated Cipher suite:%s\n", 717 SSL_CIPHER_get_name(SSL_get_current_cipher(ssl))); 718 return ssl; 719 } 720 721 static SSL * 722 ssl_sock_accept_loop(SSL_CTX *ctx, int fd, struct spdk_sock_impl_opts *impl_opts) 723 { 724 int rc; 725 SSL *ssl; 726 int ssl_get_error; 727 728 ssl = SSL_new(ctx); 729 if (!ssl) { 730 SPDK_ERRLOG("SSL_new() failed, msg = %s\n", ERR_error_string(ERR_peek_last_error(), NULL)); 731 return NULL; 732 } 733 SSL_set_fd(ssl, fd); 734 SSL_set_app_data(ssl, impl_opts); 735 SSL_set_psk_server_callback(ssl, posix_sock_tls_psk_server_cb); 736 SPDK_DEBUGLOG(sock_posix, "SSL object creation finished: %p\n", ssl); 737 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 738 while ((rc = SSL_accept(ssl)) != 1) { 739 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 740 ssl_get_error = SSL_get_error(ssl, rc); 741 SPDK_DEBUGLOG(sock_posix, "SSL_accept failed %d = SSL_accept(%p), %d = SSL_get_error(%p, %d)\n", rc, 742 ssl, ssl_get_error, ssl, rc); 743 switch (ssl_get_error) { 744 case SSL_ERROR_WANT_READ: 745 case SSL_ERROR_WANT_WRITE: 746 continue; 747 default: 748 break; 749 } 750 SPDK_ERRLOG("SSL_accept() failed, errno = %d\n", errno); 751 SSL_free(ssl); 752 return NULL; 753 } 754 SPDK_DEBUGLOG(sock_posix, "%s = SSL_state_string_long(%p)\n", SSL_state_string_long(ssl), ssl); 755 SPDK_DEBUGLOG(sock_posix, "Negotiated Cipher suite:%s\n", 756 SSL_CIPHER_get_name(SSL_get_current_cipher(ssl))); 757 return ssl; 758 } 759 760 static ssize_t 761 SSL_readv(SSL *ssl, const struct iovec *iov, int iovcnt) 762 { 763 int i, rc = 0; 764 ssize_t total = 0; 765 766 for (i = 0; i < iovcnt; i++) { 767 rc = SSL_read(ssl, iov[i].iov_base, iov[i].iov_len); 768 769 if (rc > 0) { 770 total += rc; 771 } 772 if (rc != (int)iov[i].iov_len) { 773 break; 774 } 775 } 776 if (total > 0) { 777 errno = 0; 778 return total; 779 } 780 switch (SSL_get_error(ssl, rc)) { 781 case SSL_ERROR_ZERO_RETURN: 782 errno = ENOTCONN; 783 return 0; 784 case SSL_ERROR_WANT_READ: 785 case SSL_ERROR_WANT_WRITE: 786 case SSL_ERROR_WANT_CONNECT: 787 case SSL_ERROR_WANT_ACCEPT: 788 case SSL_ERROR_WANT_X509_LOOKUP: 789 case SSL_ERROR_WANT_ASYNC: 790 case SSL_ERROR_WANT_ASYNC_JOB: 791 case SSL_ERROR_WANT_CLIENT_HELLO_CB: 792 errno = EAGAIN; 793 return -1; 794 case SSL_ERROR_SYSCALL: 795 case SSL_ERROR_SSL: 796 errno = ENOTCONN; 797 return -1; 798 default: 799 errno = ENOTCONN; 800 return -1; 801 } 802 } 803 804 static ssize_t 805 SSL_writev(SSL *ssl, struct iovec *iov, int iovcnt) 806 { 807 int i, rc = 0; 808 ssize_t total = 0; 809 810 for (i = 0; i < iovcnt; i++) { 811 rc = SSL_write(ssl, iov[i].iov_base, iov[i].iov_len); 812 813 if (rc > 0) { 814 total += rc; 815 } 816 if (rc != (int)iov[i].iov_len) { 817 break; 818 } 819 } 820 if (total > 0) { 821 errno = 0; 822 return total; 823 } 824 switch (SSL_get_error(ssl, rc)) { 825 case SSL_ERROR_ZERO_RETURN: 826 errno = ENOTCONN; 827 return 0; 828 case SSL_ERROR_WANT_READ: 829 case SSL_ERROR_WANT_WRITE: 830 case SSL_ERROR_WANT_CONNECT: 831 case SSL_ERROR_WANT_ACCEPT: 832 case SSL_ERROR_WANT_X509_LOOKUP: 833 case SSL_ERROR_WANT_ASYNC: 834 case SSL_ERROR_WANT_ASYNC_JOB: 835 case SSL_ERROR_WANT_CLIENT_HELLO_CB: 836 errno = EAGAIN; 837 return -1; 838 case SSL_ERROR_SYSCALL: 839 case SSL_ERROR_SSL: 840 errno = ENOTCONN; 841 return -1; 842 default: 843 errno = ENOTCONN; 844 return -1; 845 } 846 } 847 848 static struct spdk_sock * 849 posix_sock_create(const char *ip, int port, 850 enum posix_sock_create_type type, 851 struct spdk_sock_opts *opts, 852 bool enable_ssl) 853 { 854 struct spdk_posix_sock *sock; 855 struct spdk_sock_impl_opts impl_opts; 856 char buf[MAX_TMPBUF]; 857 char portnum[PORTNUMLEN]; 858 char *p; 859 struct addrinfo hints, *res, *res0; 860 int fd, flag; 861 int rc; 862 bool enable_zcopy_user_opts = true; 863 bool enable_zcopy_impl_opts = true; 864 SSL_CTX *ctx = 0; 865 SSL *ssl = 0; 866 867 assert(opts != NULL); 868 posix_opts_get_impl_opts(opts, &impl_opts); 869 870 if (ip == NULL) { 871 return NULL; 872 } 873 if (ip[0] == '[') { 874 snprintf(buf, sizeof(buf), "%s", ip + 1); 875 p = strchr(buf, ']'); 876 if (p != NULL) { 877 *p = '\0'; 878 } 879 ip = (const char *) &buf[0]; 880 } 881 882 snprintf(portnum, sizeof portnum, "%d", port); 883 memset(&hints, 0, sizeof hints); 884 hints.ai_family = PF_UNSPEC; 885 hints.ai_socktype = SOCK_STREAM; 886 hints.ai_flags = AI_NUMERICSERV; 887 hints.ai_flags |= AI_PASSIVE; 888 hints.ai_flags |= AI_NUMERICHOST; 889 rc = getaddrinfo(ip, portnum, &hints, &res0); 890 if (rc != 0) { 891 SPDK_ERRLOG("getaddrinfo() failed %s (%d)\n", gai_strerror(rc), rc); 892 return NULL; 893 } 894 895 /* try listen */ 896 fd = -1; 897 for (res = res0; res != NULL; res = res->ai_next) { 898 retry: 899 fd = posix_fd_create(res, opts, &impl_opts); 900 if (fd < 0) { 901 continue; 902 } 903 if (type == SPDK_SOCK_CREATE_LISTEN) { 904 rc = bind(fd, res->ai_addr, res->ai_addrlen); 905 if (rc != 0) { 906 SPDK_ERRLOG("bind() failed at port %d, errno = %d\n", port, errno); 907 switch (errno) { 908 case EINTR: 909 /* interrupted? */ 910 close(fd); 911 goto retry; 912 case EADDRNOTAVAIL: 913 SPDK_ERRLOG("IP address %s not available. " 914 "Verify IP address in config file " 915 "and make sure setup script is " 916 "run before starting spdk app.\n", ip); 917 /* FALLTHROUGH */ 918 default: 919 /* try next family */ 920 close(fd); 921 fd = -1; 922 continue; 923 } 924 } 925 /* bind OK */ 926 rc = listen(fd, 512); 927 if (rc != 0) { 928 SPDK_ERRLOG("listen() failed, errno = %d\n", errno); 929 close(fd); 930 fd = -1; 931 break; 932 } 933 enable_zcopy_impl_opts = impl_opts.enable_zerocopy_send_server; 934 } else if (type == SPDK_SOCK_CREATE_CONNECT) { 935 rc = connect(fd, res->ai_addr, res->ai_addrlen); 936 if (rc != 0) { 937 SPDK_ERRLOG("connect() failed, errno = %d\n", errno); 938 /* try next family */ 939 close(fd); 940 fd = -1; 941 continue; 942 } 943 enable_zcopy_impl_opts = impl_opts.enable_zerocopy_send_client; 944 if (enable_ssl) { 945 ctx = posix_sock_create_ssl_context(TLS_client_method(), opts, &impl_opts); 946 if (!ctx) { 947 SPDK_ERRLOG("posix_sock_create_ssl_context() failed, errno = %d\n", errno); 948 close(fd); 949 fd = -1; 950 break; 951 } 952 ssl = ssl_sock_connect_loop(ctx, fd, &impl_opts); 953 if (!ssl) { 954 SPDK_ERRLOG("ssl_sock_connect_loop() failed, errno = %d\n", errno); 955 close(fd); 956 fd = -1; 957 SSL_CTX_free(ctx); 958 break; 959 } 960 } 961 } 962 963 flag = fcntl(fd, F_GETFL); 964 if (fcntl(fd, F_SETFL, flag | O_NONBLOCK) < 0) { 965 SPDK_ERRLOG("fcntl can't set nonblocking mode for socket, fd: %d (%d)\n", fd, errno); 966 SSL_free(ssl); 967 SSL_CTX_free(ctx); 968 close(fd); 969 fd = -1; 970 break; 971 } 972 break; 973 } 974 freeaddrinfo(res0); 975 976 if (fd < 0) { 977 return NULL; 978 } 979 980 /* Only enable zero copy for non-loopback and non-ssl sockets. */ 981 enable_zcopy_user_opts = opts->zcopy && !sock_is_loopback(fd) && !enable_ssl; 982 983 sock = posix_sock_alloc(fd, &impl_opts, enable_zcopy_user_opts && enable_zcopy_impl_opts); 984 if (sock == NULL) { 985 SPDK_ERRLOG("sock allocation failed\n"); 986 SSL_free(ssl); 987 SSL_CTX_free(ctx); 988 close(fd); 989 return NULL; 990 } 991 992 if (ctx) { 993 sock->ctx = ctx; 994 } 995 996 if (ssl) { 997 sock->ssl = ssl; 998 } 999 1000 return &sock->base; 1001 } 1002 1003 static struct spdk_sock * 1004 posix_sock_listen(const char *ip, int port, struct spdk_sock_opts *opts) 1005 { 1006 return posix_sock_create(ip, port, SPDK_SOCK_CREATE_LISTEN, opts, false); 1007 } 1008 1009 static struct spdk_sock * 1010 posix_sock_connect(const char *ip, int port, struct spdk_sock_opts *opts) 1011 { 1012 return posix_sock_create(ip, port, SPDK_SOCK_CREATE_CONNECT, opts, false); 1013 } 1014 1015 static struct spdk_sock * 1016 _posix_sock_accept(struct spdk_sock *_sock, bool enable_ssl) 1017 { 1018 struct spdk_posix_sock *sock = __posix_sock(_sock); 1019 struct sockaddr_storage sa; 1020 socklen_t salen; 1021 int rc, fd; 1022 struct spdk_posix_sock *new_sock; 1023 int flag; 1024 SSL_CTX *ctx = 0; 1025 SSL *ssl = 0; 1026 1027 memset(&sa, 0, sizeof(sa)); 1028 salen = sizeof(sa); 1029 1030 assert(sock != NULL); 1031 1032 rc = accept(sock->fd, (struct sockaddr *)&sa, &salen); 1033 1034 if (rc == -1) { 1035 return NULL; 1036 } 1037 1038 fd = rc; 1039 1040 flag = fcntl(fd, F_GETFL); 1041 if ((!(flag & O_NONBLOCK)) && (fcntl(fd, F_SETFL, flag | O_NONBLOCK) < 0)) { 1042 SPDK_ERRLOG("fcntl can't set nonblocking mode for socket, fd: %d (%d)\n", fd, errno); 1043 close(fd); 1044 return NULL; 1045 } 1046 1047 #if defined(SO_PRIORITY) 1048 /* The priority is not inherited, so call this function again */ 1049 if (sock->base.opts.priority) { 1050 rc = setsockopt(fd, SOL_SOCKET, SO_PRIORITY, &sock->base.opts.priority, sizeof(int)); 1051 if (rc != 0) { 1052 close(fd); 1053 return NULL; 1054 } 1055 } 1056 #endif 1057 1058 /* Establish SSL connection */ 1059 if (enable_ssl) { 1060 ctx = posix_sock_create_ssl_context(TLS_server_method(), &sock->base.opts, &sock->base.impl_opts); 1061 if (!ctx) { 1062 SPDK_ERRLOG("posix_sock_create_ssl_context() failed, errno = %d\n", errno); 1063 close(fd); 1064 return NULL; 1065 } 1066 ssl = ssl_sock_accept_loop(ctx, fd, &sock->base.impl_opts); 1067 if (!ssl) { 1068 SPDK_ERRLOG("ssl_sock_accept_loop() failed, errno = %d\n", errno); 1069 close(fd); 1070 SSL_CTX_free(ctx); 1071 return NULL; 1072 } 1073 } 1074 1075 /* Inherit the zero copy feature from the listen socket */ 1076 new_sock = posix_sock_alloc(fd, &sock->base.impl_opts, sock->zcopy); 1077 if (new_sock == NULL) { 1078 close(fd); 1079 SSL_free(ssl); 1080 SSL_CTX_free(ctx); 1081 return NULL; 1082 } 1083 1084 if (ctx) { 1085 new_sock->ctx = ctx; 1086 } 1087 1088 if (ssl) { 1089 new_sock->ssl = ssl; 1090 } 1091 1092 return &new_sock->base; 1093 } 1094 1095 static struct spdk_sock * 1096 posix_sock_accept(struct spdk_sock *_sock) 1097 { 1098 return _posix_sock_accept(_sock, false); 1099 } 1100 1101 static int 1102 posix_sock_close(struct spdk_sock *_sock) 1103 { 1104 struct spdk_posix_sock *sock = __posix_sock(_sock); 1105 1106 assert(TAILQ_EMPTY(&_sock->pending_reqs)); 1107 1108 if (sock->ssl != NULL) { 1109 SSL_shutdown(sock->ssl); 1110 } 1111 1112 /* If the socket fails to close, the best choice is to 1113 * leak the fd but continue to free the rest of the sock 1114 * memory. */ 1115 close(sock->fd); 1116 1117 SSL_free(sock->ssl); 1118 SSL_CTX_free(sock->ctx); 1119 1120 spdk_pipe_destroy(sock->recv_pipe); 1121 free(sock->recv_buf); 1122 free(sock); 1123 1124 return 0; 1125 } 1126 1127 #ifdef SPDK_ZEROCOPY 1128 static int 1129 _sock_check_zcopy(struct spdk_sock *sock) 1130 { 1131 struct spdk_posix_sock *psock = __posix_sock(sock); 1132 struct msghdr msgh = {}; 1133 uint8_t buf[sizeof(struct cmsghdr) + sizeof(struct sock_extended_err)]; 1134 ssize_t rc; 1135 struct sock_extended_err *serr; 1136 struct cmsghdr *cm; 1137 uint32_t idx; 1138 struct spdk_sock_request *req, *treq; 1139 bool found; 1140 1141 msgh.msg_control = buf; 1142 msgh.msg_controllen = sizeof(buf); 1143 1144 while (true) { 1145 rc = recvmsg(psock->fd, &msgh, MSG_ERRQUEUE); 1146 1147 if (rc < 0) { 1148 if (errno == EWOULDBLOCK || errno == EAGAIN) { 1149 return 0; 1150 } 1151 1152 if (!TAILQ_EMPTY(&sock->pending_reqs)) { 1153 SPDK_ERRLOG("Attempting to receive from ERRQUEUE yielded error, but pending list still has orphaned entries\n"); 1154 } else { 1155 SPDK_WARNLOG("Recvmsg yielded an error!\n"); 1156 } 1157 return 0; 1158 } 1159 1160 cm = CMSG_FIRSTHDR(&msgh); 1161 if (!(cm && 1162 ((cm->cmsg_level == SOL_IP && cm->cmsg_type == IP_RECVERR) || 1163 (cm->cmsg_level == SOL_IPV6 && cm->cmsg_type == IPV6_RECVERR)))) { 1164 SPDK_WARNLOG("Unexpected cmsg level or type!\n"); 1165 return 0; 1166 } 1167 1168 serr = (struct sock_extended_err *)CMSG_DATA(cm); 1169 if (serr->ee_errno != 0 || serr->ee_origin != SO_EE_ORIGIN_ZEROCOPY) { 1170 SPDK_WARNLOG("Unexpected extended error origin\n"); 1171 return 0; 1172 } 1173 1174 /* Most of the time, the pending_reqs array is in the exact 1175 * order we need such that all of the requests to complete are 1176 * in order, in the front. It is guaranteed that all requests 1177 * belonging to the same sendmsg call are sequential, so once 1178 * we encounter one match we can stop looping as soon as a 1179 * non-match is found. 1180 */ 1181 idx = serr->ee_info; 1182 while (true) { 1183 found = false; 1184 TAILQ_FOREACH_SAFE(req, &sock->pending_reqs, internal.link, treq) { 1185 if (!req->internal.is_zcopy) { 1186 /* This wasn't a zcopy request. It was just waiting in line to complete */ 1187 rc = spdk_sock_request_put(sock, req, 0); 1188 if (rc < 0) { 1189 return rc; 1190 } 1191 } else if (req->internal.offset == idx) { 1192 found = true; 1193 rc = spdk_sock_request_put(sock, req, 0); 1194 if (rc < 0) { 1195 return rc; 1196 } 1197 } else if (found) { 1198 break; 1199 } 1200 } 1201 1202 if (idx == serr->ee_data) { 1203 break; 1204 } 1205 1206 if (idx == UINT32_MAX) { 1207 idx = 0; 1208 } else { 1209 idx++; 1210 } 1211 } 1212 } 1213 1214 return 0; 1215 } 1216 #endif 1217 1218 static int 1219 _sock_flush(struct spdk_sock *sock) 1220 { 1221 struct spdk_posix_sock *psock = __posix_sock(sock); 1222 struct msghdr msg = {}; 1223 int flags; 1224 struct iovec iovs[IOV_BATCH_SIZE]; 1225 int iovcnt; 1226 int retval; 1227 struct spdk_sock_request *req; 1228 int i; 1229 ssize_t rc, sent; 1230 unsigned int offset; 1231 size_t len; 1232 bool is_zcopy = false; 1233 1234 /* Can't flush from within a callback or we end up with recursive calls */ 1235 if (sock->cb_cnt > 0) { 1236 errno = EAGAIN; 1237 return -1; 1238 } 1239 1240 #ifdef SPDK_ZEROCOPY 1241 if (psock->zcopy) { 1242 flags = MSG_ZEROCOPY | MSG_NOSIGNAL; 1243 } else 1244 #endif 1245 { 1246 flags = MSG_NOSIGNAL; 1247 } 1248 1249 iovcnt = spdk_sock_prep_reqs(sock, iovs, 0, NULL, &flags); 1250 if (iovcnt == 0) { 1251 return 0; 1252 } 1253 1254 #ifdef SPDK_ZEROCOPY 1255 is_zcopy = flags & MSG_ZEROCOPY; 1256 #endif 1257 1258 /* Perform the vectored write */ 1259 msg.msg_iov = iovs; 1260 msg.msg_iovlen = iovcnt; 1261 1262 if (psock->ssl) { 1263 rc = SSL_writev(psock->ssl, iovs, iovcnt); 1264 } else { 1265 rc = sendmsg(psock->fd, &msg, flags); 1266 } 1267 if (rc <= 0) { 1268 if (rc == 0 || errno == EAGAIN || errno == EWOULDBLOCK || (errno == ENOBUFS && psock->zcopy)) { 1269 errno = EAGAIN; 1270 } 1271 return -1; 1272 } 1273 1274 sent = rc; 1275 1276 if (is_zcopy) { 1277 /* Handling overflow case, because we use psock->sendmsg_idx - 1 for the 1278 * req->internal.offset, so sendmsg_idx should not be zero */ 1279 if (spdk_unlikely(psock->sendmsg_idx == UINT32_MAX)) { 1280 psock->sendmsg_idx = 1; 1281 } else { 1282 psock->sendmsg_idx++; 1283 } 1284 } 1285 1286 /* Consume the requests that were actually written */ 1287 req = TAILQ_FIRST(&sock->queued_reqs); 1288 while (req) { 1289 offset = req->internal.offset; 1290 1291 /* req->internal.is_zcopy is true when the whole req or part of it is sent with zerocopy */ 1292 req->internal.is_zcopy = is_zcopy; 1293 1294 for (i = 0; i < req->iovcnt; i++) { 1295 /* Advance by the offset first */ 1296 if (offset >= SPDK_SOCK_REQUEST_IOV(req, i)->iov_len) { 1297 offset -= SPDK_SOCK_REQUEST_IOV(req, i)->iov_len; 1298 continue; 1299 } 1300 1301 /* Calculate the remaining length of this element */ 1302 len = SPDK_SOCK_REQUEST_IOV(req, i)->iov_len - offset; 1303 1304 if (len > (size_t)rc) { 1305 /* This element was partially sent. */ 1306 req->internal.offset += rc; 1307 return sent; 1308 } 1309 1310 offset = 0; 1311 req->internal.offset += len; 1312 rc -= len; 1313 } 1314 1315 /* Handled a full request. */ 1316 spdk_sock_request_pend(sock, req); 1317 1318 if (!req->internal.is_zcopy && req == TAILQ_FIRST(&sock->pending_reqs)) { 1319 /* The sendmsg syscall above isn't currently asynchronous, 1320 * so it's already done. */ 1321 retval = spdk_sock_request_put(sock, req, 0); 1322 if (retval) { 1323 break; 1324 } 1325 } else { 1326 /* Re-use the offset field to hold the sendmsg call index. The 1327 * index is 0 based, so subtract one here because we've already 1328 * incremented above. */ 1329 req->internal.offset = psock->sendmsg_idx - 1; 1330 } 1331 1332 if (rc == 0) { 1333 break; 1334 } 1335 1336 req = TAILQ_FIRST(&sock->queued_reqs); 1337 } 1338 1339 return sent; 1340 } 1341 1342 static int 1343 posix_sock_flush(struct spdk_sock *sock) 1344 { 1345 #ifdef SPDK_ZEROCOPY 1346 struct spdk_posix_sock *psock = __posix_sock(sock); 1347 1348 if (psock->zcopy && !TAILQ_EMPTY(&sock->pending_reqs)) { 1349 _sock_check_zcopy(sock); 1350 } 1351 #endif 1352 1353 return _sock_flush(sock); 1354 } 1355 1356 static ssize_t 1357 posix_sock_recv_from_pipe(struct spdk_posix_sock *sock, struct iovec *diov, int diovcnt) 1358 { 1359 struct iovec siov[2]; 1360 int sbytes; 1361 ssize_t bytes; 1362 struct spdk_posix_sock_group_impl *group; 1363 1364 sbytes = spdk_pipe_reader_get_buffer(sock->recv_pipe, sock->recv_buf_sz, siov); 1365 if (sbytes < 0) { 1366 errno = EINVAL; 1367 return -1; 1368 } else if (sbytes == 0) { 1369 errno = EAGAIN; 1370 return -1; 1371 } 1372 1373 bytes = spdk_iovcpy(siov, 2, diov, diovcnt); 1374 1375 if (bytes == 0) { 1376 /* The only way this happens is if diov is 0 length */ 1377 errno = EINVAL; 1378 return -1; 1379 } 1380 1381 spdk_pipe_reader_advance(sock->recv_pipe, bytes); 1382 1383 /* If we drained the pipe, mark it appropriately */ 1384 if (spdk_pipe_reader_bytes_available(sock->recv_pipe) == 0) { 1385 assert(sock->pipe_has_data == true); 1386 1387 group = __posix_group_impl(sock->base.group_impl); 1388 if (group && !sock->socket_has_data) { 1389 TAILQ_REMOVE(&group->socks_with_data, sock, link); 1390 } 1391 1392 sock->pipe_has_data = false; 1393 } 1394 1395 return bytes; 1396 } 1397 1398 static inline ssize_t 1399 posix_sock_read(struct spdk_posix_sock *sock) 1400 { 1401 struct iovec iov[2]; 1402 int bytes_avail, bytes_recvd; 1403 struct spdk_posix_sock_group_impl *group; 1404 1405 bytes_avail = spdk_pipe_writer_get_buffer(sock->recv_pipe, sock->recv_buf_sz, iov); 1406 1407 if (bytes_avail <= 0) { 1408 return bytes_avail; 1409 } 1410 1411 if (sock->ssl) { 1412 bytes_recvd = SSL_readv(sock->ssl, iov, 2); 1413 } else { 1414 bytes_recvd = readv(sock->fd, iov, 2); 1415 } 1416 1417 assert(sock->pipe_has_data == false); 1418 1419 if (bytes_recvd <= 0) { 1420 /* Errors count as draining the socket data */ 1421 if (sock->base.group_impl && sock->socket_has_data) { 1422 group = __posix_group_impl(sock->base.group_impl); 1423 TAILQ_REMOVE(&group->socks_with_data, sock, link); 1424 } 1425 1426 sock->socket_has_data = false; 1427 1428 return bytes_recvd; 1429 } 1430 1431 spdk_pipe_writer_advance(sock->recv_pipe, bytes_recvd); 1432 1433 #if DEBUG 1434 if (sock->base.group_impl) { 1435 assert(sock->socket_has_data == true); 1436 } 1437 #endif 1438 1439 sock->pipe_has_data = true; 1440 if (bytes_recvd < bytes_avail) { 1441 /* We drained the kernel socket entirely. */ 1442 sock->socket_has_data = false; 1443 } 1444 1445 return bytes_recvd; 1446 } 1447 1448 static ssize_t 1449 posix_sock_readv(struct spdk_sock *_sock, struct iovec *iov, int iovcnt) 1450 { 1451 struct spdk_posix_sock *sock = __posix_sock(_sock); 1452 struct spdk_posix_sock_group_impl *group = __posix_group_impl(sock->base.group_impl); 1453 int rc, i; 1454 size_t len; 1455 1456 if (sock->recv_pipe == NULL) { 1457 assert(sock->pipe_has_data == false); 1458 if (group && sock->socket_has_data) { 1459 sock->socket_has_data = false; 1460 TAILQ_REMOVE(&group->socks_with_data, sock, link); 1461 } 1462 if (sock->ssl) { 1463 return SSL_readv(sock->ssl, iov, iovcnt); 1464 } else { 1465 return readv(sock->fd, iov, iovcnt); 1466 } 1467 } 1468 1469 /* If the socket is not in a group, we must assume it always has 1470 * data waiting for us because it is not epolled */ 1471 if (!sock->pipe_has_data && (group == NULL || sock->socket_has_data)) { 1472 /* If the user is receiving a sufficiently large amount of data, 1473 * receive directly to their buffers. */ 1474 len = 0; 1475 for (i = 0; i < iovcnt; i++) { 1476 len += iov[i].iov_len; 1477 } 1478 1479 if (len >= MIN_SOCK_PIPE_SIZE) { 1480 /* TODO: Should this detect if kernel socket is drained? */ 1481 if (sock->ssl) { 1482 return SSL_readv(sock->ssl, iov, iovcnt); 1483 } else { 1484 return readv(sock->fd, iov, iovcnt); 1485 } 1486 } 1487 1488 /* Otherwise, do a big read into our pipe */ 1489 rc = posix_sock_read(sock); 1490 if (rc <= 0) { 1491 return rc; 1492 } 1493 } 1494 1495 return posix_sock_recv_from_pipe(sock, iov, iovcnt); 1496 } 1497 1498 static ssize_t 1499 posix_sock_recv(struct spdk_sock *sock, void *buf, size_t len) 1500 { 1501 struct iovec iov[1]; 1502 1503 iov[0].iov_base = buf; 1504 iov[0].iov_len = len; 1505 1506 return posix_sock_readv(sock, iov, 1); 1507 } 1508 1509 static void 1510 posix_sock_readv_async(struct spdk_sock *sock, struct spdk_sock_request *req) 1511 { 1512 req->cb_fn(req->cb_arg, -ENOTSUP); 1513 } 1514 1515 static ssize_t 1516 posix_sock_writev(struct spdk_sock *_sock, struct iovec *iov, int iovcnt) 1517 { 1518 struct spdk_posix_sock *sock = __posix_sock(_sock); 1519 int rc; 1520 1521 /* In order to process a writev, we need to flush any asynchronous writes 1522 * first. */ 1523 rc = _sock_flush(_sock); 1524 if (rc < 0) { 1525 return rc; 1526 } 1527 1528 if (!TAILQ_EMPTY(&_sock->queued_reqs)) { 1529 /* We weren't able to flush all requests */ 1530 errno = EAGAIN; 1531 return -1; 1532 } 1533 1534 if (sock->ssl) { 1535 return SSL_writev(sock->ssl, iov, iovcnt); 1536 } else { 1537 return writev(sock->fd, iov, iovcnt); 1538 } 1539 } 1540 1541 static void 1542 posix_sock_writev_async(struct spdk_sock *sock, struct spdk_sock_request *req) 1543 { 1544 int rc; 1545 1546 spdk_sock_request_queue(sock, req); 1547 1548 /* If there are a sufficient number queued, just flush them out immediately. */ 1549 if (sock->queued_iovcnt >= IOV_BATCH_SIZE) { 1550 rc = _sock_flush(sock); 1551 if (rc < 0 && errno != EAGAIN) { 1552 spdk_sock_abort_requests(sock); 1553 } 1554 } 1555 } 1556 1557 static int 1558 posix_sock_set_recvlowat(struct spdk_sock *_sock, int nbytes) 1559 { 1560 struct spdk_posix_sock *sock = __posix_sock(_sock); 1561 int val; 1562 int rc; 1563 1564 assert(sock != NULL); 1565 1566 val = nbytes; 1567 rc = setsockopt(sock->fd, SOL_SOCKET, SO_RCVLOWAT, &val, sizeof val); 1568 if (rc != 0) { 1569 return -1; 1570 } 1571 return 0; 1572 } 1573 1574 static bool 1575 posix_sock_is_ipv6(struct spdk_sock *_sock) 1576 { 1577 struct spdk_posix_sock *sock = __posix_sock(_sock); 1578 struct sockaddr_storage sa; 1579 socklen_t salen; 1580 int rc; 1581 1582 assert(sock != NULL); 1583 1584 memset(&sa, 0, sizeof sa); 1585 salen = sizeof sa; 1586 rc = getsockname(sock->fd, (struct sockaddr *) &sa, &salen); 1587 if (rc != 0) { 1588 SPDK_ERRLOG("getsockname() failed (errno=%d)\n", errno); 1589 return false; 1590 } 1591 1592 return (sa.ss_family == AF_INET6); 1593 } 1594 1595 static bool 1596 posix_sock_is_ipv4(struct spdk_sock *_sock) 1597 { 1598 struct spdk_posix_sock *sock = __posix_sock(_sock); 1599 struct sockaddr_storage sa; 1600 socklen_t salen; 1601 int rc; 1602 1603 assert(sock != NULL); 1604 1605 memset(&sa, 0, sizeof sa); 1606 salen = sizeof sa; 1607 rc = getsockname(sock->fd, (struct sockaddr *) &sa, &salen); 1608 if (rc != 0) { 1609 SPDK_ERRLOG("getsockname() failed (errno=%d)\n", errno); 1610 return false; 1611 } 1612 1613 return (sa.ss_family == AF_INET); 1614 } 1615 1616 static bool 1617 posix_sock_is_connected(struct spdk_sock *_sock) 1618 { 1619 struct spdk_posix_sock *sock = __posix_sock(_sock); 1620 uint8_t byte; 1621 int rc; 1622 1623 rc = recv(sock->fd, &byte, 1, MSG_PEEK); 1624 if (rc == 0) { 1625 return false; 1626 } 1627 1628 if (rc < 0) { 1629 if (errno == EAGAIN || errno == EWOULDBLOCK) { 1630 return true; 1631 } 1632 1633 return false; 1634 } 1635 1636 return true; 1637 } 1638 1639 static struct spdk_sock_group_impl * 1640 posix_sock_group_impl_get_optimal(struct spdk_sock *_sock, struct spdk_sock_group_impl *hint) 1641 { 1642 struct spdk_posix_sock *sock = __posix_sock(_sock); 1643 struct spdk_sock_group_impl *group_impl; 1644 1645 if (sock->placement_id != -1) { 1646 spdk_sock_map_lookup(&g_map, sock->placement_id, &group_impl, hint); 1647 return group_impl; 1648 } 1649 1650 return NULL; 1651 } 1652 1653 static struct spdk_sock_group_impl * 1654 posix_sock_group_impl_create(void) 1655 { 1656 struct spdk_posix_sock_group_impl *group_impl; 1657 int fd; 1658 1659 #if defined(SPDK_EPOLL) 1660 fd = epoll_create1(0); 1661 #elif defined(SPDK_KEVENT) 1662 fd = kqueue(); 1663 #endif 1664 if (fd == -1) { 1665 return NULL; 1666 } 1667 1668 group_impl = calloc(1, sizeof(*group_impl)); 1669 if (group_impl == NULL) { 1670 SPDK_ERRLOG("group_impl allocation failed\n"); 1671 close(fd); 1672 return NULL; 1673 } 1674 1675 group_impl->fd = fd; 1676 TAILQ_INIT(&group_impl->socks_with_data); 1677 group_impl->placement_id = -1; 1678 1679 if (g_spdk_posix_sock_impl_opts.enable_placement_id == PLACEMENT_CPU) { 1680 spdk_sock_map_insert(&g_map, spdk_env_get_current_core(), &group_impl->base); 1681 group_impl->placement_id = spdk_env_get_current_core(); 1682 } 1683 1684 return &group_impl->base; 1685 } 1686 1687 static void 1688 posix_sock_mark(struct spdk_posix_sock_group_impl *group, struct spdk_posix_sock *sock, 1689 int placement_id) 1690 { 1691 #if defined(SO_MARK) 1692 int rc; 1693 1694 rc = setsockopt(sock->fd, SOL_SOCKET, SO_MARK, 1695 &placement_id, sizeof(placement_id)); 1696 if (rc != 0) { 1697 /* Not fatal */ 1698 SPDK_ERRLOG("Error setting SO_MARK\n"); 1699 return; 1700 } 1701 1702 rc = spdk_sock_map_insert(&g_map, placement_id, &group->base); 1703 if (rc != 0) { 1704 /* Not fatal */ 1705 SPDK_ERRLOG("Failed to insert sock group into map: %d\n", rc); 1706 return; 1707 } 1708 1709 sock->placement_id = placement_id; 1710 #endif 1711 } 1712 1713 static void 1714 posix_sock_update_mark(struct spdk_sock_group_impl *_group, struct spdk_sock *_sock) 1715 { 1716 struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group); 1717 1718 if (group->placement_id == -1) { 1719 group->placement_id = spdk_sock_map_find_free(&g_map); 1720 1721 /* If a free placement id is found, update existing sockets in this group */ 1722 if (group->placement_id != -1) { 1723 struct spdk_sock *sock, *tmp; 1724 1725 TAILQ_FOREACH_SAFE(sock, &_group->socks, link, tmp) { 1726 posix_sock_mark(group, __posix_sock(sock), group->placement_id); 1727 } 1728 } 1729 } 1730 1731 if (group->placement_id != -1) { 1732 /* 1733 * group placement id is already determined for this poll group. 1734 * Mark socket with group's placement id. 1735 */ 1736 posix_sock_mark(group, __posix_sock(_sock), group->placement_id); 1737 } 1738 } 1739 1740 static int 1741 posix_sock_group_impl_add_sock(struct spdk_sock_group_impl *_group, struct spdk_sock *_sock) 1742 { 1743 struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group); 1744 struct spdk_posix_sock *sock = __posix_sock(_sock); 1745 int rc; 1746 1747 #if defined(SPDK_EPOLL) 1748 struct epoll_event event; 1749 1750 memset(&event, 0, sizeof(event)); 1751 /* EPOLLERR is always on even if we don't set it, but be explicit for clarity */ 1752 event.events = EPOLLIN | EPOLLERR; 1753 event.data.ptr = sock; 1754 1755 rc = epoll_ctl(group->fd, EPOLL_CTL_ADD, sock->fd, &event); 1756 #elif defined(SPDK_KEVENT) 1757 struct kevent event; 1758 struct timespec ts = {0}; 1759 1760 EV_SET(&event, sock->fd, EVFILT_READ, EV_ADD, 0, 0, sock); 1761 1762 rc = kevent(group->fd, &event, 1, NULL, 0, &ts); 1763 #endif 1764 1765 if (rc != 0) { 1766 return rc; 1767 } 1768 1769 /* switched from another polling group due to scheduling */ 1770 if (spdk_unlikely(sock->recv_pipe != NULL && 1771 (spdk_pipe_reader_bytes_available(sock->recv_pipe) > 0))) { 1772 sock->pipe_has_data = true; 1773 sock->socket_has_data = false; 1774 TAILQ_INSERT_TAIL(&group->socks_with_data, sock, link); 1775 } 1776 1777 if (g_spdk_posix_sock_impl_opts.enable_placement_id == PLACEMENT_MARK) { 1778 posix_sock_update_mark(_group, _sock); 1779 } else if (sock->placement_id != -1) { 1780 rc = spdk_sock_map_insert(&g_map, sock->placement_id, &group->base); 1781 if (rc != 0) { 1782 SPDK_ERRLOG("Failed to insert sock group into map: %d\n", rc); 1783 /* Do not treat this as an error. The system will continue running. */ 1784 } 1785 } 1786 1787 return rc; 1788 } 1789 1790 static int 1791 posix_sock_group_impl_remove_sock(struct spdk_sock_group_impl *_group, struct spdk_sock *_sock) 1792 { 1793 struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group); 1794 struct spdk_posix_sock *sock = __posix_sock(_sock); 1795 int rc; 1796 1797 if (sock->pipe_has_data || sock->socket_has_data) { 1798 TAILQ_REMOVE(&group->socks_with_data, sock, link); 1799 sock->pipe_has_data = false; 1800 sock->socket_has_data = false; 1801 } 1802 1803 if (sock->placement_id != -1) { 1804 spdk_sock_map_release(&g_map, sock->placement_id); 1805 } 1806 1807 #if defined(SPDK_EPOLL) 1808 struct epoll_event event; 1809 1810 /* Event parameter is ignored but some old kernel version still require it. */ 1811 rc = epoll_ctl(group->fd, EPOLL_CTL_DEL, sock->fd, &event); 1812 #elif defined(SPDK_KEVENT) 1813 struct kevent event; 1814 struct timespec ts = {0}; 1815 1816 EV_SET(&event, sock->fd, EVFILT_READ, EV_DELETE, 0, 0, NULL); 1817 1818 rc = kevent(group->fd, &event, 1, NULL, 0, &ts); 1819 if (rc == 0 && event.flags & EV_ERROR) { 1820 rc = -1; 1821 errno = event.data; 1822 } 1823 #endif 1824 1825 spdk_sock_abort_requests(_sock); 1826 1827 return rc; 1828 } 1829 1830 static int 1831 posix_sock_group_impl_poll(struct spdk_sock_group_impl *_group, int max_events, 1832 struct spdk_sock **socks) 1833 { 1834 struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group); 1835 struct spdk_sock *sock, *tmp; 1836 int num_events, i, rc; 1837 struct spdk_posix_sock *psock, *ptmp; 1838 #if defined(SPDK_EPOLL) 1839 struct epoll_event events[MAX_EVENTS_PER_POLL]; 1840 #elif defined(SPDK_KEVENT) 1841 struct kevent events[MAX_EVENTS_PER_POLL]; 1842 struct timespec ts = {0}; 1843 #endif 1844 1845 #ifdef SPDK_ZEROCOPY 1846 /* When all of the following conditions are met 1847 * - non-blocking socket 1848 * - zero copy is enabled 1849 * - interrupts suppressed (i.e. busy polling) 1850 * - the NIC tx queue is full at the time sendmsg() is called 1851 * - epoll_wait determines there is an EPOLLIN event for the socket 1852 * then we can get into a situation where data we've sent is queued 1853 * up in the kernel network stack, but interrupts have been suppressed 1854 * because other traffic is flowing so the kernel misses the signal 1855 * to flush the software tx queue. If there wasn't incoming data 1856 * pending on the socket, then epoll_wait would have been sufficient 1857 * to kick off the send operation, but since there is a pending event 1858 * epoll_wait does not trigger the necessary operation. 1859 * 1860 * We deal with this by checking for all of the above conditions and 1861 * additionally looking for EPOLLIN events that were not consumed from 1862 * the last poll loop. We take this to mean that the upper layer is 1863 * unable to consume them because it is blocked waiting for resources 1864 * to free up, and those resources are most likely freed in response 1865 * to a pending asynchronous write completing. 1866 * 1867 * Additionally, sockets that have the same placement_id actually share 1868 * an underlying hardware queue. That means polling one of them is 1869 * equivalent to polling all of them. As a quick mechanism to avoid 1870 * making extra poll() calls, stash the last placement_id during the loop 1871 * and only poll if it's not the same. The overwhelmingly common case 1872 * is that all sockets in this list have the same placement_id because 1873 * SPDK is intentionally grouping sockets by that value, so even 1874 * though this won't stop all extra calls to poll(), it's very fast 1875 * and will catch all of them in practice. 1876 */ 1877 int last_placement_id = -1; 1878 1879 TAILQ_FOREACH(psock, &group->socks_with_data, link) { 1880 if (psock->zcopy && psock->placement_id >= 0 && 1881 psock->placement_id != last_placement_id) { 1882 struct pollfd pfd = {psock->fd, POLLIN | POLLERR, 0}; 1883 1884 poll(&pfd, 1, 0); 1885 last_placement_id = psock->placement_id; 1886 } 1887 } 1888 #endif 1889 1890 /* This must be a TAILQ_FOREACH_SAFE because while flushing, 1891 * a completion callback could remove the sock from the 1892 * group. */ 1893 TAILQ_FOREACH_SAFE(sock, &_group->socks, link, tmp) { 1894 rc = _sock_flush(sock); 1895 if (rc < 0 && errno != EAGAIN) { 1896 spdk_sock_abort_requests(sock); 1897 } 1898 } 1899 1900 assert(max_events > 0); 1901 1902 #if defined(SPDK_EPOLL) 1903 num_events = epoll_wait(group->fd, events, max_events, 0); 1904 #elif defined(SPDK_KEVENT) 1905 num_events = kevent(group->fd, NULL, 0, events, max_events, &ts); 1906 #endif 1907 1908 if (num_events == -1) { 1909 return -1; 1910 } else if (num_events == 0 && !TAILQ_EMPTY(&_group->socks)) { 1911 sock = TAILQ_FIRST(&_group->socks); 1912 psock = __posix_sock(sock); 1913 /* poll() is called here to busy poll the queue associated with 1914 * first socket in list and potentially reap incoming data. 1915 */ 1916 if (sock->opts.priority) { 1917 struct pollfd pfd = {0, 0, 0}; 1918 1919 pfd.fd = psock->fd; 1920 pfd.events = POLLIN | POLLERR; 1921 poll(&pfd, 1, 0); 1922 } 1923 } 1924 1925 for (i = 0; i < num_events; i++) { 1926 #if defined(SPDK_EPOLL) 1927 sock = events[i].data.ptr; 1928 psock = __posix_sock(sock); 1929 1930 #ifdef SPDK_ZEROCOPY 1931 if (events[i].events & EPOLLERR) { 1932 rc = _sock_check_zcopy(sock); 1933 /* If the socket was closed or removed from 1934 * the group in response to a send ack, don't 1935 * add it to the array here. */ 1936 if (rc || sock->cb_fn == NULL) { 1937 continue; 1938 } 1939 } 1940 #endif 1941 if ((events[i].events & EPOLLIN) == 0) { 1942 continue; 1943 } 1944 1945 #elif defined(SPDK_KEVENT) 1946 sock = events[i].udata; 1947 psock = __posix_sock(sock); 1948 #endif 1949 1950 /* If the socket is not already in the list, add it now */ 1951 if (!psock->socket_has_data && !psock->pipe_has_data) { 1952 TAILQ_INSERT_TAIL(&group->socks_with_data, psock, link); 1953 } 1954 psock->socket_has_data = true; 1955 } 1956 1957 num_events = 0; 1958 1959 TAILQ_FOREACH_SAFE(psock, &group->socks_with_data, link, ptmp) { 1960 if (num_events == max_events) { 1961 break; 1962 } 1963 1964 /* If the socket's cb_fn is NULL, just remove it from the 1965 * list and do not add it to socks array */ 1966 if (spdk_unlikely(psock->base.cb_fn == NULL)) { 1967 psock->socket_has_data = false; 1968 psock->pipe_has_data = false; 1969 TAILQ_REMOVE(&group->socks_with_data, psock, link); 1970 continue; 1971 } 1972 1973 socks[num_events++] = &psock->base; 1974 } 1975 1976 /* Cycle the has_data list so that each time we poll things aren't 1977 * in the same order. Say we have 6 sockets in the list, named as follows: 1978 * A B C D E F 1979 * And all 6 sockets had epoll events, but max_events is only 3. That means 1980 * psock currently points at D. We want to rearrange the list to the following: 1981 * D E F A B C 1982 * 1983 * The variables below are named according to this example to make it easier to 1984 * follow the swaps. 1985 */ 1986 if (psock != NULL) { 1987 struct spdk_posix_sock *pa, *pc, *pd, *pf; 1988 1989 /* Capture pointers to the elements we need */ 1990 pd = psock; 1991 pc = TAILQ_PREV(pd, spdk_has_data_list, link); 1992 pa = TAILQ_FIRST(&group->socks_with_data); 1993 pf = TAILQ_LAST(&group->socks_with_data, spdk_has_data_list); 1994 1995 /* Break the link between C and D */ 1996 pc->link.tqe_next = NULL; 1997 1998 /* Connect F to A */ 1999 pf->link.tqe_next = pa; 2000 pa->link.tqe_prev = &pf->link.tqe_next; 2001 2002 /* Fix up the list first/last pointers */ 2003 group->socks_with_data.tqh_first = pd; 2004 group->socks_with_data.tqh_last = &pc->link.tqe_next; 2005 2006 /* D is in front of the list, make tqe prev pointer point to the head of list */ 2007 pd->link.tqe_prev = &group->socks_with_data.tqh_first; 2008 } 2009 2010 return num_events; 2011 } 2012 2013 static int 2014 posix_sock_group_impl_close(struct spdk_sock_group_impl *_group) 2015 { 2016 struct spdk_posix_sock_group_impl *group = __posix_group_impl(_group); 2017 int rc; 2018 2019 if (g_spdk_posix_sock_impl_opts.enable_placement_id == PLACEMENT_CPU) { 2020 spdk_sock_map_release(&g_map, spdk_env_get_current_core()); 2021 } 2022 2023 rc = close(group->fd); 2024 free(group); 2025 return rc; 2026 } 2027 2028 static struct spdk_net_impl g_posix_net_impl = { 2029 .name = "posix", 2030 .getaddr = posix_sock_getaddr, 2031 .connect = posix_sock_connect, 2032 .listen = posix_sock_listen, 2033 .accept = posix_sock_accept, 2034 .close = posix_sock_close, 2035 .recv = posix_sock_recv, 2036 .readv = posix_sock_readv, 2037 .readv_async = posix_sock_readv_async, 2038 .writev = posix_sock_writev, 2039 .writev_async = posix_sock_writev_async, 2040 .flush = posix_sock_flush, 2041 .set_recvlowat = posix_sock_set_recvlowat, 2042 .set_recvbuf = posix_sock_set_recvbuf, 2043 .set_sendbuf = posix_sock_set_sendbuf, 2044 .is_ipv6 = posix_sock_is_ipv6, 2045 .is_ipv4 = posix_sock_is_ipv4, 2046 .is_connected = posix_sock_is_connected, 2047 .group_impl_get_optimal = posix_sock_group_impl_get_optimal, 2048 .group_impl_create = posix_sock_group_impl_create, 2049 .group_impl_add_sock = posix_sock_group_impl_add_sock, 2050 .group_impl_remove_sock = posix_sock_group_impl_remove_sock, 2051 .group_impl_poll = posix_sock_group_impl_poll, 2052 .group_impl_close = posix_sock_group_impl_close, 2053 .get_opts = posix_sock_impl_get_opts, 2054 .set_opts = posix_sock_impl_set_opts, 2055 }; 2056 2057 SPDK_NET_IMPL_REGISTER(posix, &g_posix_net_impl, DEFAULT_SOCK_PRIORITY + 1); 2058 2059 static struct spdk_sock * 2060 ssl_sock_listen(const char *ip, int port, struct spdk_sock_opts *opts) 2061 { 2062 return posix_sock_create(ip, port, SPDK_SOCK_CREATE_LISTEN, opts, true); 2063 } 2064 2065 static struct spdk_sock * 2066 ssl_sock_connect(const char *ip, int port, struct spdk_sock_opts *opts) 2067 { 2068 return posix_sock_create(ip, port, SPDK_SOCK_CREATE_CONNECT, opts, true); 2069 } 2070 2071 static struct spdk_sock * 2072 ssl_sock_accept(struct spdk_sock *_sock) 2073 { 2074 return _posix_sock_accept(_sock, true); 2075 } 2076 2077 static struct spdk_net_impl g_ssl_net_impl = { 2078 .name = "ssl", 2079 .getaddr = posix_sock_getaddr, 2080 .connect = ssl_sock_connect, 2081 .listen = ssl_sock_listen, 2082 .accept = ssl_sock_accept, 2083 .close = posix_sock_close, 2084 .recv = posix_sock_recv, 2085 .readv = posix_sock_readv, 2086 .writev = posix_sock_writev, 2087 .writev_async = posix_sock_writev_async, 2088 .flush = posix_sock_flush, 2089 .set_recvlowat = posix_sock_set_recvlowat, 2090 .set_recvbuf = posix_sock_set_recvbuf, 2091 .set_sendbuf = posix_sock_set_sendbuf, 2092 .is_ipv6 = posix_sock_is_ipv6, 2093 .is_ipv4 = posix_sock_is_ipv4, 2094 .is_connected = posix_sock_is_connected, 2095 .group_impl_get_optimal = posix_sock_group_impl_get_optimal, 2096 .group_impl_create = posix_sock_group_impl_create, 2097 .group_impl_add_sock = posix_sock_group_impl_add_sock, 2098 .group_impl_remove_sock = posix_sock_group_impl_remove_sock, 2099 .group_impl_poll = posix_sock_group_impl_poll, 2100 .group_impl_close = posix_sock_group_impl_close, 2101 .get_opts = posix_sock_impl_get_opts, 2102 .set_opts = posix_sock_impl_set_opts, 2103 }; 2104 2105 SPDK_NET_IMPL_REGISTER(ssl, &g_ssl_net_impl, DEFAULT_SOCK_PRIORITY); 2106 SPDK_LOG_REGISTER_COMPONENT(sock_posix) 2107