1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2018 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/thread.h" 8 #include "spdk/env.h" 9 #include "spdk/event.h" 10 #include "spdk/log.h" 11 #include "spdk/string.h" 12 13 #include "spdk/sock.h" 14 #include "spdk/hexlify.h" 15 #include "spdk/nvmf.h" 16 17 #define ACCEPT_TIMEOUT_US 1000 18 #define CLOSE_TIMEOUT_US 1000000 19 #define BUFFER_SIZE 1024 20 #define ADDR_STR_LEN INET6_ADDRSTRLEN 21 22 static bool g_is_running; 23 24 static char *g_host; 25 static char *g_sock_impl_name; 26 static int g_port; 27 static bool g_is_server; 28 static int g_zcopy; 29 static int g_ktls; 30 static int g_tls_version; 31 static bool g_verbose; 32 static uint8_t g_psk_key[SPDK_TLS_PSK_MAX_LEN]; 33 static uint32_t g_psk_key_size; 34 static char *g_psk_identity; 35 36 /* 37 * We'll use this struct to gather housekeeping hello_context to pass between 38 * our events and callbacks. 39 */ 40 struct hello_context_t { 41 bool is_server; 42 char *host; 43 const char *sock_impl_name; 44 int port; 45 int zcopy; 46 int ktls; 47 int tls_version; 48 uint8_t *psk_key; 49 uint32_t psk_key_size; 50 char *psk_identity; 51 52 bool verbose; 53 int bytes_in; 54 int bytes_out; 55 56 struct spdk_sock *sock; 57 58 struct spdk_sock_group *group; 59 void *buf; 60 struct spdk_poller *poller_in; 61 struct spdk_poller *poller_out; 62 struct spdk_poller *time_out; 63 64 int rc; 65 ssize_t n; 66 }; 67 68 /* 69 * Usage function for printing parameters that are specific to this application 70 */ 71 static void 72 hello_sock_usage(void) 73 { 74 printf(" -E psk_key Default PSK KEY in hexadecimal digits, e.g. 1234567890ABCDEF (only applies when sock_impl == ssl)\n"); 75 printf(" -H host_addr host address\n"); 76 printf(" -I psk_id Default PSK ID, e.g. psk.spdk.io (only applies when sock_impl == ssl)\n"); 77 printf(" -P port port number\n"); 78 printf(" -N sock_impl socket implementation, e.g., -N posix or -N uring\n"); 79 printf(" -S start in server mode\n"); 80 printf(" -T tls_ver TLS version, e.g., -T 12 or -T 13. If omitted, auto-negotiation will take place\n"); 81 printf(" -k disable KTLS for the given sock implementation (default)\n"); 82 printf(" -K enable KTLS for the given sock implementation\n"); 83 printf(" -V print out additional information\n"); 84 printf(" -z disable zero copy send for the given sock implementation\n"); 85 printf(" -Z enable zero copy send for the given sock implementation\n"); 86 } 87 88 /* 89 * This function is called to parse the parameters that are specific to this application 90 */ 91 static int 92 hello_sock_parse_arg(int ch, char *arg) 93 { 94 char *unhexlified; 95 96 switch (ch) { 97 case 'E': 98 g_psk_key_size = strlen(arg) / 2; 99 if (g_psk_key_size > SPDK_TLS_PSK_MAX_LEN) { 100 fprintf(stderr, "Invalid PSK: too long (%"PRIu32")\n", g_psk_key_size); 101 return -EINVAL; 102 } 103 unhexlified = spdk_unhexlify(arg); 104 if (unhexlified == NULL) { 105 fprintf(stderr, "Invalid PSK: not in a hex format\n"); 106 return -EINVAL; 107 } 108 memcpy(g_psk_key, unhexlified, g_psk_key_size); 109 free(unhexlified); 110 break; 111 case 'H': 112 g_host = arg; 113 break; 114 case 'I': 115 g_psk_identity = arg; 116 break; 117 case 'N': 118 g_sock_impl_name = arg; 119 break; 120 case 'P': 121 g_port = spdk_strtol(arg, 10); 122 if (g_port < 0) { 123 fprintf(stderr, "Invalid port ID\n"); 124 return g_port; 125 } 126 break; 127 case 'S': 128 g_is_server = 1; 129 break; 130 case 'K': 131 g_ktls = 1; 132 break; 133 case 'k': 134 g_ktls = 0; 135 break; 136 case 'T': 137 g_tls_version = spdk_strtol(arg, 10); 138 if (g_tls_version < 0) { 139 fprintf(stderr, "Invalid TLS version\n"); 140 return g_tls_version; 141 } 142 break; 143 case 'V': 144 g_verbose = true; 145 break; 146 case 'Z': 147 g_zcopy = 1; 148 break; 149 case 'z': 150 g_zcopy = 0; 151 break; 152 default: 153 return -EINVAL; 154 } 155 return 0; 156 } 157 158 static int 159 hello_sock_close_timeout_poll(void *arg) 160 { 161 struct hello_context_t *ctx = arg; 162 SPDK_NOTICELOG("Connection closed\n"); 163 164 spdk_poller_unregister(&ctx->time_out); 165 spdk_poller_unregister(&ctx->poller_in); 166 spdk_sock_close(&ctx->sock); 167 spdk_sock_group_close(&ctx->group); 168 169 spdk_app_stop(ctx->rc); 170 return SPDK_POLLER_BUSY; 171 } 172 173 static int 174 hello_sock_quit(struct hello_context_t *ctx, int rc) 175 { 176 ctx->rc = rc; 177 spdk_poller_unregister(&ctx->poller_out); 178 if (!ctx->time_out) { 179 ctx->time_out = SPDK_POLLER_REGISTER(hello_sock_close_timeout_poll, ctx, 180 CLOSE_TIMEOUT_US); 181 } 182 return 0; 183 } 184 185 static int 186 hello_sock_recv_poll(void *arg) 187 { 188 struct hello_context_t *ctx = arg; 189 int rc; 190 char buf_in[BUFFER_SIZE]; 191 192 /* 193 * Get response 194 */ 195 rc = spdk_sock_recv(ctx->sock, buf_in, sizeof(buf_in) - 1); 196 197 if (rc <= 0) { 198 if (errno == EAGAIN || errno == EWOULDBLOCK) { 199 return SPDK_POLLER_IDLE; 200 } 201 202 /* This poller drains recv buffer until hello_sock_close_timeout_poll() 203 * runs out. Which starts when hello_sock_quit() is called. 204 * Quit the application just once, then patiently await the unregister. */ 205 if (ctx->rc == 0) { 206 hello_sock_quit(ctx, -1); 207 } 208 SPDK_ERRLOG_RATELIMIT("spdk_sock_recv() failed, errno %d: %s\n", 209 errno, spdk_strerror(errno)); 210 return SPDK_POLLER_BUSY; 211 } 212 213 if (rc > 0) { 214 ctx->bytes_in += rc; 215 buf_in[rc] = '\0'; 216 printf("%s", buf_in); 217 } 218 219 return SPDK_POLLER_BUSY; 220 } 221 222 static int 223 hello_sock_writev_poll(void *arg) 224 { 225 struct hello_context_t *ctx = arg; 226 int rc = 0; 227 struct iovec iov; 228 ssize_t n; 229 230 /* If previously we could not send any bytes, we should try again with the same buffer. */ 231 if (ctx->n != 0) { 232 iov.iov_base = ctx->buf; 233 iov.iov_len = ctx->n; 234 errno = 0; 235 rc = spdk_sock_writev(ctx->sock, &iov, 1); 236 if (rc < 0) { 237 if (errno == EAGAIN) { 238 return SPDK_POLLER_BUSY; 239 } 240 SPDK_ERRLOG("Write to socket failed. Closing connection...\n"); 241 hello_sock_quit(ctx, -1); 242 return SPDK_POLLER_IDLE; 243 } 244 ctx->bytes_out += rc; 245 ctx->n = 0; 246 } 247 248 n = read(STDIN_FILENO, ctx->buf, BUFFER_SIZE); 249 if (n == 0 || !g_is_running) { 250 /* EOF */ 251 SPDK_NOTICELOG("Closing connection...\n"); 252 hello_sock_quit(ctx, 0); 253 return SPDK_POLLER_IDLE; 254 } 255 if (n > 0) { 256 /* 257 * Send message to the server 258 */ 259 iov.iov_base = ctx->buf; 260 iov.iov_len = n; 261 errno = 0; 262 rc = spdk_sock_writev(ctx->sock, &iov, 1); 263 if (rc < 0) { 264 if (errno == EAGAIN) { 265 ctx->n = n; 266 } else { 267 SPDK_ERRLOG("Write to socket failed. Closing connection...\n"); 268 hello_sock_quit(ctx, -1); 269 return SPDK_POLLER_IDLE; 270 } 271 } 272 if (rc > 0) { 273 ctx->bytes_out += rc; 274 } 275 } 276 277 return rc > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 278 } 279 280 static int 281 hello_sock_connect(struct hello_context_t *ctx) 282 { 283 int rc; 284 char saddr[ADDR_STR_LEN], caddr[ADDR_STR_LEN]; 285 uint16_t cport, sport; 286 struct spdk_sock_impl_opts impl_opts; 287 size_t impl_opts_size = sizeof(impl_opts); 288 struct spdk_sock_opts opts; 289 290 spdk_sock_impl_get_opts(ctx->sock_impl_name, &impl_opts, &impl_opts_size); 291 impl_opts.enable_ktls = ctx->ktls; 292 impl_opts.tls_version = ctx->tls_version; 293 impl_opts.psk_identity = ctx->psk_identity; 294 impl_opts.tls_cipher_suites = "TLS_AES_128_GCM_SHA256"; 295 impl_opts.psk_key = ctx->psk_key; 296 impl_opts.psk_key_size = ctx->psk_key_size; 297 298 opts.opts_size = sizeof(opts); 299 spdk_sock_get_default_opts(&opts); 300 opts.zcopy = ctx->zcopy; 301 opts.impl_opts = &impl_opts; 302 opts.impl_opts_size = sizeof(impl_opts); 303 304 SPDK_NOTICELOG("Connecting to the server on %s:%d with sock_impl(%s)\n", ctx->host, ctx->port, 305 ctx->sock_impl_name); 306 307 ctx->sock = spdk_sock_connect_ext(ctx->host, ctx->port, ctx->sock_impl_name, &opts); 308 if (ctx->sock == NULL) { 309 SPDK_ERRLOG("connect error(%d): %s\n", errno, spdk_strerror(errno)); 310 return -1; 311 } 312 313 rc = spdk_sock_getaddr(ctx->sock, saddr, sizeof(saddr), &sport, caddr, sizeof(caddr), &cport); 314 if (rc < 0) { 315 SPDK_ERRLOG("Cannot get connection addresses\n"); 316 goto err; 317 } 318 319 SPDK_NOTICELOG("Connection accepted from (%s, %hu) to (%s, %hu)\n", caddr, cport, saddr, sport); 320 321 rc = fcntl(STDIN_FILENO, F_GETFL); 322 if (rc == -1) { 323 SPDK_ERRLOG("Getting file status flag failed: %s\n", strerror(errno)); 324 goto err; 325 } 326 327 if (fcntl(STDIN_FILENO, F_SETFL, rc | O_NONBLOCK) == -1) { 328 SPDK_ERRLOG("Setting file status flag failed: %s\n", strerror(errno)); 329 goto err; 330 } 331 332 g_is_running = true; 333 ctx->poller_in = SPDK_POLLER_REGISTER(hello_sock_recv_poll, ctx, 0); 334 ctx->poller_out = SPDK_POLLER_REGISTER(hello_sock_writev_poll, ctx, 0); 335 336 return 0; 337 err: 338 spdk_sock_close(&ctx->sock); 339 return -1; 340 } 341 342 static void 343 hello_sock_cb(void *arg, struct spdk_sock_group *group, struct spdk_sock *sock) 344 { 345 int rc; 346 struct hello_context_t *ctx = arg; 347 struct iovec iov = {}; 348 ssize_t n; 349 void *user_ctx; 350 351 rc = spdk_sock_recv_next(sock, &iov.iov_base, &user_ctx); 352 if (rc < 0) { 353 if (errno == EAGAIN || errno == EWOULDBLOCK) { 354 return; 355 } 356 357 if (errno != ENOTCONN && errno != ECONNRESET) { 358 SPDK_ERRLOG("spdk_sock_recv_zcopy() failed, errno %d: %s\n", 359 errno, spdk_strerror(errno)); 360 } 361 } 362 363 if (rc > 0) { 364 iov.iov_len = rc; 365 ctx->bytes_in += iov.iov_len; 366 n = spdk_sock_writev(sock, &iov, 1); 367 if (n > 0) { 368 assert(n == rc); 369 ctx->bytes_out += n; 370 } 371 372 spdk_sock_group_provide_buf(ctx->group, iov.iov_base, BUFFER_SIZE, NULL); 373 return; 374 } 375 376 /* Connection closed */ 377 SPDK_NOTICELOG("Connection closed\n"); 378 spdk_sock_group_remove_sock(group, sock); 379 spdk_sock_close(&sock); 380 } 381 382 static int 383 hello_sock_accept_poll(void *arg) 384 { 385 struct hello_context_t *ctx = arg; 386 struct spdk_sock *sock; 387 int rc; 388 int count = 0; 389 char saddr[ADDR_STR_LEN], caddr[ADDR_STR_LEN]; 390 uint16_t cport, sport; 391 392 if (!g_is_running) { 393 hello_sock_quit(ctx, 0); 394 return SPDK_POLLER_IDLE; 395 } 396 397 while (1) { 398 sock = spdk_sock_accept(ctx->sock); 399 if (sock != NULL) { 400 rc = spdk_sock_getaddr(sock, saddr, sizeof(saddr), &sport, caddr, sizeof(caddr), &cport); 401 if (rc < 0) { 402 SPDK_ERRLOG("Cannot get connection addresses\n"); 403 spdk_sock_close(&sock); 404 return SPDK_POLLER_IDLE; 405 } 406 407 SPDK_NOTICELOG("Accepting a new connection from (%s, %hu) to (%s, %hu)\n", 408 caddr, cport, saddr, sport); 409 410 rc = spdk_sock_group_add_sock(ctx->group, sock, 411 hello_sock_cb, ctx); 412 413 if (rc < 0) { 414 spdk_sock_close(&sock); 415 SPDK_ERRLOG("failed\n"); 416 break; 417 } 418 419 count++; 420 } else { 421 if (errno != EAGAIN && errno != EWOULDBLOCK) { 422 SPDK_ERRLOG("accept error(%d): %s\n", errno, spdk_strerror(errno)); 423 } 424 break; 425 } 426 } 427 428 return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 429 } 430 431 static int 432 hello_sock_group_poll(void *arg) 433 { 434 struct hello_context_t *ctx = arg; 435 int rc; 436 437 rc = spdk_sock_group_poll(ctx->group); 438 if (rc < 0) { 439 SPDK_ERRLOG("Failed to poll sock_group=%p\n", ctx->group); 440 } 441 442 return rc > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 443 } 444 445 static int 446 hello_sock_listen(struct hello_context_t *ctx) 447 { 448 struct spdk_sock_impl_opts impl_opts; 449 size_t impl_opts_size = sizeof(impl_opts); 450 struct spdk_sock_opts opts; 451 452 spdk_sock_impl_get_opts(ctx->sock_impl_name, &impl_opts, &impl_opts_size); 453 impl_opts.enable_ktls = ctx->ktls; 454 impl_opts.tls_version = ctx->tls_version; 455 impl_opts.psk_identity = ctx->psk_identity; 456 impl_opts.tls_cipher_suites = "TLS_AES_128_GCM_SHA256"; 457 impl_opts.psk_key = ctx->psk_key; 458 impl_opts.psk_key_size = ctx->psk_key_size; 459 460 opts.opts_size = sizeof(opts); 461 spdk_sock_get_default_opts(&opts); 462 opts.zcopy = ctx->zcopy; 463 opts.impl_opts = &impl_opts; 464 opts.impl_opts_size = sizeof(impl_opts); 465 466 ctx->sock = spdk_sock_listen_ext(ctx->host, ctx->port, ctx->sock_impl_name, &opts); 467 if (ctx->sock == NULL) { 468 SPDK_ERRLOG("Cannot create server socket\n"); 469 return -1; 470 } 471 472 SPDK_NOTICELOG("Listening connection on %s:%d with sock_impl(%s)\n", ctx->host, ctx->port, 473 ctx->sock_impl_name); 474 475 /* 476 * Create sock group for server socket 477 */ 478 ctx->group = spdk_sock_group_create(NULL); 479 if (ctx->group == NULL) { 480 SPDK_ERRLOG("Cannot create sock group\n"); 481 spdk_sock_close(&ctx->sock); 482 return -1; 483 } 484 485 spdk_sock_group_provide_buf(ctx->group, ctx->buf, BUFFER_SIZE, NULL); 486 487 g_is_running = true; 488 489 /* 490 * Start acceptor and group poller 491 */ 492 ctx->poller_in = SPDK_POLLER_REGISTER(hello_sock_accept_poll, ctx, 493 ACCEPT_TIMEOUT_US); 494 ctx->poller_out = SPDK_POLLER_REGISTER(hello_sock_group_poll, ctx, 0); 495 496 return 0; 497 } 498 499 static void 500 hello_sock_shutdown_cb(void) 501 { 502 g_is_running = false; 503 } 504 505 /* 506 * Our initial event that kicks off everything from main(). 507 */ 508 static void 509 hello_start(void *arg1) 510 { 511 struct hello_context_t *ctx = arg1; 512 int rc; 513 514 SPDK_NOTICELOG("Successfully started the application\n"); 515 516 if (ctx->is_server) { 517 rc = hello_sock_listen(ctx); 518 } else { 519 rc = hello_sock_connect(ctx); 520 } 521 522 if (rc) { 523 spdk_app_stop(-1); 524 return; 525 } 526 } 527 528 int 529 main(int argc, char **argv) 530 { 531 struct spdk_app_opts opts = {}; 532 int rc = 0; 533 struct hello_context_t hello_context = {}; 534 535 /* Set default values in opts structure. */ 536 spdk_app_opts_init(&opts, sizeof(opts)); 537 opts.name = "hello_sock"; 538 opts.shutdown_cb = hello_sock_shutdown_cb; 539 opts.rpc_addr = NULL; 540 541 if ((rc = spdk_app_parse_args(argc, argv, &opts, "E:H:I:kKN:P:ST:VzZ", NULL, hello_sock_parse_arg, 542 hello_sock_usage)) != SPDK_APP_PARSE_ARGS_SUCCESS) { 543 exit(rc); 544 } 545 hello_context.is_server = g_is_server; 546 hello_context.host = g_host; 547 hello_context.sock_impl_name = g_sock_impl_name; 548 hello_context.port = g_port; 549 hello_context.zcopy = g_zcopy; 550 hello_context.ktls = g_ktls; 551 hello_context.tls_version = g_tls_version; 552 hello_context.psk_key = g_psk_key; 553 hello_context.psk_key_size = g_psk_key_size; 554 hello_context.psk_identity = g_psk_identity; 555 hello_context.verbose = g_verbose; 556 557 if (hello_context.sock_impl_name == NULL) { 558 hello_context.sock_impl_name = spdk_sock_get_default_impl(); 559 560 if (hello_context.sock_impl_name == NULL) { 561 SPDK_ERRLOG("No sock implementations available!\n"); 562 exit(-1); 563 } 564 } 565 566 hello_context.buf = calloc(1, BUFFER_SIZE); 567 if (hello_context.buf == NULL) { 568 SPDK_ERRLOG("Cannot allocate memory for hello_context buffer\n"); 569 exit(-1); 570 } 571 hello_context.n = 0; 572 573 if (hello_context.is_server) { 574 struct spdk_sock_impl_opts impl_opts = {}; 575 size_t len = sizeof(impl_opts); 576 577 rc = spdk_sock_impl_get_opts(hello_context.sock_impl_name, &impl_opts, &len); 578 if (rc < 0) { 579 free(hello_context.buf); 580 exit(rc); 581 } 582 583 /* Our applications will post buffers to be used for receiving. That feature 584 * is mutually exclusive with the recv pipe, so we need to disable it. */ 585 impl_opts.enable_recv_pipe = false; 586 spdk_sock_impl_set_opts(hello_context.sock_impl_name, &impl_opts, len); 587 } 588 589 rc = spdk_app_start(&opts, hello_start, &hello_context); 590 if (rc) { 591 SPDK_ERRLOG("ERROR starting application\n"); 592 } 593 594 SPDK_NOTICELOG("Exiting from application\n"); 595 596 if (hello_context.verbose) { 597 printf("** %d bytes received, %d bytes sent **\n", 598 hello_context.bytes_in, hello_context.bytes_out); 599 } 600 601 /* Gracefully close out all of the SPDK subsystems. */ 602 spdk_app_fini(); 603 free(hello_context.buf); 604 return rc; 605 } 606