1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2018 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/thread.h" 8 #include "spdk/env.h" 9 #include "spdk/event.h" 10 #include "spdk/log.h" 11 #include "spdk/string.h" 12 13 #include "spdk/sock.h" 14 #include "spdk/hexlify.h" 15 #include "spdk/nvmf.h" 16 17 #define ACCEPT_TIMEOUT_US 1000 18 #define CLOSE_TIMEOUT_US 1000000 19 #define BUFFER_SIZE 1024 20 #define ADDR_STR_LEN INET6_ADDRSTRLEN 21 22 static bool g_is_running; 23 24 static char *g_host; 25 static char *g_sock_impl_name; 26 static int g_port; 27 static bool g_is_server; 28 static int g_zcopy; 29 static int g_ktls; 30 static int g_tls_version; 31 static bool g_verbose; 32 static uint8_t g_psk_key[SPDK_TLS_PSK_MAX_LEN]; 33 static uint32_t g_psk_key_size; 34 static char *g_psk_identity; 35 36 /* 37 * We'll use this struct to gather housekeeping hello_context to pass between 38 * our events and callbacks. 39 */ 40 struct hello_context_t { 41 bool is_server; 42 char *host; 43 const char *sock_impl_name; 44 int port; 45 int zcopy; 46 int ktls; 47 int tls_version; 48 uint8_t *psk_key; 49 uint32_t psk_key_size; 50 char *psk_identity; 51 52 bool verbose; 53 int bytes_in; 54 int bytes_out; 55 56 struct spdk_sock *sock; 57 58 struct spdk_sock_group *group; 59 void *buf; 60 struct spdk_poller *poller_in; 61 struct spdk_poller *poller_out; 62 struct spdk_poller *time_out; 63 64 int rc; 65 ssize_t n; 66 }; 67 68 /* 69 * Usage function for printing parameters that are specific to this application 70 */ 71 static void 72 hello_sock_usage(void) 73 { 74 printf(" -E psk_key Default PSK KEY in hexadecimal digits, e.g. 1234567890ABCDEF (only applies when sock_impl == ssl)\n"); 75 printf(" -H host_addr host address\n"); 76 printf(" -I psk_id Default PSK ID, e.g. psk.spdk.io (only applies when sock_impl == ssl)\n"); 77 printf(" -P port port number\n"); 78 printf(" -N sock_impl socket implementation, e.g., -N posix or -N uring\n"); 79 printf(" -S start in server mode\n"); 80 printf(" -T tls_ver TLS version, e.g., -T 12 or -T 13. If omitted, auto-negotiation will take place\n"); 81 printf(" -k disable KTLS for the given sock implementation (default)\n"); 82 printf(" -K enable KTLS for the given sock implementation\n"); 83 printf(" -V print out additional information\n"); 84 printf(" -z disable zero copy send for the given sock implementation\n"); 85 printf(" -Z enable zero copy send for the given sock implementation\n"); 86 } 87 88 /* 89 * This function is called to parse the parameters that are specific to this application 90 */ 91 static int 92 hello_sock_parse_arg(int ch, char *arg) 93 { 94 char *unhexlified; 95 96 switch (ch) { 97 case 'E': 98 g_psk_key_size = strlen(arg) / 2; 99 if (g_psk_key_size > SPDK_TLS_PSK_MAX_LEN) { 100 fprintf(stderr, "Invalid PSK: too long (%"PRIu32")\n", g_psk_key_size); 101 return -EINVAL; 102 } 103 unhexlified = spdk_unhexlify(arg); 104 if (unhexlified == NULL) { 105 fprintf(stderr, "Invalid PSK: not in a hex format\n"); 106 return -EINVAL; 107 } 108 memcpy(g_psk_key, unhexlified, g_psk_key_size); 109 free(unhexlified); 110 break; 111 case 'H': 112 g_host = arg; 113 break; 114 case 'I': 115 g_psk_identity = arg; 116 break; 117 case 'N': 118 g_sock_impl_name = arg; 119 break; 120 case 'P': 121 g_port = spdk_strtol(arg, 10); 122 if (g_port < 0) { 123 fprintf(stderr, "Invalid port ID\n"); 124 return g_port; 125 } 126 break; 127 case 'S': 128 g_is_server = 1; 129 break; 130 case 'K': 131 g_ktls = 1; 132 break; 133 case 'k': 134 g_ktls = 0; 135 break; 136 case 'T': 137 g_tls_version = spdk_strtol(arg, 10); 138 if (g_tls_version < 0) { 139 fprintf(stderr, "Invalid TLS version\n"); 140 return g_tls_version; 141 } 142 break; 143 case 'V': 144 g_verbose = true; 145 break; 146 case 'Z': 147 g_zcopy = 1; 148 break; 149 case 'z': 150 g_zcopy = 0; 151 break; 152 default: 153 return -EINVAL; 154 } 155 return 0; 156 } 157 158 static int 159 hello_sock_close_timeout_poll(void *arg) 160 { 161 struct hello_context_t *ctx = arg; 162 SPDK_NOTICELOG("Connection closed\n"); 163 164 free(ctx->buf); 165 166 spdk_poller_unregister(&ctx->time_out); 167 spdk_poller_unregister(&ctx->poller_in); 168 spdk_sock_close(&ctx->sock); 169 spdk_sock_group_close(&ctx->group); 170 171 spdk_app_stop(ctx->rc); 172 return SPDK_POLLER_BUSY; 173 } 174 175 static int 176 hello_sock_quit(struct hello_context_t *ctx, int rc) 177 { 178 ctx->rc = rc; 179 spdk_poller_unregister(&ctx->poller_out); 180 if (!ctx->time_out) { 181 ctx->time_out = SPDK_POLLER_REGISTER(hello_sock_close_timeout_poll, ctx, 182 CLOSE_TIMEOUT_US); 183 } 184 return 0; 185 } 186 187 static int 188 hello_sock_recv_poll(void *arg) 189 { 190 struct hello_context_t *ctx = arg; 191 int rc; 192 char buf_in[BUFFER_SIZE]; 193 194 /* 195 * Get response 196 */ 197 rc = spdk_sock_recv(ctx->sock, buf_in, sizeof(buf_in) - 1); 198 199 if (rc <= 0) { 200 if (errno == EAGAIN || errno == EWOULDBLOCK) { 201 return SPDK_POLLER_IDLE; 202 } 203 204 hello_sock_quit(ctx, -1); 205 SPDK_ERRLOG("spdk_sock_recv() failed, errno %d: %s\n", 206 errno, spdk_strerror(errno)); 207 return SPDK_POLLER_BUSY; 208 } 209 210 if (rc > 0) { 211 ctx->bytes_in += rc; 212 buf_in[rc] = '\0'; 213 printf("%s", buf_in); 214 } 215 216 return SPDK_POLLER_BUSY; 217 } 218 219 static int 220 hello_sock_writev_poll(void *arg) 221 { 222 struct hello_context_t *ctx = arg; 223 int rc = 0; 224 struct iovec iov; 225 ssize_t n; 226 227 /* If previously we could not send any bytes, we should try again with the same buffer. */ 228 if (ctx->n != 0) { 229 iov.iov_base = ctx->buf; 230 iov.iov_len = ctx->n; 231 errno = 0; 232 rc = spdk_sock_writev(ctx->sock, &iov, 1); 233 if (rc < 0) { 234 if (errno == EAGAIN) { 235 return SPDK_POLLER_BUSY; 236 } 237 SPDK_ERRLOG("Write to socket failed. Closing connection...\n"); 238 hello_sock_quit(ctx, -1); 239 return SPDK_POLLER_IDLE; 240 } 241 ctx->bytes_out += rc; 242 ctx->n = 0; 243 } 244 245 n = read(STDIN_FILENO, ctx->buf, BUFFER_SIZE); 246 if (n == 0 || !g_is_running) { 247 /* EOF */ 248 SPDK_NOTICELOG("Closing connection...\n"); 249 hello_sock_quit(ctx, 0); 250 return SPDK_POLLER_IDLE; 251 } 252 if (n > 0) { 253 /* 254 * Send message to the server 255 */ 256 iov.iov_base = ctx->buf; 257 iov.iov_len = n; 258 errno = 0; 259 rc = spdk_sock_writev(ctx->sock, &iov, 1); 260 if (rc < 0) { 261 if (errno == EAGAIN) { 262 ctx->n = n; 263 } else { 264 SPDK_ERRLOG("Write to socket failed. Closing connection...\n"); 265 hello_sock_quit(ctx, -1); 266 return SPDK_POLLER_IDLE; 267 } 268 } 269 if (rc > 0) { 270 ctx->bytes_out += rc; 271 } 272 } 273 274 return rc > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 275 } 276 277 static int 278 hello_sock_connect(struct hello_context_t *ctx) 279 { 280 int rc; 281 char saddr[ADDR_STR_LEN], caddr[ADDR_STR_LEN]; 282 uint16_t cport, sport; 283 struct spdk_sock_impl_opts impl_opts; 284 size_t impl_opts_size = sizeof(impl_opts); 285 struct spdk_sock_opts opts; 286 287 spdk_sock_impl_get_opts(ctx->sock_impl_name, &impl_opts, &impl_opts_size); 288 impl_opts.enable_ktls = ctx->ktls; 289 impl_opts.tls_version = ctx->tls_version; 290 impl_opts.psk_identity = ctx->psk_identity; 291 impl_opts.tls_cipher_suites = "TLS_AES_128_GCM_SHA256"; 292 impl_opts.psk_key = ctx->psk_key; 293 impl_opts.psk_key_size = ctx->psk_key_size; 294 295 opts.opts_size = sizeof(opts); 296 spdk_sock_get_default_opts(&opts); 297 opts.zcopy = ctx->zcopy; 298 opts.impl_opts = &impl_opts; 299 opts.impl_opts_size = sizeof(impl_opts); 300 301 SPDK_NOTICELOG("Connecting to the server on %s:%d with sock_impl(%s)\n", ctx->host, ctx->port, 302 ctx->sock_impl_name); 303 304 ctx->sock = spdk_sock_connect_ext(ctx->host, ctx->port, ctx->sock_impl_name, &opts); 305 if (ctx->sock == NULL) { 306 SPDK_ERRLOG("connect error(%d): %s\n", errno, spdk_strerror(errno)); 307 return -1; 308 } 309 310 rc = spdk_sock_getaddr(ctx->sock, saddr, sizeof(saddr), &sport, caddr, sizeof(caddr), &cport); 311 if (rc < 0) { 312 SPDK_ERRLOG("Cannot get connection addresses\n"); 313 goto err; 314 } 315 316 SPDK_NOTICELOG("Connection accepted from (%s, %hu) to (%s, %hu)\n", caddr, cport, saddr, sport); 317 318 rc = fcntl(STDIN_FILENO, F_GETFL); 319 if (rc == -1) { 320 SPDK_ERRLOG("Getting file status flag failed: %s\n", strerror(errno)); 321 goto err; 322 } 323 324 if (fcntl(STDIN_FILENO, F_SETFL, rc | O_NONBLOCK) == -1) { 325 SPDK_ERRLOG("Setting file status flag failed: %s\n", strerror(errno)); 326 goto err; 327 } 328 329 g_is_running = true; 330 ctx->poller_in = SPDK_POLLER_REGISTER(hello_sock_recv_poll, ctx, 0); 331 ctx->poller_out = SPDK_POLLER_REGISTER(hello_sock_writev_poll, ctx, 0); 332 333 return 0; 334 err: 335 spdk_sock_close(&ctx->sock); 336 return -1; 337 } 338 339 static void 340 hello_sock_cb(void *arg, struct spdk_sock_group *group, struct spdk_sock *sock) 341 { 342 int rc; 343 struct hello_context_t *ctx = arg; 344 struct iovec iov = {}; 345 ssize_t n; 346 void *user_ctx; 347 348 rc = spdk_sock_recv_next(sock, &iov.iov_base, &user_ctx); 349 if (rc < 0) { 350 if (errno == EAGAIN || errno == EWOULDBLOCK) { 351 return; 352 } 353 354 if (errno != ENOTCONN && errno != ECONNRESET) { 355 SPDK_ERRLOG("spdk_sock_recv_zcopy() failed, errno %d: %s\n", 356 errno, spdk_strerror(errno)); 357 } 358 } 359 360 if (rc > 0) { 361 iov.iov_len = rc; 362 ctx->bytes_in += iov.iov_len; 363 n = spdk_sock_writev(sock, &iov, 1); 364 if (n > 0) { 365 assert(n == rc); 366 ctx->bytes_out += n; 367 } 368 369 spdk_sock_group_provide_buf(ctx->group, iov.iov_base, BUFFER_SIZE, NULL); 370 return; 371 } 372 373 /* Connection closed */ 374 SPDK_NOTICELOG("Connection closed\n"); 375 spdk_sock_group_remove_sock(group, sock); 376 spdk_sock_close(&sock); 377 } 378 379 static int 380 hello_sock_accept_poll(void *arg) 381 { 382 struct hello_context_t *ctx = arg; 383 struct spdk_sock *sock; 384 int rc; 385 int count = 0; 386 char saddr[ADDR_STR_LEN], caddr[ADDR_STR_LEN]; 387 uint16_t cport, sport; 388 389 if (!g_is_running) { 390 hello_sock_quit(ctx, 0); 391 return SPDK_POLLER_IDLE; 392 } 393 394 while (1) { 395 sock = spdk_sock_accept(ctx->sock); 396 if (sock != NULL) { 397 rc = spdk_sock_getaddr(sock, saddr, sizeof(saddr), &sport, caddr, sizeof(caddr), &cport); 398 if (rc < 0) { 399 SPDK_ERRLOG("Cannot get connection addresses\n"); 400 spdk_sock_close(&sock); 401 return SPDK_POLLER_IDLE; 402 } 403 404 SPDK_NOTICELOG("Accepting a new connection from (%s, %hu) to (%s, %hu)\n", 405 caddr, cport, saddr, sport); 406 407 rc = spdk_sock_group_add_sock(ctx->group, sock, 408 hello_sock_cb, ctx); 409 410 if (rc < 0) { 411 spdk_sock_close(&sock); 412 SPDK_ERRLOG("failed\n"); 413 break; 414 } 415 416 count++; 417 } else { 418 if (errno != EAGAIN && errno != EWOULDBLOCK) { 419 SPDK_ERRLOG("accept error(%d): %s\n", errno, spdk_strerror(errno)); 420 } 421 break; 422 } 423 } 424 425 return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 426 } 427 428 static int 429 hello_sock_group_poll(void *arg) 430 { 431 struct hello_context_t *ctx = arg; 432 int rc; 433 434 rc = spdk_sock_group_poll(ctx->group); 435 if (rc < 0) { 436 SPDK_ERRLOG("Failed to poll sock_group=%p\n", ctx->group); 437 } 438 439 return rc > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 440 } 441 442 static int 443 hello_sock_listen(struct hello_context_t *ctx) 444 { 445 struct spdk_sock_impl_opts impl_opts; 446 size_t impl_opts_size = sizeof(impl_opts); 447 struct spdk_sock_opts opts; 448 449 spdk_sock_impl_get_opts(ctx->sock_impl_name, &impl_opts, &impl_opts_size); 450 impl_opts.enable_ktls = ctx->ktls; 451 impl_opts.tls_version = ctx->tls_version; 452 impl_opts.psk_identity = ctx->psk_identity; 453 impl_opts.tls_cipher_suites = "TLS_AES_128_GCM_SHA256"; 454 impl_opts.psk_key = ctx->psk_key; 455 impl_opts.psk_key_size = ctx->psk_key_size; 456 457 opts.opts_size = sizeof(opts); 458 spdk_sock_get_default_opts(&opts); 459 opts.zcopy = ctx->zcopy; 460 opts.impl_opts = &impl_opts; 461 opts.impl_opts_size = sizeof(impl_opts); 462 463 ctx->sock = spdk_sock_listen_ext(ctx->host, ctx->port, ctx->sock_impl_name, &opts); 464 if (ctx->sock == NULL) { 465 SPDK_ERRLOG("Cannot create server socket\n"); 466 return -1; 467 } 468 469 SPDK_NOTICELOG("Listening connection on %s:%d with sock_impl(%s)\n", ctx->host, ctx->port, 470 ctx->sock_impl_name); 471 472 /* 473 * Create sock group for server socket 474 */ 475 ctx->group = spdk_sock_group_create(NULL); 476 if (ctx->group == NULL) { 477 SPDK_ERRLOG("Cannot create sock group\n"); 478 spdk_sock_close(&ctx->sock); 479 return -1; 480 } 481 482 spdk_sock_group_provide_buf(ctx->group, ctx->buf, BUFFER_SIZE, NULL); 483 484 g_is_running = true; 485 486 /* 487 * Start acceptor and group poller 488 */ 489 ctx->poller_in = SPDK_POLLER_REGISTER(hello_sock_accept_poll, ctx, 490 ACCEPT_TIMEOUT_US); 491 ctx->poller_out = SPDK_POLLER_REGISTER(hello_sock_group_poll, ctx, 0); 492 493 return 0; 494 } 495 496 static void 497 hello_sock_shutdown_cb(void) 498 { 499 g_is_running = false; 500 } 501 502 /* 503 * Our initial event that kicks off everything from main(). 504 */ 505 static void 506 hello_start(void *arg1) 507 { 508 struct hello_context_t *ctx = arg1; 509 int rc; 510 511 SPDK_NOTICELOG("Successfully started the application\n"); 512 513 if (ctx->is_server) { 514 rc = hello_sock_listen(ctx); 515 } else { 516 rc = hello_sock_connect(ctx); 517 } 518 519 if (rc) { 520 spdk_app_stop(-1); 521 return; 522 } 523 } 524 525 int 526 main(int argc, char **argv) 527 { 528 struct spdk_app_opts opts = {}; 529 int rc = 0; 530 struct hello_context_t hello_context = {}; 531 532 /* Set default values in opts structure. */ 533 spdk_app_opts_init(&opts, sizeof(opts)); 534 opts.name = "hello_sock"; 535 opts.shutdown_cb = hello_sock_shutdown_cb; 536 537 if ((rc = spdk_app_parse_args(argc, argv, &opts, "E:H:I:kKN:P:ST:VzZ", NULL, hello_sock_parse_arg, 538 hello_sock_usage)) != SPDK_APP_PARSE_ARGS_SUCCESS) { 539 exit(rc); 540 } 541 hello_context.is_server = g_is_server; 542 hello_context.host = g_host; 543 hello_context.sock_impl_name = g_sock_impl_name; 544 hello_context.port = g_port; 545 hello_context.zcopy = g_zcopy; 546 hello_context.ktls = g_ktls; 547 hello_context.tls_version = g_tls_version; 548 hello_context.psk_key = g_psk_key; 549 hello_context.psk_key_size = g_psk_key_size; 550 hello_context.psk_identity = g_psk_identity; 551 hello_context.verbose = g_verbose; 552 553 if (hello_context.sock_impl_name == NULL) { 554 hello_context.sock_impl_name = spdk_sock_get_default_impl(); 555 556 if (hello_context.sock_impl_name == NULL) { 557 SPDK_ERRLOG("No sock implementations available!\n"); 558 exit(-1); 559 } 560 } 561 562 hello_context.buf = calloc(1, BUFFER_SIZE); 563 if (hello_context.buf == NULL) { 564 SPDK_ERRLOG("Cannot allocate memory for hello_context buffer\n"); 565 exit(-1); 566 } 567 hello_context.n = 0; 568 569 if (hello_context.is_server) { 570 struct spdk_sock_impl_opts impl_opts = {}; 571 size_t len = sizeof(impl_opts); 572 573 rc = spdk_sock_impl_get_opts(hello_context.sock_impl_name, &impl_opts, &len); 574 if (rc < 0) { 575 exit(rc); 576 } 577 578 /* Our applications will post buffers to be used for receiving. That feature 579 * is mutually exclusive with the recv pipe, so we need to disable it. */ 580 impl_opts.enable_recv_pipe = false; 581 spdk_sock_impl_set_opts(hello_context.sock_impl_name, &impl_opts, len); 582 } 583 584 rc = spdk_app_start(&opts, hello_start, &hello_context); 585 if (rc) { 586 SPDK_ERRLOG("ERROR starting application\n"); 587 } 588 589 SPDK_NOTICELOG("Exiting from application\n"); 590 591 if (hello_context.verbose) { 592 printf("** %d bytes received, %d bytes sent **\n", 593 hello_context.bytes_in, hello_context.bytes_out); 594 } 595 596 /* Gracefully close out all of the SPDK subsystems. */ 597 spdk_app_fini(); 598 return rc; 599 } 600