1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2018 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/thread.h" 8 #include "spdk/env.h" 9 #include "spdk/event.h" 10 #include "spdk/log.h" 11 #include "spdk/string.h" 12 13 #include "spdk/sock.h" 14 #include "spdk/hexlify.h" 15 #include "spdk/nvmf.h" 16 17 #define ACCEPT_TIMEOUT_US 1000 18 #define CLOSE_TIMEOUT_US 1000000 19 #define BUFFER_SIZE 1024 20 #define ADDR_STR_LEN INET6_ADDRSTRLEN 21 22 static bool g_is_running; 23 24 static char *g_host; 25 static char *g_sock_impl_name; 26 static int g_port; 27 static bool g_is_server; 28 static int g_zcopy; 29 static int g_ktls; 30 static int g_tls_version; 31 static bool g_verbose; 32 static uint8_t g_psk_key[SPDK_TLS_PSK_MAX_LEN]; 33 static uint32_t g_psk_key_size; 34 static char *g_psk_identity; 35 36 /* 37 * We'll use this struct to gather housekeeping hello_context to pass between 38 * our events and callbacks. 39 */ 40 struct hello_context_t { 41 bool is_server; 42 char *host; 43 const char *sock_impl_name; 44 int port; 45 int zcopy; 46 int ktls; 47 int tls_version; 48 uint8_t *psk_key; 49 uint32_t psk_key_size; 50 char *psk_identity; 51 52 bool verbose; 53 int bytes_in; 54 int bytes_out; 55 56 struct spdk_sock *sock; 57 58 struct spdk_sock_group *group; 59 void *buf; 60 struct spdk_poller *poller_in; 61 struct spdk_poller *poller_out; 62 struct spdk_poller *time_out; 63 64 int rc; 65 ssize_t n; 66 }; 67 68 /* 69 * Usage function for printing parameters that are specific to this application 70 */ 71 static void 72 hello_sock_usage(void) 73 { 74 printf(" -E psk_key Default PSK KEY in hexadecimal digits, e.g. 1234567890ABCDEF (only applies when sock_impl == ssl)\n"); 75 printf(" -H host_addr host address\n"); 76 printf(" -I psk_id Default PSK ID, e.g. psk.spdk.io (only applies when sock_impl == ssl)\n"); 77 printf(" -P port port number\n"); 78 printf(" -N sock_impl socket implementation, e.g., -N posix or -N uring\n"); 79 printf(" -S start in server mode\n"); 80 printf(" -T tls_ver TLS version, e.g., -T 12 or -T 13. If omitted, auto-negotiation will take place\n"); 81 printf(" -k disable KTLS for the given sock implementation (default)\n"); 82 printf(" -K enable KTLS for the given sock implementation\n"); 83 printf(" -V print out additional information\n"); 84 printf(" -z disable zero copy send for the given sock implementation\n"); 85 printf(" -Z enable zero copy send for the given sock implementation\n"); 86 } 87 88 /* 89 * This function is called to parse the parameters that are specific to this application 90 */ 91 static int 92 hello_sock_parse_arg(int ch, char *arg) 93 { 94 char *unhexlified; 95 96 switch (ch) { 97 case 'E': 98 g_psk_key_size = strlen(arg) / 2; 99 if (g_psk_key_size > SPDK_TLS_PSK_MAX_LEN) { 100 fprintf(stderr, "Invalid PSK: too long (%"PRIu32")\n", g_psk_key_size); 101 return -EINVAL; 102 } 103 unhexlified = spdk_unhexlify(arg); 104 if (unhexlified == NULL) { 105 fprintf(stderr, "Invalid PSK: not in a hex format\n"); 106 return -EINVAL; 107 } 108 memcpy(g_psk_key, unhexlified, g_psk_key_size); 109 free(unhexlified); 110 break; 111 case 'H': 112 g_host = arg; 113 break; 114 case 'I': 115 g_psk_identity = arg; 116 break; 117 case 'N': 118 g_sock_impl_name = arg; 119 break; 120 case 'P': 121 g_port = spdk_strtol(arg, 10); 122 if (g_port < 0) { 123 fprintf(stderr, "Invalid port ID\n"); 124 return g_port; 125 } 126 break; 127 case 'S': 128 g_is_server = 1; 129 break; 130 case 'K': 131 g_ktls = 1; 132 break; 133 case 'k': 134 g_ktls = 0; 135 break; 136 case 'T': 137 g_tls_version = spdk_strtol(arg, 10); 138 if (g_tls_version < 0) { 139 fprintf(stderr, "Invalid TLS version\n"); 140 return g_tls_version; 141 } 142 break; 143 case 'V': 144 g_verbose = true; 145 break; 146 case 'Z': 147 g_zcopy = 1; 148 break; 149 case 'z': 150 g_zcopy = 0; 151 break; 152 default: 153 return -EINVAL; 154 } 155 return 0; 156 } 157 158 static int 159 hello_sock_close_timeout_poll(void *arg) 160 { 161 struct hello_context_t *ctx = arg; 162 SPDK_NOTICELOG("Connection closed\n"); 163 164 spdk_poller_unregister(&ctx->time_out); 165 spdk_poller_unregister(&ctx->poller_in); 166 spdk_sock_close(&ctx->sock); 167 spdk_sock_group_close(&ctx->group); 168 169 spdk_app_stop(ctx->rc); 170 return SPDK_POLLER_BUSY; 171 } 172 173 static int 174 hello_sock_quit(struct hello_context_t *ctx, int rc) 175 { 176 ctx->rc = rc; 177 spdk_poller_unregister(&ctx->poller_out); 178 if (!ctx->time_out) { 179 ctx->time_out = SPDK_POLLER_REGISTER(hello_sock_close_timeout_poll, ctx, 180 CLOSE_TIMEOUT_US); 181 } 182 return 0; 183 } 184 185 static int 186 hello_sock_recv_poll(void *arg) 187 { 188 struct hello_context_t *ctx = arg; 189 int rc; 190 char buf_in[BUFFER_SIZE]; 191 192 /* 193 * Get response 194 */ 195 rc = spdk_sock_recv(ctx->sock, buf_in, sizeof(buf_in) - 1); 196 197 if (rc <= 0) { 198 if (errno == EAGAIN || errno == EWOULDBLOCK) { 199 return SPDK_POLLER_IDLE; 200 } 201 202 hello_sock_quit(ctx, -1); 203 SPDK_ERRLOG("spdk_sock_recv() failed, errno %d: %s\n", 204 errno, spdk_strerror(errno)); 205 return SPDK_POLLER_BUSY; 206 } 207 208 if (rc > 0) { 209 ctx->bytes_in += rc; 210 buf_in[rc] = '\0'; 211 printf("%s", buf_in); 212 } 213 214 return SPDK_POLLER_BUSY; 215 } 216 217 static int 218 hello_sock_writev_poll(void *arg) 219 { 220 struct hello_context_t *ctx = arg; 221 int rc = 0; 222 struct iovec iov; 223 ssize_t n; 224 225 /* If previously we could not send any bytes, we should try again with the same buffer. */ 226 if (ctx->n != 0) { 227 iov.iov_base = ctx->buf; 228 iov.iov_len = ctx->n; 229 errno = 0; 230 rc = spdk_sock_writev(ctx->sock, &iov, 1); 231 if (rc < 0) { 232 if (errno == EAGAIN) { 233 return SPDK_POLLER_BUSY; 234 } 235 SPDK_ERRLOG("Write to socket failed. Closing connection...\n"); 236 hello_sock_quit(ctx, -1); 237 return SPDK_POLLER_IDLE; 238 } 239 ctx->bytes_out += rc; 240 ctx->n = 0; 241 } 242 243 n = read(STDIN_FILENO, ctx->buf, BUFFER_SIZE); 244 if (n == 0 || !g_is_running) { 245 /* EOF */ 246 SPDK_NOTICELOG("Closing connection...\n"); 247 hello_sock_quit(ctx, 0); 248 return SPDK_POLLER_IDLE; 249 } 250 if (n > 0) { 251 /* 252 * Send message to the server 253 */ 254 iov.iov_base = ctx->buf; 255 iov.iov_len = n; 256 errno = 0; 257 rc = spdk_sock_writev(ctx->sock, &iov, 1); 258 if (rc < 0) { 259 if (errno == EAGAIN) { 260 ctx->n = n; 261 } else { 262 SPDK_ERRLOG("Write to socket failed. Closing connection...\n"); 263 hello_sock_quit(ctx, -1); 264 return SPDK_POLLER_IDLE; 265 } 266 } 267 if (rc > 0) { 268 ctx->bytes_out += rc; 269 } 270 } 271 272 return rc > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 273 } 274 275 static int 276 hello_sock_connect(struct hello_context_t *ctx) 277 { 278 int rc; 279 char saddr[ADDR_STR_LEN], caddr[ADDR_STR_LEN]; 280 uint16_t cport, sport; 281 struct spdk_sock_impl_opts impl_opts; 282 size_t impl_opts_size = sizeof(impl_opts); 283 struct spdk_sock_opts opts; 284 285 spdk_sock_impl_get_opts(ctx->sock_impl_name, &impl_opts, &impl_opts_size); 286 impl_opts.enable_ktls = ctx->ktls; 287 impl_opts.tls_version = ctx->tls_version; 288 impl_opts.psk_identity = ctx->psk_identity; 289 impl_opts.tls_cipher_suites = "TLS_AES_128_GCM_SHA256"; 290 impl_opts.psk_key = ctx->psk_key; 291 impl_opts.psk_key_size = ctx->psk_key_size; 292 293 opts.opts_size = sizeof(opts); 294 spdk_sock_get_default_opts(&opts); 295 opts.zcopy = ctx->zcopy; 296 opts.impl_opts = &impl_opts; 297 opts.impl_opts_size = sizeof(impl_opts); 298 299 SPDK_NOTICELOG("Connecting to the server on %s:%d with sock_impl(%s)\n", ctx->host, ctx->port, 300 ctx->sock_impl_name); 301 302 ctx->sock = spdk_sock_connect_ext(ctx->host, ctx->port, ctx->sock_impl_name, &opts); 303 if (ctx->sock == NULL) { 304 SPDK_ERRLOG("connect error(%d): %s\n", errno, spdk_strerror(errno)); 305 return -1; 306 } 307 308 rc = spdk_sock_getaddr(ctx->sock, saddr, sizeof(saddr), &sport, caddr, sizeof(caddr), &cport); 309 if (rc < 0) { 310 SPDK_ERRLOG("Cannot get connection addresses\n"); 311 goto err; 312 } 313 314 SPDK_NOTICELOG("Connection accepted from (%s, %hu) to (%s, %hu)\n", caddr, cport, saddr, sport); 315 316 rc = fcntl(STDIN_FILENO, F_GETFL); 317 if (rc == -1) { 318 SPDK_ERRLOG("Getting file status flag failed: %s\n", strerror(errno)); 319 goto err; 320 } 321 322 if (fcntl(STDIN_FILENO, F_SETFL, rc | O_NONBLOCK) == -1) { 323 SPDK_ERRLOG("Setting file status flag failed: %s\n", strerror(errno)); 324 goto err; 325 } 326 327 g_is_running = true; 328 ctx->poller_in = SPDK_POLLER_REGISTER(hello_sock_recv_poll, ctx, 0); 329 ctx->poller_out = SPDK_POLLER_REGISTER(hello_sock_writev_poll, ctx, 0); 330 331 return 0; 332 err: 333 spdk_sock_close(&ctx->sock); 334 return -1; 335 } 336 337 static void 338 hello_sock_cb(void *arg, struct spdk_sock_group *group, struct spdk_sock *sock) 339 { 340 int rc; 341 struct hello_context_t *ctx = arg; 342 struct iovec iov = {}; 343 ssize_t n; 344 void *user_ctx; 345 346 rc = spdk_sock_recv_next(sock, &iov.iov_base, &user_ctx); 347 if (rc < 0) { 348 if (errno == EAGAIN || errno == EWOULDBLOCK) { 349 return; 350 } 351 352 if (errno != ENOTCONN && errno != ECONNRESET) { 353 SPDK_ERRLOG("spdk_sock_recv_zcopy() failed, errno %d: %s\n", 354 errno, spdk_strerror(errno)); 355 } 356 } 357 358 if (rc > 0) { 359 iov.iov_len = rc; 360 ctx->bytes_in += iov.iov_len; 361 n = spdk_sock_writev(sock, &iov, 1); 362 if (n > 0) { 363 assert(n == rc); 364 ctx->bytes_out += n; 365 } 366 367 spdk_sock_group_provide_buf(ctx->group, iov.iov_base, BUFFER_SIZE, NULL); 368 return; 369 } 370 371 /* Connection closed */ 372 SPDK_NOTICELOG("Connection closed\n"); 373 spdk_sock_group_remove_sock(group, sock); 374 spdk_sock_close(&sock); 375 } 376 377 static int 378 hello_sock_accept_poll(void *arg) 379 { 380 struct hello_context_t *ctx = arg; 381 struct spdk_sock *sock; 382 int rc; 383 int count = 0; 384 char saddr[ADDR_STR_LEN], caddr[ADDR_STR_LEN]; 385 uint16_t cport, sport; 386 387 if (!g_is_running) { 388 hello_sock_quit(ctx, 0); 389 return SPDK_POLLER_IDLE; 390 } 391 392 while (1) { 393 sock = spdk_sock_accept(ctx->sock); 394 if (sock != NULL) { 395 rc = spdk_sock_getaddr(sock, saddr, sizeof(saddr), &sport, caddr, sizeof(caddr), &cport); 396 if (rc < 0) { 397 SPDK_ERRLOG("Cannot get connection addresses\n"); 398 spdk_sock_close(&sock); 399 return SPDK_POLLER_IDLE; 400 } 401 402 SPDK_NOTICELOG("Accepting a new connection from (%s, %hu) to (%s, %hu)\n", 403 caddr, cport, saddr, sport); 404 405 rc = spdk_sock_group_add_sock(ctx->group, sock, 406 hello_sock_cb, ctx); 407 408 if (rc < 0) { 409 spdk_sock_close(&sock); 410 SPDK_ERRLOG("failed\n"); 411 break; 412 } 413 414 count++; 415 } else { 416 if (errno != EAGAIN && errno != EWOULDBLOCK) { 417 SPDK_ERRLOG("accept error(%d): %s\n", errno, spdk_strerror(errno)); 418 } 419 break; 420 } 421 } 422 423 return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 424 } 425 426 static int 427 hello_sock_group_poll(void *arg) 428 { 429 struct hello_context_t *ctx = arg; 430 int rc; 431 432 rc = spdk_sock_group_poll(ctx->group); 433 if (rc < 0) { 434 SPDK_ERRLOG("Failed to poll sock_group=%p\n", ctx->group); 435 } 436 437 return rc > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 438 } 439 440 static int 441 hello_sock_listen(struct hello_context_t *ctx) 442 { 443 struct spdk_sock_impl_opts impl_opts; 444 size_t impl_opts_size = sizeof(impl_opts); 445 struct spdk_sock_opts opts; 446 447 spdk_sock_impl_get_opts(ctx->sock_impl_name, &impl_opts, &impl_opts_size); 448 impl_opts.enable_ktls = ctx->ktls; 449 impl_opts.tls_version = ctx->tls_version; 450 impl_opts.psk_identity = ctx->psk_identity; 451 impl_opts.tls_cipher_suites = "TLS_AES_128_GCM_SHA256"; 452 impl_opts.psk_key = ctx->psk_key; 453 impl_opts.psk_key_size = ctx->psk_key_size; 454 455 opts.opts_size = sizeof(opts); 456 spdk_sock_get_default_opts(&opts); 457 opts.zcopy = ctx->zcopy; 458 opts.impl_opts = &impl_opts; 459 opts.impl_opts_size = sizeof(impl_opts); 460 461 ctx->sock = spdk_sock_listen_ext(ctx->host, ctx->port, ctx->sock_impl_name, &opts); 462 if (ctx->sock == NULL) { 463 SPDK_ERRLOG("Cannot create server socket\n"); 464 return -1; 465 } 466 467 SPDK_NOTICELOG("Listening connection on %s:%d with sock_impl(%s)\n", ctx->host, ctx->port, 468 ctx->sock_impl_name); 469 470 /* 471 * Create sock group for server socket 472 */ 473 ctx->group = spdk_sock_group_create(NULL); 474 if (ctx->group == NULL) { 475 SPDK_ERRLOG("Cannot create sock group\n"); 476 spdk_sock_close(&ctx->sock); 477 return -1; 478 } 479 480 spdk_sock_group_provide_buf(ctx->group, ctx->buf, BUFFER_SIZE, NULL); 481 482 g_is_running = true; 483 484 /* 485 * Start acceptor and group poller 486 */ 487 ctx->poller_in = SPDK_POLLER_REGISTER(hello_sock_accept_poll, ctx, 488 ACCEPT_TIMEOUT_US); 489 ctx->poller_out = SPDK_POLLER_REGISTER(hello_sock_group_poll, ctx, 0); 490 491 return 0; 492 } 493 494 static void 495 hello_sock_shutdown_cb(void) 496 { 497 g_is_running = false; 498 } 499 500 /* 501 * Our initial event that kicks off everything from main(). 502 */ 503 static void 504 hello_start(void *arg1) 505 { 506 struct hello_context_t *ctx = arg1; 507 int rc; 508 509 SPDK_NOTICELOG("Successfully started the application\n"); 510 511 if (ctx->is_server) { 512 rc = hello_sock_listen(ctx); 513 } else { 514 rc = hello_sock_connect(ctx); 515 } 516 517 if (rc) { 518 spdk_app_stop(-1); 519 return; 520 } 521 } 522 523 int 524 main(int argc, char **argv) 525 { 526 struct spdk_app_opts opts = {}; 527 int rc = 0; 528 struct hello_context_t hello_context = {}; 529 530 /* Set default values in opts structure. */ 531 spdk_app_opts_init(&opts, sizeof(opts)); 532 opts.name = "hello_sock"; 533 opts.shutdown_cb = hello_sock_shutdown_cb; 534 opts.rpc_addr = NULL; 535 536 if ((rc = spdk_app_parse_args(argc, argv, &opts, "E:H:I:kKN:P:ST:VzZ", NULL, hello_sock_parse_arg, 537 hello_sock_usage)) != SPDK_APP_PARSE_ARGS_SUCCESS) { 538 exit(rc); 539 } 540 hello_context.is_server = g_is_server; 541 hello_context.host = g_host; 542 hello_context.sock_impl_name = g_sock_impl_name; 543 hello_context.port = g_port; 544 hello_context.zcopy = g_zcopy; 545 hello_context.ktls = g_ktls; 546 hello_context.tls_version = g_tls_version; 547 hello_context.psk_key = g_psk_key; 548 hello_context.psk_key_size = g_psk_key_size; 549 hello_context.psk_identity = g_psk_identity; 550 hello_context.verbose = g_verbose; 551 552 if (hello_context.sock_impl_name == NULL) { 553 hello_context.sock_impl_name = spdk_sock_get_default_impl(); 554 555 if (hello_context.sock_impl_name == NULL) { 556 SPDK_ERRLOG("No sock implementations available!\n"); 557 exit(-1); 558 } 559 } 560 561 hello_context.buf = calloc(1, BUFFER_SIZE); 562 if (hello_context.buf == NULL) { 563 SPDK_ERRLOG("Cannot allocate memory for hello_context buffer\n"); 564 exit(-1); 565 } 566 hello_context.n = 0; 567 568 if (hello_context.is_server) { 569 struct spdk_sock_impl_opts impl_opts = {}; 570 size_t len = sizeof(impl_opts); 571 572 rc = spdk_sock_impl_get_opts(hello_context.sock_impl_name, &impl_opts, &len); 573 if (rc < 0) { 574 free(hello_context.buf); 575 exit(rc); 576 } 577 578 /* Our applications will post buffers to be used for receiving. That feature 579 * is mutually exclusive with the recv pipe, so we need to disable it. */ 580 impl_opts.enable_recv_pipe = false; 581 spdk_sock_impl_set_opts(hello_context.sock_impl_name, &impl_opts, len); 582 } 583 584 rc = spdk_app_start(&opts, hello_start, &hello_context); 585 if (rc) { 586 SPDK_ERRLOG("ERROR starting application\n"); 587 } 588 589 SPDK_NOTICELOG("Exiting from application\n"); 590 591 if (hello_context.verbose) { 592 printf("** %d bytes received, %d bytes sent **\n", 593 hello_context.bytes_in, hello_context.bytes_out); 594 } 595 596 /* Gracefully close out all of the SPDK subsystems. */ 597 spdk_app_fini(); 598 free(hello_context.buf); 599 return rc; 600 } 601