1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2018 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/thread.h" 8 #include "spdk/env.h" 9 #include "spdk/event.h" 10 #include "spdk/log.h" 11 #include "spdk/string.h" 12 13 #include "spdk/sock.h" 14 #include "spdk/hexlify.h" 15 #include "spdk/nvmf.h" 16 17 #define ACCEPT_TIMEOUT_US 1000 18 #define CLOSE_TIMEOUT_US 1000000 19 #define BUFFER_SIZE 1024 20 #define ADDR_STR_LEN INET6_ADDRSTRLEN 21 22 static bool g_is_running; 23 24 static char *g_host; 25 static char *g_sock_impl_name; 26 static int g_port; 27 static bool g_is_server; 28 static int g_zcopy; 29 static int g_ktls; 30 static int g_tls_version; 31 static bool g_verbose; 32 static char *g_psk_key; 33 static char *g_psk_identity; 34 35 /* 36 * We'll use this struct to gather housekeeping hello_context to pass between 37 * our events and callbacks. 38 */ 39 struct hello_context_t { 40 bool is_server; 41 char *host; 42 const char *sock_impl_name; 43 int port; 44 int zcopy; 45 int ktls; 46 int tls_version; 47 char *psk_key; 48 char *psk_identity; 49 50 bool verbose; 51 int bytes_in; 52 int bytes_out; 53 54 struct spdk_sock *sock; 55 56 struct spdk_sock_group *group; 57 void *buf; 58 struct spdk_poller *poller_in; 59 struct spdk_poller *poller_out; 60 struct spdk_poller *time_out; 61 62 int rc; 63 }; 64 65 /* 66 * Usage function for printing parameters that are specific to this application 67 */ 68 static void 69 hello_sock_usage(void) 70 { 71 printf(" -E psk_key Default PSK KEY in hexadecimal digits, e.g. 1234567890ABCDEF (only applies when sock_impl == ssl)\n"); 72 printf(" -H host_addr host address\n"); 73 printf(" -I psk_id Default PSK ID, e.g. psk.spdk.io (only applies when sock_impl == ssl)\n"); 74 printf(" -P port port number\n"); 75 printf(" -N sock_impl socket implementation, e.g., -N posix or -N uring\n"); 76 printf(" -S start in server mode\n"); 77 printf(" -T tls_ver TLS version, e.g., -T 12 or -T 13. If omitted, auto-negotiation will take place\n"); 78 printf(" -k disable KTLS for the given sock implementation (default)\n"); 79 printf(" -K enable KTLS for the given sock implementation\n"); 80 printf(" -V print out additional information\n"); 81 printf(" -z disable zero copy send for the given sock implementation\n"); 82 printf(" -Z enable zero copy send for the given sock implementation\n"); 83 } 84 85 /* 86 * This function is called to parse the parameters that are specific to this application 87 */ 88 static int 89 hello_sock_parse_arg(int ch, char *arg) 90 { 91 switch (ch) { 92 case 'E': 93 g_psk_key = arg; 94 break; 95 case 'H': 96 g_host = arg; 97 break; 98 case 'I': 99 g_psk_identity = arg; 100 break; 101 case 'N': 102 g_sock_impl_name = arg; 103 break; 104 case 'P': 105 g_port = spdk_strtol(arg, 10); 106 if (g_port < 0) { 107 fprintf(stderr, "Invalid port ID\n"); 108 return g_port; 109 } 110 break; 111 case 'S': 112 g_is_server = 1; 113 break; 114 case 'K': 115 g_ktls = 1; 116 break; 117 case 'k': 118 g_ktls = 0; 119 break; 120 case 'T': 121 g_tls_version = spdk_strtol(arg, 10); 122 if (g_tls_version < 0) { 123 fprintf(stderr, "Invalid TLS version\n"); 124 return g_tls_version; 125 } 126 break; 127 case 'V': 128 g_verbose = true; 129 break; 130 case 'Z': 131 g_zcopy = 1; 132 break; 133 case 'z': 134 g_zcopy = 0; 135 break; 136 default: 137 return -EINVAL; 138 } 139 return 0; 140 } 141 142 static int 143 hello_sock_close_timeout_poll(void *arg) 144 { 145 struct hello_context_t *ctx = arg; 146 SPDK_NOTICELOG("Connection closed\n"); 147 148 free(ctx->buf); 149 150 spdk_poller_unregister(&ctx->time_out); 151 spdk_poller_unregister(&ctx->poller_in); 152 spdk_sock_close(&ctx->sock); 153 spdk_sock_group_close(&ctx->group); 154 155 spdk_app_stop(ctx->rc); 156 return SPDK_POLLER_BUSY; 157 } 158 159 static int 160 hello_sock_quit(struct hello_context_t *ctx, int rc) 161 { 162 ctx->rc = rc; 163 spdk_poller_unregister(&ctx->poller_out); 164 if (!ctx->time_out) { 165 ctx->time_out = SPDK_POLLER_REGISTER(hello_sock_close_timeout_poll, ctx, 166 CLOSE_TIMEOUT_US); 167 } 168 return 0; 169 } 170 171 static int 172 hello_sock_recv_poll(void *arg) 173 { 174 struct hello_context_t *ctx = arg; 175 int rc; 176 char buf_in[BUFFER_SIZE]; 177 178 /* 179 * Get response 180 */ 181 rc = spdk_sock_recv(ctx->sock, buf_in, sizeof(buf_in) - 1); 182 183 if (rc <= 0) { 184 if (errno == EAGAIN || errno == EWOULDBLOCK) { 185 return SPDK_POLLER_IDLE; 186 } 187 188 SPDK_ERRLOG("spdk_sock_recv() failed, errno %d: %s\n", 189 errno, spdk_strerror(errno)); 190 return SPDK_POLLER_BUSY; 191 } 192 193 if (rc > 0) { 194 ctx->bytes_in += rc; 195 buf_in[rc] = '\0'; 196 printf("%s", buf_in); 197 } 198 199 return SPDK_POLLER_BUSY; 200 } 201 202 static int 203 hello_sock_writev_poll(void *arg) 204 { 205 struct hello_context_t *ctx = arg; 206 int rc = 0; 207 char buf_out[BUFFER_SIZE]; 208 struct iovec iov; 209 ssize_t n; 210 211 n = read(STDIN_FILENO, buf_out, sizeof(buf_out)); 212 if (n == 0 || !g_is_running) { 213 /* EOF */ 214 SPDK_NOTICELOG("Closing connection...\n"); 215 hello_sock_quit(ctx, 0); 216 return SPDK_POLLER_IDLE; 217 } 218 if (n > 0) { 219 /* 220 * Send message to the server 221 */ 222 iov.iov_base = buf_out; 223 iov.iov_len = n; 224 rc = spdk_sock_writev(ctx->sock, &iov, 1); 225 if (rc > 0) { 226 ctx->bytes_out += rc; 227 } 228 } 229 return rc > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 230 } 231 232 static int 233 hello_sock_connect(struct hello_context_t *ctx) 234 { 235 int rc; 236 char saddr[ADDR_STR_LEN], caddr[ADDR_STR_LEN]; 237 uint16_t cport, sport; 238 struct spdk_sock_impl_opts impl_opts; 239 size_t impl_opts_size = sizeof(impl_opts); 240 struct spdk_sock_opts opts; 241 char psk[SPDK_TLS_PSK_MAX_LEN] = {}; 242 char *unhexlified; 243 244 spdk_sock_impl_get_opts(ctx->sock_impl_name, &impl_opts, &impl_opts_size); 245 impl_opts.enable_ktls = ctx->ktls; 246 impl_opts.tls_version = ctx->tls_version; 247 impl_opts.psk_identity = ctx->psk_identity; 248 impl_opts.tls_cipher_suites = "TLS_AES_128_GCM_SHA256"; 249 250 opts.opts_size = sizeof(opts); 251 spdk_sock_get_default_opts(&opts); 252 opts.zcopy = ctx->zcopy; 253 opts.impl_opts = &impl_opts; 254 opts.impl_opts_size = sizeof(impl_opts); 255 256 if (ctx->psk_key) { 257 impl_opts.psk_key_size = strlen(ctx->psk_key) / 2; 258 if (impl_opts.psk_key_size > SPDK_TLS_PSK_MAX_LEN) { 259 SPDK_ERRLOG("Insufficient buffer size for PSK"); 260 return -EINVAL; 261 } 262 unhexlified = spdk_unhexlify(ctx->psk_key); 263 if (unhexlified == NULL) { 264 SPDK_ERRLOG("Could not unhexlify PSK"); 265 return -EINVAL; 266 } 267 memcpy(psk, unhexlified, impl_opts.psk_key_size); 268 free(unhexlified); 269 impl_opts.psk_key = psk; 270 } 271 272 SPDK_NOTICELOG("Connecting to the server on %s:%d with sock_impl(%s)\n", ctx->host, ctx->port, 273 ctx->sock_impl_name); 274 275 ctx->sock = spdk_sock_connect_ext(ctx->host, ctx->port, ctx->sock_impl_name, &opts); 276 if (ctx->sock == NULL) { 277 SPDK_ERRLOG("connect error(%d): %s\n", errno, spdk_strerror(errno)); 278 return -1; 279 } 280 281 rc = spdk_sock_getaddr(ctx->sock, saddr, sizeof(saddr), &sport, caddr, sizeof(caddr), &cport); 282 if (rc < 0) { 283 SPDK_ERRLOG("Cannot get connection addresses\n"); 284 goto err; 285 } 286 287 SPDK_NOTICELOG("Connection accepted from (%s, %hu) to (%s, %hu)\n", caddr, cport, saddr, sport); 288 289 rc = fcntl(STDIN_FILENO, F_GETFL); 290 if (rc == -1) { 291 SPDK_ERRLOG("Getting file status flag failed: %s\n", strerror(errno)); 292 goto err; 293 } 294 295 if (fcntl(STDIN_FILENO, F_SETFL, rc | O_NONBLOCK) == -1) { 296 SPDK_ERRLOG("Setting file status flag failed: %s\n", strerror(errno)); 297 goto err; 298 } 299 300 g_is_running = true; 301 ctx->poller_in = SPDK_POLLER_REGISTER(hello_sock_recv_poll, ctx, 0); 302 ctx->poller_out = SPDK_POLLER_REGISTER(hello_sock_writev_poll, ctx, 0); 303 304 return 0; 305 err: 306 spdk_sock_close(&ctx->sock); 307 return -1; 308 } 309 310 static void 311 hello_sock_cb(void *arg, struct spdk_sock_group *group, struct spdk_sock *sock) 312 { 313 int rc; 314 struct hello_context_t *ctx = arg; 315 struct iovec iov = {}; 316 ssize_t n; 317 void *user_ctx; 318 319 rc = spdk_sock_recv_next(sock, &iov.iov_base, &user_ctx); 320 if (rc < 0) { 321 if (errno == EAGAIN || errno == EWOULDBLOCK) { 322 return; 323 } 324 325 if (errno != ENOTCONN && errno != ECONNRESET) { 326 SPDK_ERRLOG("spdk_sock_recv_zcopy() failed, errno %d: %s\n", 327 errno, spdk_strerror(errno)); 328 } 329 } 330 331 iov.iov_len = rc; 332 333 if (iov.iov_len > 0) { 334 ctx->bytes_in += iov.iov_len; 335 n = spdk_sock_writev(sock, &iov, 1); 336 if (n > 0) { 337 assert(n == rc); 338 ctx->bytes_out += n; 339 } 340 341 spdk_sock_group_provide_buf(ctx->group, iov.iov_base, BUFFER_SIZE, NULL); 342 return; 343 } 344 345 /* Connection closed */ 346 SPDK_NOTICELOG("Connection closed\n"); 347 spdk_sock_group_remove_sock(group, sock); 348 spdk_sock_close(&sock); 349 } 350 351 static int 352 hello_sock_accept_poll(void *arg) 353 { 354 struct hello_context_t *ctx = arg; 355 struct spdk_sock *sock; 356 int rc; 357 int count = 0; 358 char saddr[ADDR_STR_LEN], caddr[ADDR_STR_LEN]; 359 uint16_t cport, sport; 360 361 if (!g_is_running) { 362 hello_sock_quit(ctx, 0); 363 return SPDK_POLLER_IDLE; 364 } 365 366 while (1) { 367 sock = spdk_sock_accept(ctx->sock); 368 if (sock != NULL) { 369 rc = spdk_sock_getaddr(sock, saddr, sizeof(saddr), &sport, caddr, sizeof(caddr), &cport); 370 if (rc < 0) { 371 SPDK_ERRLOG("Cannot get connection addresses\n"); 372 spdk_sock_close(&sock); 373 return SPDK_POLLER_IDLE; 374 } 375 376 SPDK_NOTICELOG("Accepting a new connection from (%s, %hu) to (%s, %hu)\n", 377 caddr, cport, saddr, sport); 378 379 rc = spdk_sock_group_add_sock(ctx->group, sock, 380 hello_sock_cb, ctx); 381 382 if (rc < 0) { 383 spdk_sock_close(&sock); 384 SPDK_ERRLOG("failed\n"); 385 break; 386 } 387 388 count++; 389 } else { 390 if (errno != EAGAIN && errno != EWOULDBLOCK) { 391 SPDK_ERRLOG("accept error(%d): %s\n", errno, spdk_strerror(errno)); 392 } 393 break; 394 } 395 } 396 397 return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 398 } 399 400 static int 401 hello_sock_group_poll(void *arg) 402 { 403 struct hello_context_t *ctx = arg; 404 int rc; 405 406 rc = spdk_sock_group_poll(ctx->group); 407 if (rc < 0) { 408 SPDK_ERRLOG("Failed to poll sock_group=%p\n", ctx->group); 409 } 410 411 return rc > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 412 } 413 414 static int 415 hello_sock_listen(struct hello_context_t *ctx) 416 { 417 struct spdk_sock_impl_opts impl_opts; 418 size_t impl_opts_size = sizeof(impl_opts); 419 struct spdk_sock_opts opts; 420 static char psk[SPDK_TLS_PSK_MAX_LEN] = {}; 421 char *unhexlified; 422 423 spdk_sock_impl_get_opts(ctx->sock_impl_name, &impl_opts, &impl_opts_size); 424 impl_opts.enable_ktls = ctx->ktls; 425 impl_opts.tls_version = ctx->tls_version; 426 impl_opts.psk_identity = ctx->psk_identity; 427 impl_opts.tls_cipher_suites = "TLS_AES_128_GCM_SHA256"; 428 429 opts.opts_size = sizeof(opts); 430 spdk_sock_get_default_opts(&opts); 431 opts.zcopy = ctx->zcopy; 432 opts.impl_opts = &impl_opts; 433 opts.impl_opts_size = sizeof(impl_opts); 434 435 if (ctx->psk_key) { 436 impl_opts.psk_key_size = strlen(ctx->psk_key) / 2; 437 if (impl_opts.psk_key_size > SPDK_TLS_PSK_MAX_LEN) { 438 SPDK_ERRLOG("Insufficient buffer size for PSK"); 439 return -EINVAL; 440 } 441 unhexlified = spdk_unhexlify(ctx->psk_key); 442 if (unhexlified == NULL) { 443 SPDK_ERRLOG("Could not unhexlify PSK"); 444 return -EINVAL; 445 } 446 memcpy(psk, unhexlified, impl_opts.psk_key_size); 447 free(unhexlified); 448 impl_opts.psk_key = psk; 449 } 450 451 ctx->sock = spdk_sock_listen_ext(ctx->host, ctx->port, ctx->sock_impl_name, &opts); 452 if (ctx->sock == NULL) { 453 SPDK_ERRLOG("Cannot create server socket\n"); 454 return -1; 455 } 456 457 SPDK_NOTICELOG("Listening connection on %s:%d with sock_impl(%s)\n", ctx->host, ctx->port, 458 ctx->sock_impl_name); 459 460 /* 461 * Create sock group for server socket 462 */ 463 ctx->group = spdk_sock_group_create(NULL); 464 if (ctx->group == NULL) { 465 SPDK_ERRLOG("Cannot create sock group\n"); 466 spdk_sock_close(&ctx->sock); 467 return -1; 468 } 469 470 /* 471 * Provide a buffer to the group to be used with receive. 472 */ 473 ctx->buf = calloc(1, BUFFER_SIZE); 474 if (ctx->buf == NULL) { 475 SPDK_ERRLOG("Cannot allocate memory for sock group\n"); 476 spdk_sock_close(&ctx->sock); 477 return -1; 478 } 479 480 spdk_sock_group_provide_buf(ctx->group, ctx->buf, BUFFER_SIZE, NULL); 481 482 g_is_running = true; 483 484 /* 485 * Start acceptor and group poller 486 */ 487 ctx->poller_in = SPDK_POLLER_REGISTER(hello_sock_accept_poll, ctx, 488 ACCEPT_TIMEOUT_US); 489 ctx->poller_out = SPDK_POLLER_REGISTER(hello_sock_group_poll, ctx, 0); 490 491 return 0; 492 } 493 494 static void 495 hello_sock_shutdown_cb(void) 496 { 497 g_is_running = false; 498 } 499 500 /* 501 * Our initial event that kicks off everything from main(). 502 */ 503 static void 504 hello_start(void *arg1) 505 { 506 struct hello_context_t *ctx = arg1; 507 int rc; 508 509 SPDK_NOTICELOG("Successfully started the application\n"); 510 511 if (ctx->is_server) { 512 rc = hello_sock_listen(ctx); 513 } else { 514 rc = hello_sock_connect(ctx); 515 } 516 517 if (rc) { 518 spdk_app_stop(-1); 519 return; 520 } 521 } 522 523 int 524 main(int argc, char **argv) 525 { 526 struct spdk_app_opts opts = {}; 527 int rc = 0; 528 struct hello_context_t hello_context = {}; 529 530 /* Set default values in opts structure. */ 531 spdk_app_opts_init(&opts, sizeof(opts)); 532 opts.name = "hello_sock"; 533 opts.shutdown_cb = hello_sock_shutdown_cb; 534 535 if ((rc = spdk_app_parse_args(argc, argv, &opts, "E:H:I:kKN:P:ST:VzZ", NULL, hello_sock_parse_arg, 536 hello_sock_usage)) != SPDK_APP_PARSE_ARGS_SUCCESS) { 537 exit(rc); 538 } 539 hello_context.is_server = g_is_server; 540 hello_context.host = g_host; 541 hello_context.sock_impl_name = g_sock_impl_name; 542 hello_context.port = g_port; 543 hello_context.zcopy = g_zcopy; 544 hello_context.ktls = g_ktls; 545 hello_context.tls_version = g_tls_version; 546 hello_context.psk_key = g_psk_key; 547 hello_context.psk_identity = g_psk_identity; 548 hello_context.verbose = g_verbose; 549 550 if (hello_context.sock_impl_name == NULL) { 551 hello_context.sock_impl_name = spdk_sock_get_default_impl(); 552 553 if (hello_context.sock_impl_name == NULL) { 554 SPDK_ERRLOG("No sock implementations available!\n"); 555 exit(-1); 556 } 557 } 558 559 if (hello_context.is_server) { 560 struct spdk_sock_impl_opts impl_opts = {}; 561 size_t len = sizeof(impl_opts); 562 563 rc = spdk_sock_impl_get_opts(hello_context.sock_impl_name, &impl_opts, &len); 564 if (rc < 0) { 565 exit(rc); 566 } 567 568 /* Our applications will post buffers to be used for receiving. That feature 569 * is mutually exclusive with the recv pipe, so we need to disable it. */ 570 impl_opts.enable_recv_pipe = false; 571 spdk_sock_impl_set_opts(hello_context.sock_impl_name, &impl_opts, len); 572 } 573 574 rc = spdk_app_start(&opts, hello_start, &hello_context); 575 if (rc) { 576 SPDK_ERRLOG("ERROR starting application\n"); 577 } 578 579 SPDK_NOTICELOG("Exiting from application\n"); 580 581 if (hello_context.verbose) { 582 printf("** %d bytes received, %d bytes sent **\n", 583 hello_context.bytes_in, hello_context.bytes_out); 584 } 585 586 /* Gracefully close out all of the SPDK subsystems. */ 587 spdk_app_fini(); 588 return rc; 589 } 590