1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2018 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 #include "spdk/thread.h" 8 #include "spdk/env.h" 9 #include "spdk/event.h" 10 #include "spdk/log.h" 11 #include "spdk/string.h" 12 13 #include "spdk/sock.h" 14 #include "spdk/hexlify.h" 15 #include "spdk/nvmf.h" 16 17 #define ACCEPT_TIMEOUT_US 1000 18 #define CLOSE_TIMEOUT_US 1000000 19 #define BUFFER_SIZE 1024 20 #define ADDR_STR_LEN INET6_ADDRSTRLEN 21 22 static bool g_is_running; 23 24 static char *g_host; 25 static char *g_sock_impl_name; 26 static int g_port; 27 static bool g_is_server; 28 static int g_zcopy; 29 static int g_ktls; 30 static int g_tls_version; 31 static bool g_verbose; 32 static char *g_psk_key; 33 static char *g_psk_identity; 34 35 /* 36 * We'll use this struct to gather housekeeping hello_context to pass between 37 * our events and callbacks. 38 */ 39 struct hello_context_t { 40 bool is_server; 41 char *host; 42 const char *sock_impl_name; 43 int port; 44 int zcopy; 45 int ktls; 46 int tls_version; 47 char *psk_key; 48 char *psk_identity; 49 50 bool verbose; 51 int bytes_in; 52 int bytes_out; 53 54 struct spdk_sock *sock; 55 56 struct spdk_sock_group *group; 57 void *buf; 58 struct spdk_poller *poller_in; 59 struct spdk_poller *poller_out; 60 struct spdk_poller *time_out; 61 62 int rc; 63 }; 64 65 /* 66 * Usage function for printing parameters that are specific to this application 67 */ 68 static void 69 hello_sock_usage(void) 70 { 71 printf(" -E psk_key Default PSK KEY in hexadecimal digits, e.g. 1234567890ABCDEF (only applies when sock_impl == ssl)\n"); 72 printf(" -H host_addr host address\n"); 73 printf(" -I psk_id Default PSK ID, e.g. psk.spdk.io (only applies when sock_impl == ssl)\n"); 74 printf(" -P port port number\n"); 75 printf(" -N sock_impl socket implementation, e.g., -N posix or -N uring\n"); 76 printf(" -S start in server mode\n"); 77 printf(" -T tls_ver TLS version, e.g., -T 12 or -T 13. If omitted, auto-negotiation will take place\n"); 78 printf(" -k disable KTLS for the given sock implementation (default)\n"); 79 printf(" -K enable KTLS for the given sock implementation\n"); 80 printf(" -V print out additional information\n"); 81 printf(" -z disable zero copy send for the given sock implementation\n"); 82 printf(" -Z enable zero copy send for the given sock implementation\n"); 83 } 84 85 /* 86 * This function is called to parse the parameters that are specific to this application 87 */ 88 static int 89 hello_sock_parse_arg(int ch, char *arg) 90 { 91 switch (ch) { 92 case 'E': 93 g_psk_key = arg; 94 break; 95 case 'H': 96 g_host = arg; 97 break; 98 case 'I': 99 g_psk_identity = arg; 100 break; 101 case 'N': 102 g_sock_impl_name = arg; 103 break; 104 case 'P': 105 g_port = spdk_strtol(arg, 10); 106 if (g_port < 0) { 107 fprintf(stderr, "Invalid port ID\n"); 108 return g_port; 109 } 110 break; 111 case 'S': 112 g_is_server = 1; 113 break; 114 case 'K': 115 g_ktls = 1; 116 break; 117 case 'k': 118 g_ktls = 0; 119 break; 120 case 'T': 121 g_tls_version = spdk_strtol(arg, 10); 122 if (g_tls_version < 0) { 123 fprintf(stderr, "Invalid TLS version\n"); 124 return g_tls_version; 125 } 126 break; 127 case 'V': 128 g_verbose = true; 129 break; 130 case 'Z': 131 g_zcopy = 1; 132 break; 133 case 'z': 134 g_zcopy = 0; 135 break; 136 default: 137 return -EINVAL; 138 } 139 return 0; 140 } 141 142 static int 143 hello_sock_close_timeout_poll(void *arg) 144 { 145 struct hello_context_t *ctx = arg; 146 SPDK_NOTICELOG("Connection closed\n"); 147 148 free(ctx->buf); 149 150 spdk_poller_unregister(&ctx->time_out); 151 spdk_poller_unregister(&ctx->poller_in); 152 spdk_sock_close(&ctx->sock); 153 spdk_sock_group_close(&ctx->group); 154 155 spdk_app_stop(ctx->rc); 156 return SPDK_POLLER_BUSY; 157 } 158 159 static int 160 hello_sock_quit(struct hello_context_t *ctx, int rc) 161 { 162 ctx->rc = rc; 163 spdk_poller_unregister(&ctx->poller_out); 164 if (!ctx->time_out) { 165 ctx->time_out = SPDK_POLLER_REGISTER(hello_sock_close_timeout_poll, ctx, 166 CLOSE_TIMEOUT_US); 167 } 168 return 0; 169 } 170 171 static int 172 hello_sock_recv_poll(void *arg) 173 { 174 struct hello_context_t *ctx = arg; 175 int rc; 176 char buf_in[BUFFER_SIZE]; 177 178 /* 179 * Get response 180 */ 181 rc = spdk_sock_recv(ctx->sock, buf_in, sizeof(buf_in) - 1); 182 183 if (rc <= 0) { 184 if (errno == EAGAIN || errno == EWOULDBLOCK) { 185 return SPDK_POLLER_IDLE; 186 } 187 188 SPDK_ERRLOG("spdk_sock_recv() failed, errno %d: %s\n", 189 errno, spdk_strerror(errno)); 190 return SPDK_POLLER_BUSY; 191 } 192 193 if (rc > 0) { 194 ctx->bytes_in += rc; 195 buf_in[rc] = '\0'; 196 printf("%s", buf_in); 197 } 198 199 return SPDK_POLLER_BUSY; 200 } 201 202 static int 203 hello_sock_writev_poll(void *arg) 204 { 205 struct hello_context_t *ctx = arg; 206 int rc = 0; 207 char buf_out[BUFFER_SIZE]; 208 struct iovec iov; 209 ssize_t n; 210 211 n = read(STDIN_FILENO, buf_out, sizeof(buf_out)); 212 if (n == 0 || !g_is_running) { 213 /* EOF */ 214 SPDK_NOTICELOG("Closing connection...\n"); 215 hello_sock_quit(ctx, 0); 216 return SPDK_POLLER_IDLE; 217 } 218 if (n > 0) { 219 /* 220 * Send message to the server 221 */ 222 iov.iov_base = buf_out; 223 iov.iov_len = n; 224 rc = spdk_sock_writev(ctx->sock, &iov, 1); 225 if (rc > 0) { 226 ctx->bytes_out += rc; 227 } 228 } 229 return rc > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 230 } 231 232 static int 233 hello_sock_connect(struct hello_context_t *ctx) 234 { 235 int rc; 236 char saddr[ADDR_STR_LEN], caddr[ADDR_STR_LEN]; 237 uint16_t cport, sport; 238 struct spdk_sock_impl_opts impl_opts; 239 size_t impl_opts_size = sizeof(impl_opts); 240 struct spdk_sock_opts opts; 241 char psk[SPDK_TLS_PSK_MAX_LEN] = {}; 242 char *unhexlified; 243 244 spdk_sock_impl_get_opts(ctx->sock_impl_name, &impl_opts, &impl_opts_size); 245 impl_opts.enable_ktls = ctx->ktls; 246 impl_opts.tls_version = ctx->tls_version; 247 impl_opts.psk_identity = ctx->psk_identity; 248 impl_opts.tls_cipher_suites = "TLS_AES_128_GCM_SHA256"; 249 250 opts.opts_size = sizeof(opts); 251 spdk_sock_get_default_opts(&opts); 252 opts.zcopy = ctx->zcopy; 253 opts.impl_opts = &impl_opts; 254 opts.impl_opts_size = sizeof(impl_opts); 255 256 if (ctx->psk_key) { 257 impl_opts.psk_key_size = strlen(ctx->psk_key) / 2; 258 if (impl_opts.psk_key_size > SPDK_TLS_PSK_MAX_LEN) { 259 SPDK_ERRLOG("Insufficient buffer size for PSK"); 260 return -EINVAL; 261 } 262 unhexlified = spdk_unhexlify(ctx->psk_key); 263 if (unhexlified == NULL) { 264 SPDK_ERRLOG("Could not unhexlify PSK"); 265 return -EINVAL; 266 } 267 memcpy(psk, unhexlified, impl_opts.psk_key_size); 268 free(unhexlified); 269 impl_opts.psk_key = psk; 270 } 271 272 SPDK_NOTICELOG("Connecting to the server on %s:%d with sock_impl(%s)\n", ctx->host, ctx->port, 273 ctx->sock_impl_name); 274 275 ctx->sock = spdk_sock_connect_ext(ctx->host, ctx->port, ctx->sock_impl_name, &opts); 276 if (ctx->sock == NULL) { 277 SPDK_ERRLOG("connect error(%d): %s\n", errno, spdk_strerror(errno)); 278 return -1; 279 } 280 281 rc = spdk_sock_getaddr(ctx->sock, saddr, sizeof(saddr), &sport, caddr, sizeof(caddr), &cport); 282 if (rc < 0) { 283 SPDK_ERRLOG("Cannot get connection addresses\n"); 284 goto err; 285 } 286 287 SPDK_NOTICELOG("Connection accepted from (%s, %hu) to (%s, %hu)\n", caddr, cport, saddr, sport); 288 289 rc = fcntl(STDIN_FILENO, F_GETFL); 290 if (rc == -1) { 291 SPDK_ERRLOG("Getting file status flag failed: %s\n", strerror(errno)); 292 goto err; 293 } 294 295 if (fcntl(STDIN_FILENO, F_SETFL, rc | O_NONBLOCK) == -1) { 296 SPDK_ERRLOG("Setting file status flag failed: %s\n", strerror(errno)); 297 goto err; 298 } 299 300 g_is_running = true; 301 ctx->poller_in = SPDK_POLLER_REGISTER(hello_sock_recv_poll, ctx, 0); 302 ctx->poller_out = SPDK_POLLER_REGISTER(hello_sock_writev_poll, ctx, 0); 303 304 return 0; 305 err: 306 spdk_sock_close(&ctx->sock); 307 return -1; 308 } 309 310 static void 311 hello_sock_cb(void *arg, struct spdk_sock_group *group, struct spdk_sock *sock) 312 { 313 int rc; 314 struct hello_context_t *ctx = arg; 315 struct iovec iov = {}; 316 ssize_t n; 317 void *user_ctx; 318 319 rc = spdk_sock_recv_next(sock, &iov.iov_base, &user_ctx); 320 if (rc < 0) { 321 if (errno == EAGAIN || errno == EWOULDBLOCK) { 322 return; 323 } 324 325 if (errno != ENOTCONN && errno != ECONNRESET) { 326 SPDK_ERRLOG("spdk_sock_recv_zcopy() failed, errno %d: %s\n", 327 errno, spdk_strerror(errno)); 328 } 329 } 330 331 if (rc > 0) { 332 iov.iov_len = rc; 333 ctx->bytes_in += iov.iov_len; 334 n = spdk_sock_writev(sock, &iov, 1); 335 if (n > 0) { 336 assert(n == rc); 337 ctx->bytes_out += n; 338 } 339 340 spdk_sock_group_provide_buf(ctx->group, iov.iov_base, BUFFER_SIZE, NULL); 341 return; 342 } 343 344 /* Connection closed */ 345 SPDK_NOTICELOG("Connection closed\n"); 346 spdk_sock_group_remove_sock(group, sock); 347 spdk_sock_close(&sock); 348 } 349 350 static int 351 hello_sock_accept_poll(void *arg) 352 { 353 struct hello_context_t *ctx = arg; 354 struct spdk_sock *sock; 355 int rc; 356 int count = 0; 357 char saddr[ADDR_STR_LEN], caddr[ADDR_STR_LEN]; 358 uint16_t cport, sport; 359 360 if (!g_is_running) { 361 hello_sock_quit(ctx, 0); 362 return SPDK_POLLER_IDLE; 363 } 364 365 while (1) { 366 sock = spdk_sock_accept(ctx->sock); 367 if (sock != NULL) { 368 rc = spdk_sock_getaddr(sock, saddr, sizeof(saddr), &sport, caddr, sizeof(caddr), &cport); 369 if (rc < 0) { 370 SPDK_ERRLOG("Cannot get connection addresses\n"); 371 spdk_sock_close(&sock); 372 return SPDK_POLLER_IDLE; 373 } 374 375 SPDK_NOTICELOG("Accepting a new connection from (%s, %hu) to (%s, %hu)\n", 376 caddr, cport, saddr, sport); 377 378 rc = spdk_sock_group_add_sock(ctx->group, sock, 379 hello_sock_cb, ctx); 380 381 if (rc < 0) { 382 spdk_sock_close(&sock); 383 SPDK_ERRLOG("failed\n"); 384 break; 385 } 386 387 count++; 388 } else { 389 if (errno != EAGAIN && errno != EWOULDBLOCK) { 390 SPDK_ERRLOG("accept error(%d): %s\n", errno, spdk_strerror(errno)); 391 } 392 break; 393 } 394 } 395 396 return count > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 397 } 398 399 static int 400 hello_sock_group_poll(void *arg) 401 { 402 struct hello_context_t *ctx = arg; 403 int rc; 404 405 rc = spdk_sock_group_poll(ctx->group); 406 if (rc < 0) { 407 SPDK_ERRLOG("Failed to poll sock_group=%p\n", ctx->group); 408 } 409 410 return rc > 0 ? SPDK_POLLER_BUSY : SPDK_POLLER_IDLE; 411 } 412 413 static int 414 hello_sock_listen(struct hello_context_t *ctx) 415 { 416 struct spdk_sock_impl_opts impl_opts; 417 size_t impl_opts_size = sizeof(impl_opts); 418 struct spdk_sock_opts opts; 419 static char psk[SPDK_TLS_PSK_MAX_LEN] = {}; 420 char *unhexlified; 421 422 spdk_sock_impl_get_opts(ctx->sock_impl_name, &impl_opts, &impl_opts_size); 423 impl_opts.enable_ktls = ctx->ktls; 424 impl_opts.tls_version = ctx->tls_version; 425 impl_opts.psk_identity = ctx->psk_identity; 426 impl_opts.tls_cipher_suites = "TLS_AES_128_GCM_SHA256"; 427 428 opts.opts_size = sizeof(opts); 429 spdk_sock_get_default_opts(&opts); 430 opts.zcopy = ctx->zcopy; 431 opts.impl_opts = &impl_opts; 432 opts.impl_opts_size = sizeof(impl_opts); 433 434 if (ctx->psk_key) { 435 impl_opts.psk_key_size = strlen(ctx->psk_key) / 2; 436 if (impl_opts.psk_key_size > SPDK_TLS_PSK_MAX_LEN) { 437 SPDK_ERRLOG("Insufficient buffer size for PSK"); 438 return -EINVAL; 439 } 440 unhexlified = spdk_unhexlify(ctx->psk_key); 441 if (unhexlified == NULL) { 442 SPDK_ERRLOG("Could not unhexlify PSK"); 443 return -EINVAL; 444 } 445 memcpy(psk, unhexlified, impl_opts.psk_key_size); 446 free(unhexlified); 447 impl_opts.psk_key = psk; 448 } 449 450 ctx->sock = spdk_sock_listen_ext(ctx->host, ctx->port, ctx->sock_impl_name, &opts); 451 if (ctx->sock == NULL) { 452 SPDK_ERRLOG("Cannot create server socket\n"); 453 return -1; 454 } 455 456 SPDK_NOTICELOG("Listening connection on %s:%d with sock_impl(%s)\n", ctx->host, ctx->port, 457 ctx->sock_impl_name); 458 459 /* 460 * Create sock group for server socket 461 */ 462 ctx->group = spdk_sock_group_create(NULL); 463 if (ctx->group == NULL) { 464 SPDK_ERRLOG("Cannot create sock group\n"); 465 spdk_sock_close(&ctx->sock); 466 return -1; 467 } 468 469 /* 470 * Provide a buffer to the group to be used with receive. 471 */ 472 ctx->buf = calloc(1, BUFFER_SIZE); 473 if (ctx->buf == NULL) { 474 SPDK_ERRLOG("Cannot allocate memory for sock group\n"); 475 spdk_sock_close(&ctx->sock); 476 return -1; 477 } 478 479 spdk_sock_group_provide_buf(ctx->group, ctx->buf, BUFFER_SIZE, NULL); 480 481 g_is_running = true; 482 483 /* 484 * Start acceptor and group poller 485 */ 486 ctx->poller_in = SPDK_POLLER_REGISTER(hello_sock_accept_poll, ctx, 487 ACCEPT_TIMEOUT_US); 488 ctx->poller_out = SPDK_POLLER_REGISTER(hello_sock_group_poll, ctx, 0); 489 490 return 0; 491 } 492 493 static void 494 hello_sock_shutdown_cb(void) 495 { 496 g_is_running = false; 497 } 498 499 /* 500 * Our initial event that kicks off everything from main(). 501 */ 502 static void 503 hello_start(void *arg1) 504 { 505 struct hello_context_t *ctx = arg1; 506 int rc; 507 508 SPDK_NOTICELOG("Successfully started the application\n"); 509 510 if (ctx->is_server) { 511 rc = hello_sock_listen(ctx); 512 } else { 513 rc = hello_sock_connect(ctx); 514 } 515 516 if (rc) { 517 spdk_app_stop(-1); 518 return; 519 } 520 } 521 522 int 523 main(int argc, char **argv) 524 { 525 struct spdk_app_opts opts = {}; 526 int rc = 0; 527 struct hello_context_t hello_context = {}; 528 529 /* Set default values in opts structure. */ 530 spdk_app_opts_init(&opts, sizeof(opts)); 531 opts.name = "hello_sock"; 532 opts.shutdown_cb = hello_sock_shutdown_cb; 533 534 if ((rc = spdk_app_parse_args(argc, argv, &opts, "E:H:I:kKN:P:ST:VzZ", NULL, hello_sock_parse_arg, 535 hello_sock_usage)) != SPDK_APP_PARSE_ARGS_SUCCESS) { 536 exit(rc); 537 } 538 hello_context.is_server = g_is_server; 539 hello_context.host = g_host; 540 hello_context.sock_impl_name = g_sock_impl_name; 541 hello_context.port = g_port; 542 hello_context.zcopy = g_zcopy; 543 hello_context.ktls = g_ktls; 544 hello_context.tls_version = g_tls_version; 545 hello_context.psk_key = g_psk_key; 546 hello_context.psk_identity = g_psk_identity; 547 hello_context.verbose = g_verbose; 548 549 if (hello_context.sock_impl_name == NULL) { 550 hello_context.sock_impl_name = spdk_sock_get_default_impl(); 551 552 if (hello_context.sock_impl_name == NULL) { 553 SPDK_ERRLOG("No sock implementations available!\n"); 554 exit(-1); 555 } 556 } 557 558 if (hello_context.is_server) { 559 struct spdk_sock_impl_opts impl_opts = {}; 560 size_t len = sizeof(impl_opts); 561 562 rc = spdk_sock_impl_get_opts(hello_context.sock_impl_name, &impl_opts, &len); 563 if (rc < 0) { 564 exit(rc); 565 } 566 567 /* Our applications will post buffers to be used for receiving. That feature 568 * is mutually exclusive with the recv pipe, so we need to disable it. */ 569 impl_opts.enable_recv_pipe = false; 570 spdk_sock_impl_set_opts(hello_context.sock_impl_name, &impl_opts, len); 571 } 572 573 rc = spdk_app_start(&opts, hello_start, &hello_context); 574 if (rc) { 575 SPDK_ERRLOG("ERROR starting application\n"); 576 } 577 578 SPDK_NOTICELOG("Exiting from application\n"); 579 580 if (hello_context.verbose) { 581 printf("** %d bytes received, %d bytes sent **\n", 582 hello_context.bytes_in, hello_context.bytes_out); 583 } 584 585 /* Gracefully close out all of the SPDK subsystems. */ 586 spdk_app_fini(); 587 return rc; 588 } 589