1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) 2024 NVIDIA CORPORATION & AFFILIATES. All rights reserved. 3 */ 4 5 #include "spdk/stdinc.h" 6 #include "spdk/thread.h" 7 #include "spdk/fsdev.h" 8 #include "spdk/env.h" 9 #include "spdk/event.h" 10 #include "spdk/log.h" 11 #include "spdk/string.h" 12 13 #define TEST_FILENAME "hello_file" 14 #define DATA_SIZE 512 15 #define ROOT_NODEID 1 16 17 static char *g_fsdev_name = "Fs0"; 18 int g_result = 0; 19 20 /* 21 * We'll use this struct to gather housekeeping hello_context to pass between 22 * our events and callbacks. 23 */ 24 struct hello_context_t { 25 struct spdk_thread *app_thread; 26 struct spdk_fsdev_desc *fsdev_desc; 27 struct spdk_io_channel *fsdev_io_channel; 28 struct spdk_fsdev_file_object *root_fobject; 29 char *fsdev_name; 30 int thread_count; 31 }; 32 33 struct hello_thread_t { 34 struct hello_context_t *hello_context; 35 struct spdk_thread *thread; 36 struct spdk_io_channel *fsdev_io_channel; 37 uint64_t unique; 38 uint8_t *buf; 39 char *file_name; 40 struct spdk_fsdev_file_object *fobject; 41 struct spdk_fsdev_file_handle *fhandle; 42 struct iovec iov[2]; 43 }; 44 45 /* 46 * Usage function for printing parameters that are specific to this application 47 */ 48 static void 49 hello_fsdev_usage(void) 50 { 51 printf(" -f <fs> name of the fsdev to use\n"); 52 } 53 54 /* 55 * This function is called to parse the parameters that are specific to this application 56 */ 57 static int 58 hello_fsdev_parse_arg(int ch, char *arg) 59 { 60 switch (ch) { 61 case 'f': 62 g_fsdev_name = arg; 63 break; 64 default: 65 return -EINVAL; 66 } 67 return 0; 68 } 69 70 static void 71 hello_app_done(struct hello_context_t *hello_context, int rc) 72 { 73 spdk_put_io_channel(hello_context->fsdev_io_channel); 74 spdk_fsdev_close(hello_context->fsdev_desc); 75 SPDK_NOTICELOG("Stopping app: rc %d\n", rc); 76 spdk_app_stop(rc); 77 } 78 79 static void 80 root_forget_complete(void *cb_arg, struct spdk_io_channel *ch, int status) 81 { 82 struct hello_context_t *hello_context = cb_arg; 83 84 SPDK_NOTICELOG("Root forget complete (status=%d)\n", status); 85 if (status) { 86 SPDK_ERRLOG("Root forget failed: error %d\n", status); 87 g_result = EINVAL; 88 } 89 90 hello_app_done(hello_context, g_result); 91 } 92 93 static void 94 hello_root_release(struct hello_context_t *hello_context) 95 { 96 int res; 97 98 SPDK_NOTICELOG("Forget root\n"); 99 res = spdk_fsdev_forget(hello_context->fsdev_desc, hello_context->fsdev_io_channel, 0, 100 hello_context->root_fobject, 1, 101 root_forget_complete, hello_context); 102 if (res) { 103 SPDK_ERRLOG("Failed to forget root (err=%d)\n", res); 104 hello_app_done(hello_context, EINVAL); 105 } 106 } 107 108 static void 109 hello_app_notify_thread_done(void *ctx) 110 { 111 struct hello_context_t *hello_context = (struct hello_context_t *)ctx; 112 113 assert(hello_context->thread_count > 0); 114 hello_context->thread_count--; 115 if (hello_context->thread_count == 0) { 116 hello_root_release(hello_context); 117 } 118 } 119 120 static void 121 hello_thread_done(struct hello_thread_t *hello_thread, int rc) 122 { 123 struct hello_context_t *hello_context = hello_thread->hello_context; 124 125 spdk_put_io_channel(hello_thread->fsdev_io_channel); 126 free(hello_thread->buf); 127 free(hello_thread->file_name); 128 SPDK_NOTICELOG("Thread %s done: rc %d\n", 129 spdk_thread_get_name(hello_thread->thread), rc); 130 spdk_thread_exit(hello_thread->thread); 131 free(hello_thread); 132 if (rc) { 133 g_result = rc; 134 } 135 136 spdk_thread_send_msg(hello_context->app_thread, hello_app_notify_thread_done, hello_context); 137 } 138 139 static bool 140 hello_check_complete(struct hello_thread_t *hello_thread, int status, const char *op) 141 { 142 hello_thread->unique++; 143 if (status) { 144 SPDK_ERRLOG("%s failed with %d\n", op, status); 145 hello_thread_done(hello_thread, EIO); 146 return false; 147 } 148 149 return true; 150 } 151 152 static void 153 unlink_complete(void *cb_arg, struct spdk_io_channel *ch, int status) 154 { 155 struct hello_thread_t *hello_thread = cb_arg; 156 157 SPDK_NOTICELOG("Unlink complete (status=%d)\n", status); 158 if (!hello_check_complete(hello_thread, status, "unlink")) { 159 return; 160 } 161 162 hello_thread->fobject = NULL; 163 hello_thread_done(hello_thread, 0); 164 } 165 166 static void 167 hello_unlink(struct hello_thread_t *hello_thread) 168 { 169 struct hello_context_t *hello_context = hello_thread->hello_context; 170 int res; 171 172 SPDK_NOTICELOG("Unlink file %s\n", hello_thread->file_name); 173 174 res = spdk_fsdev_unlink(hello_context->fsdev_desc, hello_thread->fsdev_io_channel, 175 hello_thread->unique, hello_context->root_fobject, hello_thread->file_name, 176 unlink_complete, hello_thread); 177 if (res) { 178 SPDK_ERRLOG("unlink failed with %d\n", res); 179 hello_thread_done(hello_thread, EIO); 180 } 181 } 182 183 static void 184 release_complete(void *cb_arg, struct spdk_io_channel *ch, int status) 185 { 186 struct hello_thread_t *hello_thread = cb_arg; 187 188 SPDK_NOTICELOG("Release complete (status=%d)\n", status); 189 if (!hello_check_complete(hello_thread, status, "release")) { 190 return; 191 } 192 193 hello_thread->fhandle = NULL; 194 hello_unlink(hello_thread); 195 } 196 197 static void 198 hello_release(struct hello_thread_t *hello_thread) 199 { 200 struct hello_context_t *hello_context = hello_thread->hello_context; 201 int res; 202 203 SPDK_NOTICELOG("Release file handle %p\n", hello_thread->fhandle); 204 205 res = spdk_fsdev_release(hello_context->fsdev_desc, hello_thread->fsdev_io_channel, 206 hello_thread->unique, hello_thread->fobject, hello_thread->fhandle, 207 release_complete, hello_thread); 208 if (res) { 209 SPDK_ERRLOG("release failed with %d\n", res); 210 hello_thread_done(hello_thread, EIO); 211 } 212 } 213 214 static void 215 read_complete(void *cb_arg, struct spdk_io_channel *ch, int status, uint32_t data_size) 216 { 217 struct hello_thread_t *hello_thread = cb_arg; 218 uint8_t data = spdk_env_get_current_core(); 219 uint32_t i; 220 221 SPDK_NOTICELOG("Read complete (status=%d, %" PRIu32 "bytes read)\n", status, data_size); 222 if (!hello_check_complete(hello_thread, status, "read")) { 223 return; 224 } 225 226 assert(data_size == DATA_SIZE); 227 228 for (i = 0; i < DATA_SIZE; ++i) { 229 if (hello_thread->buf[i] != data) { 230 SPDK_NOTICELOG("Bad read data at offset %d, 0x%02X != 0x%02X\n", 231 i, hello_thread->buf[i], data); 232 break; 233 } 234 } 235 236 hello_release(hello_thread); 237 } 238 239 static void 240 hello_read(struct hello_thread_t *hello_thread) 241 { 242 struct hello_context_t *hello_context = hello_thread->hello_context; 243 int res; 244 245 SPDK_NOTICELOG("Read from file handle %p\n", hello_thread->fhandle); 246 247 memset(hello_thread->buf, 0xFF, DATA_SIZE); 248 249 hello_thread->iov[0].iov_base = hello_thread->buf; 250 hello_thread->iov[0].iov_len = DATA_SIZE / 4; 251 hello_thread->iov[1].iov_base = hello_thread->buf + hello_thread->iov[0].iov_len; 252 hello_thread->iov[1].iov_len = DATA_SIZE - hello_thread->iov[0].iov_len; 253 254 res = spdk_fsdev_read(hello_context->fsdev_desc, hello_thread->fsdev_io_channel, 255 hello_thread->unique, hello_thread->fobject, hello_thread->fhandle, 256 DATA_SIZE, 0, 0, hello_thread->iov, 2, NULL, 257 read_complete, hello_thread); 258 if (res) { 259 SPDK_ERRLOG("write failed with %d\n", res); 260 hello_thread_done(hello_thread, EIO); 261 } 262 } 263 264 static void 265 write_complete(void *cb_arg, struct spdk_io_channel *ch, int status, uint32_t data_size) 266 { 267 struct hello_thread_t *hello_thread = cb_arg; 268 269 SPDK_NOTICELOG("Write complete (status=%d, %" PRIu32 "bytes written)\n", status, data_size); 270 if (!hello_check_complete(hello_thread, status, "write")) { 271 return; 272 } 273 274 assert(data_size == DATA_SIZE); 275 hello_read(hello_thread); 276 } 277 278 static void 279 hello_write(struct hello_thread_t *hello_thread) 280 { 281 uint8_t data = spdk_env_get_current_core(); 282 struct hello_context_t *hello_context = hello_thread->hello_context; 283 int res; 284 285 SPDK_NOTICELOG("Write to file handle %p\n", hello_thread->fhandle); 286 287 memset(hello_thread->buf, data, DATA_SIZE); 288 289 hello_thread->iov[0].iov_base = hello_thread->buf; 290 hello_thread->iov[0].iov_len = DATA_SIZE / 2; 291 hello_thread->iov[1].iov_base = hello_thread->buf + hello_thread->iov[0].iov_len; 292 hello_thread->iov[1].iov_len = DATA_SIZE - hello_thread->iov[0].iov_len; 293 294 res = spdk_fsdev_write(hello_context->fsdev_desc, hello_thread->fsdev_io_channel, 295 hello_thread->unique, hello_thread->fobject, hello_thread->fhandle, 296 DATA_SIZE, 0, 0, hello_thread->iov, 2, NULL, 297 write_complete, hello_thread); 298 if (res) { 299 SPDK_ERRLOG("write failed with %d\n", res); 300 hello_thread_done(hello_thread, EIO); 301 } 302 } 303 304 static void 305 fopen_complete(void *cb_arg, struct spdk_io_channel *ch, int status, 306 struct spdk_fsdev_file_handle *fhandle) 307 { 308 struct hello_thread_t *hello_thread = cb_arg; 309 310 SPDK_NOTICELOG("Open complete (status=%d)\n", status); 311 if (!hello_check_complete(hello_thread, status, "open")) { 312 return; 313 } 314 315 hello_thread->fhandle = fhandle; 316 hello_write(hello_thread); 317 } 318 319 static void 320 hello_open(struct hello_thread_t *hello_thread) 321 { 322 struct hello_context_t *hello_context = hello_thread->hello_context; 323 int res; 324 325 SPDK_NOTICELOG("Open fobject %p\n", hello_thread->fobject); 326 327 res = spdk_fsdev_fopen(hello_context->fsdev_desc, hello_thread->fsdev_io_channel, 328 hello_thread->unique, hello_thread->fobject, O_RDWR, 329 fopen_complete, hello_thread); 330 if (res) { 331 SPDK_ERRLOG("open failed with %d\n", res); 332 hello_thread_done(hello_thread, EIO); 333 } 334 } 335 336 static void 337 lookup_complete(void *cb_arg, struct spdk_io_channel *ch, int status, 338 struct spdk_fsdev_file_object *fobject, const struct spdk_fsdev_file_attr *attr) 339 { 340 struct hello_thread_t *hello_thread = cb_arg; 341 342 SPDK_NOTICELOG("Lookup complete (status=%d)\n", status); 343 if (!hello_check_complete(hello_thread, status, "lookup")) { 344 return; 345 } 346 347 assert(hello_thread->fobject == fobject); 348 hello_open(hello_thread); 349 } 350 351 static void 352 hello_lookup(struct hello_thread_t *hello_thread) 353 { 354 struct hello_context_t *hello_context = hello_thread->hello_context; 355 int res; 356 357 SPDK_NOTICELOG("Lookup file %s\n", hello_thread->file_name); 358 359 res = spdk_fsdev_lookup(hello_context->fsdev_desc, hello_thread->fsdev_io_channel, 360 hello_thread->unique, hello_context->root_fobject, hello_thread->file_name, 361 lookup_complete, hello_thread); 362 if (res) { 363 SPDK_ERRLOG("lookup failed with %d\n", res); 364 hello_thread_done(hello_thread, EIO); 365 } 366 } 367 368 static void 369 mknod_complete(void *cb_arg, struct spdk_io_channel *ch, int status, 370 struct spdk_fsdev_file_object *fobject, const struct spdk_fsdev_file_attr *attr) 371 { 372 struct hello_thread_t *hello_thread = cb_arg; 373 374 SPDK_NOTICELOG("Mknod complete (status=%d)\n", status); 375 if (!hello_check_complete(hello_thread, status, "mknod")) { 376 return; 377 } 378 379 hello_thread->fobject = fobject; 380 hello_lookup(hello_thread); 381 } 382 383 static void 384 hello_mknod(void *ctx) 385 { 386 struct hello_thread_t *hello_thread = (struct hello_thread_t *)ctx; 387 struct hello_context_t *hello_context = hello_thread->hello_context; 388 int res; 389 390 SPDK_NOTICELOG("Mknod file %s\n", hello_thread->file_name); 391 392 res = spdk_fsdev_mknod(hello_context->fsdev_desc, hello_thread->fsdev_io_channel, 393 hello_thread->unique, hello_context->root_fobject, hello_thread->file_name, 394 S_IFREG | S_IRWXU | S_IRWXG | S_IRWXO, 0, 0, 0, mknod_complete, hello_thread); 395 if (res) { 396 SPDK_ERRLOG("mknod failed with %d\n", res); 397 hello_thread_done(hello_thread, EIO); 398 } 399 } 400 401 static void 402 hello_start_thread(void *ctx) 403 { 404 struct hello_context_t *hello_context = (struct hello_context_t *)ctx; 405 struct hello_thread_t *hello_thread; 406 /* File name size assumes that core number will fit into 3 characters */ 407 const int filename_size = strlen(TEST_FILENAME) + 5; 408 409 hello_thread = calloc(1, sizeof(struct hello_thread_t)); 410 if (!hello_thread) { 411 SPDK_ERRLOG("Failed to allocate thread context\n"); 412 spdk_thread_send_msg(hello_context->app_thread, hello_app_notify_thread_done, hello_context); 413 return; 414 } 415 416 hello_thread->hello_context = hello_context; 417 hello_thread->thread = spdk_get_thread(); 418 hello_thread->unique = 1; 419 hello_thread->buf = (char *)malloc(DATA_SIZE); 420 if (!hello_thread->buf) { 421 SPDK_ERRLOG("Could not allocate data buffer\n"); 422 hello_thread_done(hello_thread, ENOMEM); 423 return; 424 } 425 426 hello_thread->file_name = (char *)malloc(filename_size); 427 if (!hello_thread->file_name) { 428 SPDK_ERRLOG("Could not allocate file name buffer\n"); 429 hello_thread_done(hello_thread, ENOMEM); 430 return; 431 } 432 433 if (snprintf(hello_thread->file_name, filename_size, "%s_%u", 434 TEST_FILENAME, spdk_env_get_current_core()) >= filename_size) { 435 SPDK_ERRLOG("File name size doesn't fit into buffer\n"); 436 hello_thread_done(hello_thread, ENOMEM); 437 return; 438 } 439 440 hello_thread->fsdev_io_channel = spdk_fsdev_get_io_channel(hello_thread->hello_context->fsdev_desc); 441 if (!hello_thread->fsdev_io_channel) { 442 SPDK_ERRLOG("Could not create fsdev I/O channel!\n"); 443 hello_thread_done(hello_thread, ENOMEM); 444 return; 445 } 446 447 SPDK_NOTICELOG("Started thread %s on core %u\n", 448 spdk_thread_get_name(hello_thread->thread), 449 spdk_env_get_current_core()); 450 spdk_thread_send_msg(hello_thread->thread, hello_mknod, hello_thread); 451 } 452 453 static void 454 hello_create_threads(struct hello_context_t *hello_context) 455 { 456 uint32_t cpu; 457 char thread_name[32]; 458 struct spdk_cpuset mask = {}; 459 struct spdk_thread *thread; 460 461 SPDK_ENV_FOREACH_CORE(cpu) { 462 snprintf(thread_name, sizeof(thread_name), "hello_fsdev_%u", cpu); 463 spdk_cpuset_zero(&mask); 464 spdk_cpuset_set_cpu(&mask, cpu, true); 465 thread = spdk_thread_create(thread_name, &mask); 466 assert(thread != NULL); 467 hello_context->thread_count++; 468 spdk_thread_send_msg(thread, hello_start_thread, hello_context); 469 } 470 } 471 472 473 static void 474 root_lookup_complete(void *cb_arg, struct spdk_io_channel *ch, int status, 475 struct spdk_fsdev_file_object *fobject, const struct spdk_fsdev_file_attr *attr) 476 { 477 struct hello_context_t *hello_context = cb_arg; 478 479 SPDK_NOTICELOG("Root lookup complete (status=%d)\n", status); 480 if (status) { 481 SPDK_ERRLOG("Fuse init failed: error %d\n", status); 482 hello_app_done(hello_context, status); 483 return; 484 } 485 486 hello_context->root_fobject = fobject; 487 488 hello_create_threads(hello_context); 489 } 490 491 static void 492 root_lookup(struct hello_context_t *hello_context) 493 { 494 int res; 495 496 SPDK_NOTICELOG("Lookup for the root\n"); 497 498 res = spdk_fsdev_lookup(hello_context->fsdev_desc, hello_context->fsdev_io_channel, 0, 499 NULL /* root */, "" /* will be ignored */, root_lookup_complete, hello_context); 500 if (res) { 501 SPDK_ERRLOG("Failed to initiate lookup for the root (err=%d)\n", res); 502 hello_app_done(hello_context, res); 503 return; 504 } 505 } 506 507 static void 508 hello_fsdev_event_cb(enum spdk_fsdev_event_type type, struct spdk_fsdev *fsdev, void *event_ctx) 509 { 510 SPDK_NOTICELOG("Unsupported fsdev event: type %d\n", type); 511 } 512 513 /* 514 * Our initial event that kicks off everything from main(). 515 */ 516 static void 517 hello_start(void *arg1) 518 { 519 struct hello_context_t *hello_context = arg1; 520 int rc = 0; 521 hello_context->fsdev_desc = NULL; 522 523 SPDK_NOTICELOG("Successfully started the application\n"); 524 525 hello_context->app_thread = spdk_get_thread(); 526 527 /* 528 * There can be many fsdevs configured, but this application will only use 529 * the one input by the user at runtime. 530 * 531 * Open the fs by calling spdk_fsdev_open() with its name. 532 * The function will return a descriptor 533 */ 534 SPDK_NOTICELOG("Opening the fsdev %s\n", hello_context->fsdev_name); 535 rc = spdk_fsdev_open(hello_context->fsdev_name, 536 hello_fsdev_event_cb, NULL, NULL, 537 &hello_context->fsdev_desc); 538 if (rc) { 539 SPDK_ERRLOG("Could not open fsdev: %s\n", hello_context->fsdev_name); 540 spdk_app_stop(-1); 541 return; 542 } 543 544 SPDK_NOTICELOG("Opening io channel\n"); 545 /* Open I/O channel */ 546 hello_context->fsdev_io_channel = spdk_fsdev_get_io_channel(hello_context->fsdev_desc); 547 if (!hello_context->fsdev_io_channel) { 548 SPDK_ERRLOG("Could not create fsdev I/O channel!\n"); 549 spdk_fsdev_close(hello_context->fsdev_desc); 550 spdk_app_stop(-1); 551 return; 552 } 553 554 root_lookup(hello_context); 555 } 556 557 int 558 main(int argc, char **argv) 559 { 560 struct spdk_app_opts opts = {}; 561 int rc = 0; 562 struct hello_context_t hello_context = {}; 563 564 /* Set default values in opts structure. */ 565 spdk_app_opts_init(&opts, sizeof(opts)); 566 opts.name = "hello_fsdev"; 567 568 /* 569 * Parse built-in SPDK command line parameters as well 570 * as our custom one(s). 571 */ 572 if ((rc = spdk_app_parse_args(argc, argv, &opts, "f:", NULL, hello_fsdev_parse_arg, 573 hello_fsdev_usage)) != SPDK_APP_PARSE_ARGS_SUCCESS) { 574 exit(rc); 575 } 576 hello_context.fsdev_name = g_fsdev_name; 577 578 /* 579 * spdk_app_start() will initialize the SPDK framework, call hello_start(), 580 * and then block until spdk_app_stop() is called (or if an initialization 581 * error occurs, spdk_app_start() will return with rc even without calling 582 * hello_start(). 583 */ 584 rc = spdk_app_start(&opts, hello_start, &hello_context); 585 if (rc) { 586 SPDK_ERRLOG("ERROR starting application\n"); 587 } 588 589 /* At this point either spdk_app_stop() was called, or spdk_app_start() 590 * failed because of internal error. 591 */ 592 593 /* Gracefully close out all of the SPDK subsystems. */ 594 spdk_app_fini(); 595 return rc; 596 } 597