1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include "spdk_internal/accel_module.h" 9 10 #include "accel_internal.h" 11 12 #include "spdk/env.h" 13 #include "spdk/likely.h" 14 #include "spdk/log.h" 15 #include "spdk/thread.h" 16 #include "spdk/json.h" 17 #include "spdk/crc32.h" 18 #include "spdk/util.h" 19 20 /* Accelerator Framework: The following provides a top level 21 * generic API for the accelerator functions defined here. Modules, 22 * such as the one in /module/accel/ioat, supply the implementation 23 * with the exception of the pure software implementation contained 24 * later in this file. 25 */ 26 27 #define ALIGN_4K 0x1000 28 #define MAX_TASKS_PER_CHANNEL 0x800 29 30 /* Largest context size for all accel modules */ 31 static size_t g_max_accel_module_size = sizeof(struct spdk_accel_task); 32 33 static struct spdk_accel_module_if *g_accel_module = NULL; 34 static spdk_accel_fini_cb g_fini_cb_fn = NULL; 35 static void *g_fini_cb_arg = NULL; 36 static bool g_modules_started = false; 37 38 /* Global list of registered accelerator modules */ 39 static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list = 40 TAILQ_HEAD_INITIALIZER(spdk_accel_module_list); 41 42 /* Global array mapping capabilities to modules */ 43 static struct spdk_accel_module_if *g_modules_opc[ACCEL_OPC_LAST] = {}; 44 static char *g_modules_opc_override[ACCEL_OPC_LAST] = {}; 45 46 struct accel_io_channel { 47 struct spdk_io_channel *module_ch[ACCEL_OPC_LAST]; 48 void *task_pool_base; 49 TAILQ_HEAD(, spdk_accel_task) task_pool; 50 }; 51 52 int 53 spdk_accel_get_opc_module_name(enum accel_opcode opcode, const char **module_name) 54 { 55 if (opcode >= ACCEL_OPC_LAST) { 56 /* invalid opcode */ 57 return -EINVAL; 58 } 59 60 if (g_modules_opc[opcode]) { 61 *module_name = g_modules_opc[opcode]->name; 62 } else { 63 return -ENOENT; 64 } 65 66 return 0; 67 } 68 69 void 70 _accel_for_each_module(struct module_info *info, _accel_for_each_module_fn fn) 71 { 72 struct spdk_accel_module_if *accel_module; 73 enum accel_opcode opcode; 74 int j = 0; 75 76 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 77 for (opcode = 0; opcode < ACCEL_OPC_LAST; opcode++) { 78 if (accel_module->supports_opcode(opcode)) { 79 info->ops[j] = opcode; 80 j++; 81 } 82 } 83 info->name = accel_module->name; 84 info->num_ops = j; 85 fn(info); 86 j = 0; 87 } 88 } 89 90 int 91 spdk_accel_assign_opc(enum accel_opcode opcode, const char *name) 92 { 93 if (g_modules_started == true) { 94 /* we don't allow re-assignment once things have started */ 95 return -EINVAL; 96 } 97 98 if (opcode >= ACCEL_OPC_LAST) { 99 /* invalid opcode */ 100 return -EINVAL; 101 } 102 103 /* module selection will be validated after the framework starts. */ 104 g_modules_opc_override[opcode] = strdup(name); 105 106 return 0; 107 } 108 109 void 110 spdk_accel_task_complete(struct spdk_accel_task *accel_task, int status) 111 { 112 struct accel_io_channel *accel_ch = accel_task->accel_ch; 113 spdk_accel_completion_cb cb_fn = accel_task->cb_fn; 114 void *cb_arg = accel_task->cb_arg; 115 116 /* We should put the accel_task into the list firstly in order to avoid 117 * the accel task list is exhausted when there is recursive call to 118 * allocate accel_task in user's call back function (cb_fn) 119 */ 120 TAILQ_INSERT_HEAD(&accel_ch->task_pool, accel_task, link); 121 122 cb_fn(cb_arg, status); 123 } 124 125 inline static struct spdk_accel_task * 126 _get_task(struct accel_io_channel *accel_ch, spdk_accel_completion_cb cb_fn, void *cb_arg) 127 { 128 struct spdk_accel_task *accel_task; 129 130 accel_task = TAILQ_FIRST(&accel_ch->task_pool); 131 if (accel_task == NULL) { 132 return NULL; 133 } 134 135 TAILQ_REMOVE(&accel_ch->task_pool, accel_task, link); 136 accel_task->link.tqe_next = NULL; 137 accel_task->link.tqe_prev = NULL; 138 139 accel_task->cb_fn = cb_fn; 140 accel_task->cb_arg = cb_arg; 141 accel_task->accel_ch = accel_ch; 142 143 return accel_task; 144 } 145 146 147 148 /* Accel framework public API for copy function */ 149 int 150 spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, 151 uint64_t nbytes, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) 152 { 153 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 154 struct spdk_accel_task *accel_task; 155 struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_COPY]; 156 struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_COPY]; 157 158 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 159 if (accel_task == NULL) { 160 return -ENOMEM; 161 } 162 163 accel_task->dst = dst; 164 accel_task->src = src; 165 accel_task->op_code = ACCEL_OPC_COPY; 166 accel_task->nbytes = nbytes; 167 accel_task->flags = flags; 168 169 return module->submit_tasks(module_ch, accel_task); 170 } 171 172 /* Accel framework public API for dual cast copy function */ 173 int 174 spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, 175 void *dst2, void *src, uint64_t nbytes, int flags, 176 spdk_accel_completion_cb cb_fn, void *cb_arg) 177 { 178 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 179 struct spdk_accel_task *accel_task; 180 struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_DUALCAST]; 181 struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_DUALCAST]; 182 183 if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) { 184 SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n"); 185 return -EINVAL; 186 } 187 188 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 189 if (accel_task == NULL) { 190 return -ENOMEM; 191 } 192 193 accel_task->src = src; 194 accel_task->dst = dst1; 195 accel_task->dst2 = dst2; 196 accel_task->nbytes = nbytes; 197 accel_task->flags = flags; 198 accel_task->op_code = ACCEL_OPC_DUALCAST; 199 200 return module->submit_tasks(module_ch, accel_task); 201 } 202 203 /* Accel framework public API for compare function */ 204 int 205 spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, 206 void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn, 207 void *cb_arg) 208 { 209 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 210 struct spdk_accel_task *accel_task; 211 struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_COMPARE]; 212 struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_COMPARE]; 213 214 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 215 if (accel_task == NULL) { 216 return -ENOMEM; 217 } 218 219 accel_task->src = src1; 220 accel_task->src2 = src2; 221 accel_task->nbytes = nbytes; 222 accel_task->op_code = ACCEL_OPC_COMPARE; 223 224 return module->submit_tasks(module_ch, accel_task); 225 } 226 227 /* Accel framework public API for fill function */ 228 int 229 spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, 230 uint8_t fill, uint64_t nbytes, int flags, 231 spdk_accel_completion_cb cb_fn, void *cb_arg) 232 { 233 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 234 struct spdk_accel_task *accel_task; 235 struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_FILL]; 236 struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_FILL]; 237 238 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 239 if (accel_task == NULL) { 240 return -ENOMEM; 241 } 242 243 accel_task->dst = dst; 244 memset(&accel_task->fill_pattern, fill, sizeof(uint64_t)); 245 accel_task->nbytes = nbytes; 246 accel_task->flags = flags; 247 accel_task->op_code = ACCEL_OPC_FILL; 248 249 return module->submit_tasks(module_ch, accel_task); 250 } 251 252 /* Accel framework public API for CRC-32C function */ 253 int 254 spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, 255 void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn, 256 void *cb_arg) 257 { 258 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 259 struct spdk_accel_task *accel_task; 260 struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_CRC32C]; 261 struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_CRC32C]; 262 263 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 264 if (accel_task == NULL) { 265 return -ENOMEM; 266 } 267 268 accel_task->crc_dst = crc_dst; 269 accel_task->src = src; 270 accel_task->s.iovcnt = 0; 271 accel_task->seed = seed; 272 accel_task->nbytes = nbytes; 273 accel_task->op_code = ACCEL_OPC_CRC32C; 274 275 return module->submit_tasks(module_ch, accel_task); 276 } 277 278 /* Accel framework public API for chained CRC-32C function */ 279 int 280 spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, 281 struct iovec *iov, uint32_t iov_cnt, uint32_t seed, 282 spdk_accel_completion_cb cb_fn, void *cb_arg) 283 { 284 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 285 struct spdk_accel_task *accel_task; 286 struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_CRC32C]; 287 struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_CRC32C]; 288 289 if (iov == NULL) { 290 SPDK_ERRLOG("iov should not be NULL"); 291 return -EINVAL; 292 } 293 294 if (!iov_cnt) { 295 SPDK_ERRLOG("iovcnt should not be zero value\n"); 296 return -EINVAL; 297 } 298 299 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 300 if (accel_task == NULL) { 301 SPDK_ERRLOG("no memory\n"); 302 assert(0); 303 return -ENOMEM; 304 } 305 306 accel_task->s.iovs = iov; 307 accel_task->s.iovcnt = iov_cnt; 308 accel_task->crc_dst = crc_dst; 309 accel_task->seed = seed; 310 accel_task->op_code = ACCEL_OPC_CRC32C; 311 312 return module->submit_tasks(module_ch, accel_task); 313 } 314 315 /* Accel framework public API for copy with CRC-32C function */ 316 int 317 spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, 318 void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes, 319 int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) 320 { 321 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 322 struct spdk_accel_task *accel_task; 323 struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_COPY_CRC32C]; 324 struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_COPY_CRC32C]; 325 326 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 327 if (accel_task == NULL) { 328 return -ENOMEM; 329 } 330 331 accel_task->dst = dst; 332 accel_task->src = src; 333 accel_task->crc_dst = crc_dst; 334 accel_task->s.iovcnt = 0; 335 accel_task->seed = seed; 336 accel_task->nbytes = nbytes; 337 accel_task->flags = flags; 338 accel_task->op_code = ACCEL_OPC_COPY_CRC32C; 339 340 return module->submit_tasks(module_ch, accel_task); 341 } 342 343 /* Accel framework public API for chained copy + CRC-32C function */ 344 int 345 spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, 346 struct iovec *src_iovs, uint32_t iov_cnt, uint32_t *crc_dst, 347 uint32_t seed, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) 348 { 349 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 350 struct spdk_accel_task *accel_task; 351 struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_COPY_CRC32C]; 352 struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_COPY_CRC32C]; 353 uint64_t nbytes; 354 uint32_t i; 355 356 if (src_iovs == NULL) { 357 SPDK_ERRLOG("iov should not be NULL"); 358 return -EINVAL; 359 } 360 361 if (!iov_cnt) { 362 SPDK_ERRLOG("iovcnt should not be zero value\n"); 363 return -EINVAL; 364 } 365 366 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 367 if (accel_task == NULL) { 368 SPDK_ERRLOG("no memory\n"); 369 assert(0); 370 return -ENOMEM; 371 } 372 373 nbytes = 0; 374 for (i = 0; i < iov_cnt; i++) { 375 nbytes += src_iovs[i].iov_len; 376 } 377 378 accel_task->s.iovs = src_iovs; 379 accel_task->s.iovcnt = iov_cnt; 380 accel_task->dst = (void *)dst; 381 accel_task->crc_dst = crc_dst; 382 accel_task->seed = seed; 383 accel_task->nbytes = nbytes; 384 accel_task->flags = flags; 385 accel_task->op_code = ACCEL_OPC_COPY_CRC32C; 386 387 return module->submit_tasks(module_ch, accel_task); 388 } 389 390 int 391 spdk_accel_submit_compress(struct spdk_io_channel *ch, void *dst, uint64_t nbytes, 392 struct iovec *src_iovs, size_t src_iovcnt, uint32_t *output_size, int flags, 393 spdk_accel_completion_cb cb_fn, void *cb_arg) 394 { 395 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 396 struct spdk_accel_task *accel_task; 397 struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_COMPRESS]; 398 struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_COMPRESS]; 399 size_t i, src_len = 0; 400 401 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 402 if (accel_task == NULL) { 403 return -ENOMEM; 404 } 405 406 for (i = 0; i < src_iovcnt; i++) { 407 src_len += src_iovs[i].iov_len; 408 } 409 410 accel_task->nbytes = src_len; 411 accel_task->output_size = output_size; 412 accel_task->s.iovs = src_iovs; 413 accel_task->s.iovcnt = src_iovcnt; 414 accel_task->dst = dst; 415 accel_task->nbytes_dst = nbytes; 416 accel_task->flags = flags; 417 accel_task->op_code = ACCEL_OPC_COMPRESS; 418 419 return module->submit_tasks(module_ch, accel_task); 420 421 return 0; 422 } 423 424 int 425 spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_iovs, 426 size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt, 427 int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) 428 { 429 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 430 struct spdk_accel_task *accel_task; 431 struct spdk_accel_module_if *module = g_modules_opc[ACCEL_OPC_DECOMPRESS]; 432 struct spdk_io_channel *module_ch = accel_ch->module_ch[ACCEL_OPC_DECOMPRESS]; 433 434 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 435 if (accel_task == NULL) { 436 return -ENOMEM; 437 } 438 439 accel_task->s.iovs = src_iovs; 440 accel_task->s.iovcnt = src_iovcnt; 441 accel_task->d.iovs = dst_iovs; 442 accel_task->d.iovcnt = dst_iovcnt; 443 accel_task->flags = flags; 444 accel_task->op_code = ACCEL_OPC_DECOMPRESS; 445 446 return module->submit_tasks(module_ch, accel_task); 447 448 return 0; 449 } 450 451 452 static struct spdk_accel_module_if * 453 _module_find_by_name(const char *name) 454 { 455 struct spdk_accel_module_if *accel_module = NULL; 456 457 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 458 if (strcmp(name, accel_module->name) == 0) { 459 break; 460 } 461 } 462 463 return accel_module; 464 } 465 466 /* Helper function when when accel modules register with the framework. */ 467 void 468 spdk_accel_module_list_add(struct spdk_accel_module_if *accel_module) 469 { 470 if (_module_find_by_name(accel_module->name)) { 471 SPDK_NOTICELOG("Accel module %s already registered\n", accel_module->name); 472 assert(false); 473 return; 474 } 475 476 /* Make sure that the software module is at the head of the list, this 477 * will assure that all opcodes are later assigned to software first and 478 * then udpated to HW modules as they are registered. 479 */ 480 if (strcmp(accel_module->name, "software") == 0) { 481 TAILQ_INSERT_HEAD(&spdk_accel_module_list, accel_module, tailq); 482 } else { 483 TAILQ_INSERT_TAIL(&spdk_accel_module_list, accel_module, tailq); 484 } 485 486 if (accel_module->get_ctx_size && accel_module->get_ctx_size() > g_max_accel_module_size) { 487 g_max_accel_module_size = accel_module->get_ctx_size(); 488 } 489 } 490 491 /* Framework level channel create callback. */ 492 static int 493 accel_create_channel(void *io_device, void *ctx_buf) 494 { 495 struct accel_io_channel *accel_ch = ctx_buf; 496 struct spdk_accel_task *accel_task; 497 uint8_t *task_mem; 498 int i, j; 499 500 accel_ch->task_pool_base = calloc(MAX_TASKS_PER_CHANNEL, g_max_accel_module_size); 501 if (accel_ch->task_pool_base == NULL) { 502 return -ENOMEM; 503 } 504 505 TAILQ_INIT(&accel_ch->task_pool); 506 task_mem = accel_ch->task_pool_base; 507 for (i = 0 ; i < MAX_TASKS_PER_CHANNEL; i++) { 508 accel_task = (struct spdk_accel_task *)task_mem; 509 TAILQ_INSERT_TAIL(&accel_ch->task_pool, accel_task, link); 510 task_mem += g_max_accel_module_size; 511 } 512 513 /* Assign modules and get IO channels for each */ 514 for (i = 0; i < ACCEL_OPC_LAST; i++) { 515 accel_ch->module_ch[i] = g_modules_opc[i]->get_io_channel(); 516 /* This can happen if idxd runs out of channels. */ 517 if (accel_ch->module_ch[i] == NULL) { 518 goto err; 519 } 520 } 521 522 return 0; 523 err: 524 for (j = 0; j < i; j++) { 525 spdk_put_io_channel(accel_ch->module_ch[j]); 526 } 527 free(accel_ch->task_pool_base); 528 return -ENOMEM; 529 } 530 531 /* Framework level channel destroy callback. */ 532 static void 533 accel_destroy_channel(void *io_device, void *ctx_buf) 534 { 535 struct accel_io_channel *accel_ch = ctx_buf; 536 int i; 537 538 for (i = 0; i < ACCEL_OPC_LAST; i++) { 539 assert(accel_ch->module_ch[i] != NULL); 540 spdk_put_io_channel(accel_ch->module_ch[i]); 541 accel_ch->module_ch[i] = NULL; 542 } 543 544 free(accel_ch->task_pool_base); 545 } 546 547 struct spdk_io_channel * 548 spdk_accel_get_io_channel(void) 549 { 550 return spdk_get_io_channel(&spdk_accel_module_list); 551 } 552 553 static void 554 accel_module_initialize(void) 555 { 556 struct spdk_accel_module_if *accel_module; 557 558 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 559 accel_module->module_init(); 560 } 561 } 562 563 int 564 spdk_accel_initialize(void) 565 { 566 enum accel_opcode op; 567 struct spdk_accel_module_if *accel_module = NULL; 568 569 g_modules_started = true; 570 accel_module_initialize(); 571 572 /* Create our priority global map of opcodes to modules, we populate starting 573 * with the software module (guaranteed to be first on the list) and then 574 * updating opcodes with HW modules that have been initilaized. 575 * NOTE: all opcodes must be suported by software in the event that no HW 576 * modules are initilaized to support the operation. 577 */ 578 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 579 for (op = 0; op < ACCEL_OPC_LAST; op++) { 580 if (accel_module->supports_opcode(op)) { 581 g_modules_opc[op] = accel_module; 582 SPDK_DEBUGLOG(accel, "OPC 0x%x now assigned to %s\n", op, accel_module->name); 583 } 584 } 585 } 586 587 /* Now lets check for overrides and apply all that exist */ 588 for (op = 0; op < ACCEL_OPC_LAST; op++) { 589 if (g_modules_opc_override[op] != NULL) { 590 accel_module = _module_find_by_name(g_modules_opc_override[op]); 591 if (accel_module == NULL) { 592 SPDK_ERRLOG("Invalid module name of %s\n", g_modules_opc_override[op]); 593 return -EINVAL; 594 } 595 if (accel_module->supports_opcode(op) == false) { 596 SPDK_ERRLOG("Module %s does not support op code %d\n", accel_module->name, op); 597 return -EINVAL; 598 } 599 g_modules_opc[op] = accel_module; 600 } 601 } 602 603 #ifdef DEBUG 604 for (op = 0; op < ACCEL_OPC_LAST; op++) { 605 assert(g_modules_opc[op] != NULL); 606 } 607 #endif 608 /* 609 * We need a unique identifier for the accel framework, so use the 610 * spdk_accel_module_list address for this purpose. 611 */ 612 spdk_io_device_register(&spdk_accel_module_list, accel_create_channel, accel_destroy_channel, 613 sizeof(struct accel_io_channel), "accel"); 614 615 return 0; 616 } 617 618 static void 619 accel_module_finish_cb(void) 620 { 621 spdk_accel_fini_cb cb_fn = g_fini_cb_fn; 622 623 cb_fn(g_fini_cb_arg); 624 g_fini_cb_fn = NULL; 625 g_fini_cb_arg = NULL; 626 } 627 628 void 629 spdk_accel_write_config_json(struct spdk_json_write_ctx *w) 630 { 631 struct spdk_accel_module_if *accel_module; 632 633 /* 634 * The accel fw has no config, there may be some in 635 * the modules though. 636 */ 637 spdk_json_write_array_begin(w); 638 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 639 if (accel_module->write_config_json) { 640 accel_module->write_config_json(w); 641 } 642 } 643 spdk_json_write_array_end(w); 644 } 645 646 void 647 spdk_accel_module_finish(void) 648 { 649 if (!g_accel_module) { 650 g_accel_module = TAILQ_FIRST(&spdk_accel_module_list); 651 } else { 652 g_accel_module = TAILQ_NEXT(g_accel_module, tailq); 653 } 654 655 if (!g_accel_module) { 656 accel_module_finish_cb(); 657 return; 658 } 659 660 if (g_accel_module->module_fini) { 661 spdk_thread_send_msg(spdk_get_thread(), g_accel_module->module_fini, NULL); 662 } else { 663 spdk_accel_module_finish(); 664 } 665 } 666 667 void 668 spdk_accel_finish(spdk_accel_fini_cb cb_fn, void *cb_arg) 669 { 670 enum accel_opcode op; 671 672 assert(cb_fn != NULL); 673 674 g_fini_cb_fn = cb_fn; 675 g_fini_cb_arg = cb_arg; 676 677 for (op = 0; op < ACCEL_OPC_LAST; op++) { 678 if (g_modules_opc_override[op] != NULL) { 679 free(g_modules_opc_override[op]); 680 g_modules_opc_override[op] = NULL; 681 } 682 g_modules_opc[op] = NULL; 683 } 684 685 spdk_io_device_unregister(&spdk_accel_module_list, NULL); 686 spdk_accel_module_finish(); 687 } 688 689 SPDK_LOG_REGISTER_COMPONENT(accel) 690