1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. 4 * All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk_internal/accel_module.h" 10 11 #include "accel_internal.h" 12 13 #include "spdk/dma.h" 14 #include "spdk/env.h" 15 #include "spdk/likely.h" 16 #include "spdk/log.h" 17 #include "spdk/thread.h" 18 #include "spdk/json.h" 19 #include "spdk/crc32.h" 20 #include "spdk/util.h" 21 #include "spdk/hexlify.h" 22 23 /* Accelerator Framework: The following provides a top level 24 * generic API for the accelerator functions defined here. Modules, 25 * such as the one in /module/accel/ioat, supply the implementation 26 * with the exception of the pure software implementation contained 27 * later in this file. 28 */ 29 30 #define ALIGN_4K 0x1000 31 #define MAX_TASKS_PER_CHANNEL 0x800 32 #define ACCEL_SMALL_CACHE_SIZE 128 33 #define ACCEL_LARGE_CACHE_SIZE 16 34 /* Set MSB, so we don't return NULL pointers as buffers */ 35 #define ACCEL_BUFFER_BASE ((void *)(1ull << 63)) 36 #define ACCEL_BUFFER_OFFSET_MASK ((uintptr_t)ACCEL_BUFFER_BASE - 1) 37 38 struct accel_module { 39 struct spdk_accel_module_if *module; 40 bool supports_memory_domains; 41 }; 42 43 /* Largest context size for all accel modules */ 44 static size_t g_max_accel_module_size = sizeof(struct spdk_accel_task); 45 46 static struct spdk_accel_module_if *g_accel_module = NULL; 47 static spdk_accel_fini_cb g_fini_cb_fn = NULL; 48 static void *g_fini_cb_arg = NULL; 49 static bool g_modules_started = false; 50 static struct spdk_memory_domain *g_accel_domain; 51 52 /* Global list of registered accelerator modules */ 53 static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list = 54 TAILQ_HEAD_INITIALIZER(spdk_accel_module_list); 55 56 /* Crypto keyring */ 57 static TAILQ_HEAD(, spdk_accel_crypto_key) g_keyring = TAILQ_HEAD_INITIALIZER(g_keyring); 58 static struct spdk_spinlock g_keyring_spin; 59 60 /* Global array mapping capabilities to modules */ 61 static struct accel_module g_modules_opc[ACCEL_OPC_LAST] = {}; 62 static char *g_modules_opc_override[ACCEL_OPC_LAST] = {}; 63 TAILQ_HEAD(, spdk_accel_driver) g_accel_drivers = TAILQ_HEAD_INITIALIZER(g_accel_drivers); 64 static struct spdk_accel_driver *g_accel_driver; 65 static struct spdk_accel_opts g_opts = { 66 .small_cache_size = ACCEL_SMALL_CACHE_SIZE, 67 .large_cache_size = ACCEL_LARGE_CACHE_SIZE, 68 }; 69 70 static const char *g_opcode_strings[ACCEL_OPC_LAST] = { 71 "copy", "fill", "dualcast", "compare", "crc32c", "copy_crc32c", 72 "compress", "decompress", "encrypt", "decrypt", "xor" 73 }; 74 75 enum accel_sequence_state { 76 ACCEL_SEQUENCE_STATE_INIT, 77 ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF, 78 ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF, 79 ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF, 80 ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF, 81 ACCEL_SEQUENCE_STATE_PULL_DATA, 82 ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA, 83 ACCEL_SEQUENCE_STATE_EXEC_TASK, 84 ACCEL_SEQUENCE_STATE_AWAIT_TASK, 85 ACCEL_SEQUENCE_STATE_COMPLETE_TASK, 86 ACCEL_SEQUENCE_STATE_NEXT_TASK, 87 ACCEL_SEQUENCE_STATE_PUSH_DATA, 88 ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA, 89 ACCEL_SEQUENCE_STATE_DRIVER_EXEC, 90 ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASK, 91 ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE, 92 ACCEL_SEQUENCE_STATE_ERROR, 93 ACCEL_SEQUENCE_STATE_MAX, 94 }; 95 96 static const char *g_seq_states[] 97 __attribute__((unused)) = { 98 [ACCEL_SEQUENCE_STATE_INIT] = "init", 99 [ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF] = "check-virtbuf", 100 [ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF] = "await-virtbuf", 101 [ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF] = "check-bouncebuf", 102 [ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF] = "await-bouncebuf", 103 [ACCEL_SEQUENCE_STATE_PULL_DATA] = "pull-data", 104 [ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA] = "await-pull-data", 105 [ACCEL_SEQUENCE_STATE_EXEC_TASK] = "exec-task", 106 [ACCEL_SEQUENCE_STATE_AWAIT_TASK] = "await-task", 107 [ACCEL_SEQUENCE_STATE_COMPLETE_TASK] = "complete-task", 108 [ACCEL_SEQUENCE_STATE_NEXT_TASK] = "next-task", 109 [ACCEL_SEQUENCE_STATE_PUSH_DATA] = "push-data", 110 [ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA] = "await-push-data", 111 [ACCEL_SEQUENCE_STATE_DRIVER_EXEC] = "driver-exec", 112 [ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASK] = "driver-await-task", 113 [ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE] = "driver-complete", 114 [ACCEL_SEQUENCE_STATE_ERROR] = "error", 115 [ACCEL_SEQUENCE_STATE_MAX] = "", 116 }; 117 118 #define ACCEL_SEQUENCE_STATE_STRING(s) \ 119 (((s) >= ACCEL_SEQUENCE_STATE_INIT && (s) < ACCEL_SEQUENCE_STATE_MAX) \ 120 ? g_seq_states[s] : "unknown") 121 122 struct accel_buffer { 123 struct spdk_accel_sequence *seq; 124 void *buf; 125 uint64_t len; 126 struct spdk_iobuf_entry iobuf; 127 spdk_accel_sequence_get_buf_cb cb_fn; 128 void *cb_ctx; 129 TAILQ_ENTRY(accel_buffer) link; 130 }; 131 132 struct accel_io_channel { 133 struct spdk_io_channel *module_ch[ACCEL_OPC_LAST]; 134 void *task_pool_base; 135 struct spdk_accel_sequence *seq_pool_base; 136 struct accel_buffer *buf_pool_base; 137 TAILQ_HEAD(, spdk_accel_task) task_pool; 138 TAILQ_HEAD(, spdk_accel_sequence) seq_pool; 139 TAILQ_HEAD(, accel_buffer) buf_pool; 140 struct spdk_iobuf_channel iobuf; 141 struct accel_stats stats; 142 }; 143 144 TAILQ_HEAD(accel_sequence_tasks, spdk_accel_task); 145 146 struct spdk_accel_sequence { 147 struct accel_io_channel *ch; 148 struct accel_sequence_tasks tasks; 149 struct accel_sequence_tasks completed; 150 TAILQ_HEAD(, accel_buffer) bounce_bufs; 151 enum accel_sequence_state state; 152 int status; 153 bool in_process_sequence; 154 spdk_accel_completion_cb cb_fn; 155 void *cb_arg; 156 TAILQ_ENTRY(spdk_accel_sequence) link; 157 }; 158 159 #define accel_update_stats(ch, event) (ch)->stats.event++ 160 #define accel_update_task_stats(ch, task, event) \ 161 accel_update_stats(ch, operations[(task)->op_code].event) 162 163 static inline void 164 accel_sequence_set_state(struct spdk_accel_sequence *seq, enum accel_sequence_state state) 165 { 166 SPDK_DEBUGLOG(accel, "seq=%p, setting state: %s -> %s\n", seq, 167 ACCEL_SEQUENCE_STATE_STRING(seq->state), ACCEL_SEQUENCE_STATE_STRING(state)); 168 assert(seq->state != ACCEL_SEQUENCE_STATE_ERROR || state == ACCEL_SEQUENCE_STATE_ERROR); 169 seq->state = state; 170 } 171 172 static void 173 accel_sequence_set_fail(struct spdk_accel_sequence *seq, int status) 174 { 175 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR); 176 assert(status != 0); 177 seq->status = status; 178 } 179 180 int 181 spdk_accel_get_opc_module_name(enum accel_opcode opcode, const char **module_name) 182 { 183 if (opcode >= ACCEL_OPC_LAST) { 184 /* invalid opcode */ 185 return -EINVAL; 186 } 187 188 if (g_modules_opc[opcode].module) { 189 *module_name = g_modules_opc[opcode].module->name; 190 } else { 191 return -ENOENT; 192 } 193 194 return 0; 195 } 196 197 void 198 _accel_for_each_module(struct module_info *info, _accel_for_each_module_fn fn) 199 { 200 struct spdk_accel_module_if *accel_module; 201 enum accel_opcode opcode; 202 int j = 0; 203 204 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 205 for (opcode = 0; opcode < ACCEL_OPC_LAST; opcode++) { 206 if (accel_module->supports_opcode(opcode)) { 207 info->ops[j] = opcode; 208 j++; 209 } 210 } 211 info->name = accel_module->name; 212 info->num_ops = j; 213 fn(info); 214 j = 0; 215 } 216 } 217 218 int 219 _accel_get_opc_name(enum accel_opcode opcode, const char **opcode_name) 220 { 221 int rc = 0; 222 223 if (opcode < ACCEL_OPC_LAST) { 224 *opcode_name = g_opcode_strings[opcode]; 225 } else { 226 /* invalid opcode */ 227 rc = -EINVAL; 228 } 229 230 return rc; 231 } 232 233 int 234 spdk_accel_assign_opc(enum accel_opcode opcode, const char *name) 235 { 236 if (g_modules_started == true) { 237 /* we don't allow re-assignment once things have started */ 238 return -EINVAL; 239 } 240 241 if (opcode >= ACCEL_OPC_LAST) { 242 /* invalid opcode */ 243 return -EINVAL; 244 } 245 246 /* module selection will be validated after the framework starts. */ 247 g_modules_opc_override[opcode] = strdup(name); 248 249 return 0; 250 } 251 252 void 253 spdk_accel_task_complete(struct spdk_accel_task *accel_task, int status) 254 { 255 struct accel_io_channel *accel_ch = accel_task->accel_ch; 256 spdk_accel_completion_cb cb_fn = accel_task->cb_fn; 257 void *cb_arg = accel_task->cb_arg; 258 259 /* We should put the accel_task into the list firstly in order to avoid 260 * the accel task list is exhausted when there is recursive call to 261 * allocate accel_task in user's call back function (cb_fn) 262 */ 263 TAILQ_INSERT_HEAD(&accel_ch->task_pool, accel_task, link); 264 265 accel_update_task_stats(accel_ch, accel_task, executed); 266 if (spdk_unlikely(status != 0)) { 267 accel_update_task_stats(accel_ch, accel_task, failed); 268 } 269 270 cb_fn(cb_arg, status); 271 } 272 273 inline static struct spdk_accel_task * 274 _get_task(struct accel_io_channel *accel_ch, spdk_accel_completion_cb cb_fn, void *cb_arg) 275 { 276 struct spdk_accel_task *accel_task; 277 278 accel_task = TAILQ_FIRST(&accel_ch->task_pool); 279 if (accel_task == NULL) { 280 return NULL; 281 } 282 283 TAILQ_REMOVE(&accel_ch->task_pool, accel_task, link); 284 accel_task->link.tqe_next = NULL; 285 accel_task->link.tqe_prev = NULL; 286 287 accel_task->cb_fn = cb_fn; 288 accel_task->cb_arg = cb_arg; 289 accel_task->accel_ch = accel_ch; 290 accel_task->bounce.s.orig_iovs = NULL; 291 accel_task->bounce.d.orig_iovs = NULL; 292 293 return accel_task; 294 } 295 296 static inline int 297 accel_submit_task(struct accel_io_channel *accel_ch, struct spdk_accel_task *task) 298 { 299 struct spdk_io_channel *module_ch = accel_ch->module_ch[task->op_code]; 300 struct spdk_accel_module_if *module = g_modules_opc[task->op_code].module; 301 int rc; 302 303 rc = module->submit_tasks(module_ch, task); 304 if (spdk_unlikely(rc != 0)) { 305 accel_update_task_stats(accel_ch, task, failed); 306 } 307 308 return rc; 309 } 310 311 /* Accel framework public API for copy function */ 312 int 313 spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, 314 uint64_t nbytes, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) 315 { 316 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 317 struct spdk_accel_task *accel_task; 318 319 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 320 if (accel_task == NULL) { 321 return -ENOMEM; 322 } 323 324 accel_task->s.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_SRC]; 325 accel_task->d.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 326 accel_task->d.iovs[0].iov_base = dst; 327 accel_task->d.iovs[0].iov_len = nbytes; 328 accel_task->d.iovcnt = 1; 329 accel_task->s.iovs[0].iov_base = src; 330 accel_task->s.iovs[0].iov_len = nbytes; 331 accel_task->s.iovcnt = 1; 332 accel_task->op_code = ACCEL_OPC_COPY; 333 accel_task->flags = flags; 334 accel_task->src_domain = NULL; 335 accel_task->dst_domain = NULL; 336 accel_task->step_cb_fn = NULL; 337 338 return accel_submit_task(accel_ch, accel_task); 339 } 340 341 /* Accel framework public API for dual cast copy function */ 342 int 343 spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, 344 void *dst2, void *src, uint64_t nbytes, int flags, 345 spdk_accel_completion_cb cb_fn, void *cb_arg) 346 { 347 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 348 struct spdk_accel_task *accel_task; 349 350 if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) { 351 SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n"); 352 return -EINVAL; 353 } 354 355 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 356 if (accel_task == NULL) { 357 return -ENOMEM; 358 } 359 360 accel_task->s.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_SRC]; 361 accel_task->d.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 362 accel_task->d2.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST2]; 363 accel_task->d.iovs[0].iov_base = dst1; 364 accel_task->d.iovs[0].iov_len = nbytes; 365 accel_task->d.iovcnt = 1; 366 accel_task->d2.iovs[0].iov_base = dst2; 367 accel_task->d2.iovs[0].iov_len = nbytes; 368 accel_task->d2.iovcnt = 1; 369 accel_task->s.iovs[0].iov_base = src; 370 accel_task->s.iovs[0].iov_len = nbytes; 371 accel_task->s.iovcnt = 1; 372 accel_task->flags = flags; 373 accel_task->op_code = ACCEL_OPC_DUALCAST; 374 accel_task->src_domain = NULL; 375 accel_task->dst_domain = NULL; 376 accel_task->step_cb_fn = NULL; 377 378 return accel_submit_task(accel_ch, accel_task); 379 } 380 381 /* Accel framework public API for compare function */ 382 int 383 spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, 384 void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn, 385 void *cb_arg) 386 { 387 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 388 struct spdk_accel_task *accel_task; 389 390 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 391 if (accel_task == NULL) { 392 return -ENOMEM; 393 } 394 395 accel_task->s.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_SRC]; 396 accel_task->s2.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_SRC2]; 397 accel_task->s.iovs[0].iov_base = src1; 398 accel_task->s.iovs[0].iov_len = nbytes; 399 accel_task->s.iovcnt = 1; 400 accel_task->s2.iovs[0].iov_base = src2; 401 accel_task->s2.iovs[0].iov_len = nbytes; 402 accel_task->s2.iovcnt = 1; 403 accel_task->op_code = ACCEL_OPC_COMPARE; 404 accel_task->src_domain = NULL; 405 accel_task->dst_domain = NULL; 406 accel_task->step_cb_fn = NULL; 407 408 return accel_submit_task(accel_ch, accel_task); 409 } 410 411 /* Accel framework public API for fill function */ 412 int 413 spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, 414 uint8_t fill, uint64_t nbytes, int flags, 415 spdk_accel_completion_cb cb_fn, void *cb_arg) 416 { 417 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 418 struct spdk_accel_task *accel_task; 419 420 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 421 if (accel_task == NULL) { 422 return -ENOMEM; 423 } 424 425 accel_task->d.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 426 accel_task->d.iovs[0].iov_base = dst; 427 accel_task->d.iovs[0].iov_len = nbytes; 428 accel_task->d.iovcnt = 1; 429 memset(&accel_task->fill_pattern, fill, sizeof(uint64_t)); 430 accel_task->flags = flags; 431 accel_task->op_code = ACCEL_OPC_FILL; 432 accel_task->src_domain = NULL; 433 accel_task->dst_domain = NULL; 434 accel_task->step_cb_fn = NULL; 435 436 return accel_submit_task(accel_ch, accel_task); 437 } 438 439 /* Accel framework public API for CRC-32C function */ 440 int 441 spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, 442 void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn, 443 void *cb_arg) 444 { 445 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 446 struct spdk_accel_task *accel_task; 447 448 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 449 if (accel_task == NULL) { 450 return -ENOMEM; 451 } 452 453 accel_task->s.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_SRC]; 454 accel_task->s.iovs[0].iov_base = src; 455 accel_task->s.iovs[0].iov_len = nbytes; 456 accel_task->s.iovcnt = 1; 457 accel_task->crc_dst = crc_dst; 458 accel_task->seed = seed; 459 accel_task->op_code = ACCEL_OPC_CRC32C; 460 accel_task->src_domain = NULL; 461 accel_task->dst_domain = NULL; 462 accel_task->step_cb_fn = NULL; 463 464 return accel_submit_task(accel_ch, accel_task); 465 } 466 467 /* Accel framework public API for chained CRC-32C function */ 468 int 469 spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, 470 struct iovec *iov, uint32_t iov_cnt, uint32_t seed, 471 spdk_accel_completion_cb cb_fn, void *cb_arg) 472 { 473 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 474 struct spdk_accel_task *accel_task; 475 476 if (iov == NULL) { 477 SPDK_ERRLOG("iov should not be NULL"); 478 return -EINVAL; 479 } 480 481 if (!iov_cnt) { 482 SPDK_ERRLOG("iovcnt should not be zero value\n"); 483 return -EINVAL; 484 } 485 486 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 487 if (accel_task == NULL) { 488 SPDK_ERRLOG("no memory\n"); 489 assert(0); 490 return -ENOMEM; 491 } 492 493 accel_task->s.iovs = iov; 494 accel_task->s.iovcnt = iov_cnt; 495 accel_task->crc_dst = crc_dst; 496 accel_task->seed = seed; 497 accel_task->op_code = ACCEL_OPC_CRC32C; 498 accel_task->src_domain = NULL; 499 accel_task->dst_domain = NULL; 500 accel_task->step_cb_fn = NULL; 501 502 return accel_submit_task(accel_ch, accel_task); 503 } 504 505 /* Accel framework public API for copy with CRC-32C function */ 506 int 507 spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, 508 void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes, 509 int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) 510 { 511 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 512 struct spdk_accel_task *accel_task; 513 514 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 515 if (accel_task == NULL) { 516 return -ENOMEM; 517 } 518 519 accel_task->s.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_SRC]; 520 accel_task->d.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 521 accel_task->d.iovs[0].iov_base = dst; 522 accel_task->d.iovs[0].iov_len = nbytes; 523 accel_task->d.iovcnt = 1; 524 accel_task->s.iovs[0].iov_base = src; 525 accel_task->s.iovs[0].iov_len = nbytes; 526 accel_task->s.iovcnt = 1; 527 accel_task->crc_dst = crc_dst; 528 accel_task->seed = seed; 529 accel_task->flags = flags; 530 accel_task->op_code = ACCEL_OPC_COPY_CRC32C; 531 accel_task->src_domain = NULL; 532 accel_task->dst_domain = NULL; 533 accel_task->step_cb_fn = NULL; 534 535 return accel_submit_task(accel_ch, accel_task); 536 } 537 538 /* Accel framework public API for chained copy + CRC-32C function */ 539 int 540 spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, 541 struct iovec *src_iovs, uint32_t iov_cnt, uint32_t *crc_dst, 542 uint32_t seed, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) 543 { 544 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 545 struct spdk_accel_task *accel_task; 546 uint64_t nbytes; 547 uint32_t i; 548 549 if (src_iovs == NULL) { 550 SPDK_ERRLOG("iov should not be NULL"); 551 return -EINVAL; 552 } 553 554 if (!iov_cnt) { 555 SPDK_ERRLOG("iovcnt should not be zero value\n"); 556 return -EINVAL; 557 } 558 559 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 560 if (accel_task == NULL) { 561 SPDK_ERRLOG("no memory\n"); 562 assert(0); 563 return -ENOMEM; 564 } 565 566 nbytes = 0; 567 for (i = 0; i < iov_cnt; i++) { 568 nbytes += src_iovs[i].iov_len; 569 } 570 571 accel_task->d.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 572 accel_task->d.iovs[0].iov_base = dst; 573 accel_task->d.iovs[0].iov_len = nbytes; 574 accel_task->d.iovcnt = 1; 575 accel_task->s.iovs = src_iovs; 576 accel_task->s.iovcnt = iov_cnt; 577 accel_task->crc_dst = crc_dst; 578 accel_task->seed = seed; 579 accel_task->flags = flags; 580 accel_task->op_code = ACCEL_OPC_COPY_CRC32C; 581 accel_task->src_domain = NULL; 582 accel_task->dst_domain = NULL; 583 accel_task->step_cb_fn = NULL; 584 585 return accel_submit_task(accel_ch, accel_task); 586 } 587 588 int 589 spdk_accel_submit_compress(struct spdk_io_channel *ch, void *dst, uint64_t nbytes, 590 struct iovec *src_iovs, size_t src_iovcnt, uint32_t *output_size, int flags, 591 spdk_accel_completion_cb cb_fn, void *cb_arg) 592 { 593 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 594 struct spdk_accel_task *accel_task; 595 596 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 597 if (accel_task == NULL) { 598 return -ENOMEM; 599 } 600 601 accel_task->d.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 602 accel_task->d.iovs[0].iov_base = dst; 603 accel_task->d.iovs[0].iov_len = nbytes; 604 accel_task->d.iovcnt = 1; 605 accel_task->output_size = output_size; 606 accel_task->s.iovs = src_iovs; 607 accel_task->s.iovcnt = src_iovcnt; 608 accel_task->flags = flags; 609 accel_task->op_code = ACCEL_OPC_COMPRESS; 610 accel_task->src_domain = NULL; 611 accel_task->dst_domain = NULL; 612 accel_task->step_cb_fn = NULL; 613 614 return accel_submit_task(accel_ch, accel_task); 615 } 616 617 int 618 spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_iovs, 619 size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt, 620 uint32_t *output_size, int flags, spdk_accel_completion_cb cb_fn, 621 void *cb_arg) 622 { 623 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 624 struct spdk_accel_task *accel_task; 625 626 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 627 if (accel_task == NULL) { 628 return -ENOMEM; 629 } 630 631 accel_task->output_size = output_size; 632 accel_task->s.iovs = src_iovs; 633 accel_task->s.iovcnt = src_iovcnt; 634 accel_task->d.iovs = dst_iovs; 635 accel_task->d.iovcnt = dst_iovcnt; 636 accel_task->flags = flags; 637 accel_task->op_code = ACCEL_OPC_DECOMPRESS; 638 accel_task->src_domain = NULL; 639 accel_task->dst_domain = NULL; 640 accel_task->step_cb_fn = NULL; 641 642 return accel_submit_task(accel_ch, accel_task); 643 } 644 645 int 646 spdk_accel_submit_encrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key, 647 struct iovec *dst_iovs, uint32_t dst_iovcnt, 648 struct iovec *src_iovs, uint32_t src_iovcnt, 649 uint64_t iv, uint32_t block_size, int flags, 650 spdk_accel_completion_cb cb_fn, void *cb_arg) 651 { 652 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 653 struct spdk_accel_task *accel_task; 654 655 if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) { 656 return -EINVAL; 657 } 658 659 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 660 if (accel_task == NULL) { 661 return -ENOMEM; 662 } 663 664 accel_task->crypto_key = key; 665 accel_task->s.iovs = src_iovs; 666 accel_task->s.iovcnt = src_iovcnt; 667 accel_task->d.iovs = dst_iovs; 668 accel_task->d.iovcnt = dst_iovcnt; 669 accel_task->iv = iv; 670 accel_task->block_size = block_size; 671 accel_task->flags = flags; 672 accel_task->op_code = ACCEL_OPC_ENCRYPT; 673 accel_task->src_domain = NULL; 674 accel_task->dst_domain = NULL; 675 accel_task->step_cb_fn = NULL; 676 677 return accel_submit_task(accel_ch, accel_task); 678 } 679 680 int 681 spdk_accel_submit_decrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key, 682 struct iovec *dst_iovs, uint32_t dst_iovcnt, 683 struct iovec *src_iovs, uint32_t src_iovcnt, 684 uint64_t iv, uint32_t block_size, int flags, 685 spdk_accel_completion_cb cb_fn, void *cb_arg) 686 { 687 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 688 struct spdk_accel_task *accel_task; 689 690 if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) { 691 return -EINVAL; 692 } 693 694 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 695 if (accel_task == NULL) { 696 return -ENOMEM; 697 } 698 699 accel_task->crypto_key = key; 700 accel_task->s.iovs = src_iovs; 701 accel_task->s.iovcnt = src_iovcnt; 702 accel_task->d.iovs = dst_iovs; 703 accel_task->d.iovcnt = dst_iovcnt; 704 accel_task->iv = iv; 705 accel_task->block_size = block_size; 706 accel_task->flags = flags; 707 accel_task->op_code = ACCEL_OPC_DECRYPT; 708 accel_task->src_domain = NULL; 709 accel_task->dst_domain = NULL; 710 accel_task->step_cb_fn = NULL; 711 712 return accel_submit_task(accel_ch, accel_task); 713 } 714 715 int 716 spdk_accel_submit_xor(struct spdk_io_channel *ch, void *dst, void **sources, uint32_t nsrcs, 717 uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg) 718 { 719 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 720 struct spdk_accel_task *accel_task; 721 722 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 723 if (accel_task == NULL) { 724 return -ENOMEM; 725 } 726 727 accel_task->nsrcs.srcs = sources; 728 accel_task->nsrcs.cnt = nsrcs; 729 accel_task->d.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 730 accel_task->d.iovs[0].iov_base = dst; 731 accel_task->d.iovs[0].iov_len = nbytes; 732 accel_task->d.iovcnt = 1; 733 accel_task->op_code = ACCEL_OPC_XOR; 734 accel_task->src_domain = NULL; 735 accel_task->dst_domain = NULL; 736 accel_task->step_cb_fn = NULL; 737 738 return accel_submit_task(accel_ch, accel_task); 739 } 740 741 static inline struct accel_buffer * 742 accel_get_buf(struct accel_io_channel *ch, uint64_t len) 743 { 744 struct accel_buffer *buf; 745 746 buf = TAILQ_FIRST(&ch->buf_pool); 747 if (spdk_unlikely(buf == NULL)) { 748 return NULL; 749 } 750 751 TAILQ_REMOVE(&ch->buf_pool, buf, link); 752 buf->len = len; 753 buf->buf = NULL; 754 buf->seq = NULL; 755 buf->cb_fn = NULL; 756 757 return buf; 758 } 759 760 static inline void 761 accel_put_buf(struct accel_io_channel *ch, struct accel_buffer *buf) 762 { 763 if (buf->buf != NULL) { 764 spdk_iobuf_put(&ch->iobuf, buf->buf, buf->len); 765 } 766 767 TAILQ_INSERT_HEAD(&ch->buf_pool, buf, link); 768 } 769 770 static inline struct spdk_accel_sequence * 771 accel_sequence_get(struct accel_io_channel *ch) 772 { 773 struct spdk_accel_sequence *seq; 774 775 seq = TAILQ_FIRST(&ch->seq_pool); 776 if (seq == NULL) { 777 return NULL; 778 } 779 780 TAILQ_REMOVE(&ch->seq_pool, seq, link); 781 782 TAILQ_INIT(&seq->tasks); 783 TAILQ_INIT(&seq->completed); 784 TAILQ_INIT(&seq->bounce_bufs); 785 786 seq->ch = ch; 787 seq->status = 0; 788 seq->state = ACCEL_SEQUENCE_STATE_INIT; 789 seq->in_process_sequence = false; 790 791 return seq; 792 } 793 794 static inline void 795 accel_sequence_put(struct spdk_accel_sequence *seq) 796 { 797 struct accel_io_channel *ch = seq->ch; 798 struct accel_buffer *buf; 799 800 while (!TAILQ_EMPTY(&seq->bounce_bufs)) { 801 buf = TAILQ_FIRST(&seq->bounce_bufs); 802 TAILQ_REMOVE(&seq->bounce_bufs, buf, link); 803 accel_put_buf(seq->ch, buf); 804 } 805 806 assert(TAILQ_EMPTY(&seq->tasks)); 807 assert(TAILQ_EMPTY(&seq->completed)); 808 seq->ch = NULL; 809 810 TAILQ_INSERT_HEAD(&ch->seq_pool, seq, link); 811 } 812 813 static void accel_sequence_task_cb(void *cb_arg, int status); 814 815 static inline struct spdk_accel_task * 816 accel_sequence_get_task(struct accel_io_channel *ch, struct spdk_accel_sequence *seq, 817 spdk_accel_step_cb cb_fn, void *cb_arg) 818 { 819 struct spdk_accel_task *task; 820 821 task = _get_task(ch, accel_sequence_task_cb, seq); 822 if (task == NULL) { 823 return task; 824 } 825 826 task->step_cb_fn = cb_fn; 827 task->step_cb_arg = cb_arg; 828 829 return task; 830 } 831 832 int 833 spdk_accel_append_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 834 struct iovec *dst_iovs, uint32_t dst_iovcnt, 835 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 836 struct iovec *src_iovs, uint32_t src_iovcnt, 837 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 838 int flags, spdk_accel_step_cb cb_fn, void *cb_arg) 839 { 840 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 841 struct spdk_accel_task *task; 842 struct spdk_accel_sequence *seq = *pseq; 843 844 if (seq == NULL) { 845 seq = accel_sequence_get(accel_ch); 846 if (spdk_unlikely(seq == NULL)) { 847 return -ENOMEM; 848 } 849 } 850 851 assert(seq->ch == accel_ch); 852 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 853 if (spdk_unlikely(task == NULL)) { 854 if (*pseq == NULL) { 855 accel_sequence_put(seq); 856 } 857 858 return -ENOMEM; 859 } 860 861 task->dst_domain = dst_domain; 862 task->dst_domain_ctx = dst_domain_ctx; 863 task->d.iovs = dst_iovs; 864 task->d.iovcnt = dst_iovcnt; 865 task->src_domain = src_domain; 866 task->src_domain_ctx = src_domain_ctx; 867 task->s.iovs = src_iovs; 868 task->s.iovcnt = src_iovcnt; 869 task->flags = flags; 870 task->op_code = ACCEL_OPC_COPY; 871 872 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 873 *pseq = seq; 874 875 return 0; 876 } 877 878 int 879 spdk_accel_append_fill(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 880 void *buf, uint64_t len, 881 struct spdk_memory_domain *domain, void *domain_ctx, uint8_t pattern, 882 int flags, spdk_accel_step_cb cb_fn, void *cb_arg) 883 { 884 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 885 struct spdk_accel_task *task; 886 struct spdk_accel_sequence *seq = *pseq; 887 888 if (seq == NULL) { 889 seq = accel_sequence_get(accel_ch); 890 if (spdk_unlikely(seq == NULL)) { 891 return -ENOMEM; 892 } 893 } 894 895 assert(seq->ch == accel_ch); 896 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 897 if (spdk_unlikely(task == NULL)) { 898 if (*pseq == NULL) { 899 accel_sequence_put(seq); 900 } 901 902 return -ENOMEM; 903 } 904 905 memset(&task->fill_pattern, pattern, sizeof(uint64_t)); 906 907 task->d.iovs = &task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 908 task->d.iovs[0].iov_base = buf; 909 task->d.iovs[0].iov_len = len; 910 task->d.iovcnt = 1; 911 task->src_domain = NULL; 912 task->dst_domain = domain; 913 task->dst_domain_ctx = domain_ctx; 914 task->flags = flags; 915 task->op_code = ACCEL_OPC_FILL; 916 917 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 918 *pseq = seq; 919 920 return 0; 921 } 922 923 int 924 spdk_accel_append_decompress(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 925 struct iovec *dst_iovs, size_t dst_iovcnt, 926 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 927 struct iovec *src_iovs, size_t src_iovcnt, 928 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 929 int flags, spdk_accel_step_cb cb_fn, void *cb_arg) 930 { 931 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 932 struct spdk_accel_task *task; 933 struct spdk_accel_sequence *seq = *pseq; 934 935 if (seq == NULL) { 936 seq = accel_sequence_get(accel_ch); 937 if (spdk_unlikely(seq == NULL)) { 938 return -ENOMEM; 939 } 940 } 941 942 assert(seq->ch == accel_ch); 943 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 944 if (spdk_unlikely(task == NULL)) { 945 if (*pseq == NULL) { 946 accel_sequence_put(seq); 947 } 948 949 return -ENOMEM; 950 } 951 952 /* TODO: support output_size for chaining */ 953 task->output_size = NULL; 954 task->dst_domain = dst_domain; 955 task->dst_domain_ctx = dst_domain_ctx; 956 task->d.iovs = dst_iovs; 957 task->d.iovcnt = dst_iovcnt; 958 task->src_domain = src_domain; 959 task->src_domain_ctx = src_domain_ctx; 960 task->s.iovs = src_iovs; 961 task->s.iovcnt = src_iovcnt; 962 task->flags = flags; 963 task->op_code = ACCEL_OPC_DECOMPRESS; 964 965 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 966 *pseq = seq; 967 968 return 0; 969 } 970 971 int 972 spdk_accel_append_encrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 973 struct spdk_accel_crypto_key *key, 974 struct iovec *dst_iovs, uint32_t dst_iovcnt, 975 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 976 struct iovec *src_iovs, uint32_t src_iovcnt, 977 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 978 uint64_t iv, uint32_t block_size, int flags, 979 spdk_accel_step_cb cb_fn, void *cb_arg) 980 { 981 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 982 struct spdk_accel_task *task; 983 struct spdk_accel_sequence *seq = *pseq; 984 985 if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || 986 !block_size)) { 987 return -EINVAL; 988 } 989 990 if (seq == NULL) { 991 seq = accel_sequence_get(accel_ch); 992 if (spdk_unlikely(seq == NULL)) { 993 return -ENOMEM; 994 } 995 } 996 997 assert(seq->ch == accel_ch); 998 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 999 if (spdk_unlikely(task == NULL)) { 1000 if (*pseq == NULL) { 1001 accel_sequence_put(seq); 1002 } 1003 1004 return -ENOMEM; 1005 } 1006 1007 task->crypto_key = key; 1008 task->src_domain = src_domain; 1009 task->src_domain_ctx = src_domain_ctx; 1010 task->s.iovs = src_iovs; 1011 task->s.iovcnt = src_iovcnt; 1012 task->dst_domain = dst_domain; 1013 task->dst_domain_ctx = dst_domain_ctx; 1014 task->d.iovs = dst_iovs; 1015 task->d.iovcnt = dst_iovcnt; 1016 task->iv = iv; 1017 task->block_size = block_size; 1018 task->flags = flags; 1019 task->op_code = ACCEL_OPC_ENCRYPT; 1020 1021 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1022 *pseq = seq; 1023 1024 return 0; 1025 } 1026 1027 int 1028 spdk_accel_append_decrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1029 struct spdk_accel_crypto_key *key, 1030 struct iovec *dst_iovs, uint32_t dst_iovcnt, 1031 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 1032 struct iovec *src_iovs, uint32_t src_iovcnt, 1033 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 1034 uint64_t iv, uint32_t block_size, int flags, 1035 spdk_accel_step_cb cb_fn, void *cb_arg) 1036 { 1037 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1038 struct spdk_accel_task *task; 1039 struct spdk_accel_sequence *seq = *pseq; 1040 1041 if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || 1042 !block_size)) { 1043 return -EINVAL; 1044 } 1045 1046 if (seq == NULL) { 1047 seq = accel_sequence_get(accel_ch); 1048 if (spdk_unlikely(seq == NULL)) { 1049 return -ENOMEM; 1050 } 1051 } 1052 1053 assert(seq->ch == accel_ch); 1054 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1055 if (spdk_unlikely(task == NULL)) { 1056 if (*pseq == NULL) { 1057 accel_sequence_put(seq); 1058 } 1059 1060 return -ENOMEM; 1061 } 1062 1063 task->crypto_key = key; 1064 task->src_domain = src_domain; 1065 task->src_domain_ctx = src_domain_ctx; 1066 task->s.iovs = src_iovs; 1067 task->s.iovcnt = src_iovcnt; 1068 task->dst_domain = dst_domain; 1069 task->dst_domain_ctx = dst_domain_ctx; 1070 task->d.iovs = dst_iovs; 1071 task->d.iovcnt = dst_iovcnt; 1072 task->iv = iv; 1073 task->block_size = block_size; 1074 task->flags = flags; 1075 task->op_code = ACCEL_OPC_DECRYPT; 1076 1077 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1078 *pseq = seq; 1079 1080 return 0; 1081 } 1082 1083 int 1084 spdk_accel_get_buf(struct spdk_io_channel *ch, uint64_t len, void **buf, 1085 struct spdk_memory_domain **domain, void **domain_ctx) 1086 { 1087 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1088 struct accel_buffer *accel_buf; 1089 1090 accel_buf = accel_get_buf(accel_ch, len); 1091 if (spdk_unlikely(accel_buf == NULL)) { 1092 return -ENOMEM; 1093 } 1094 1095 /* We always return the same pointer and identify the buffers through domain_ctx */ 1096 *buf = ACCEL_BUFFER_BASE; 1097 *domain_ctx = accel_buf; 1098 *domain = g_accel_domain; 1099 1100 return 0; 1101 } 1102 1103 void 1104 spdk_accel_put_buf(struct spdk_io_channel *ch, void *buf, 1105 struct spdk_memory_domain *domain, void *domain_ctx) 1106 { 1107 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1108 struct accel_buffer *accel_buf = domain_ctx; 1109 1110 assert(domain == g_accel_domain); 1111 assert(buf == ACCEL_BUFFER_BASE); 1112 1113 accel_put_buf(accel_ch, accel_buf); 1114 } 1115 1116 static void 1117 accel_sequence_complete_tasks(struct spdk_accel_sequence *seq) 1118 { 1119 struct spdk_accel_task *task; 1120 struct accel_io_channel *ch = seq->ch; 1121 spdk_accel_step_cb cb_fn; 1122 void *cb_arg; 1123 1124 while (!TAILQ_EMPTY(&seq->completed)) { 1125 task = TAILQ_FIRST(&seq->completed); 1126 TAILQ_REMOVE(&seq->completed, task, seq_link); 1127 cb_fn = task->step_cb_fn; 1128 cb_arg = task->step_cb_arg; 1129 TAILQ_INSERT_HEAD(&ch->task_pool, task, link); 1130 if (cb_fn != NULL) { 1131 cb_fn(cb_arg); 1132 } 1133 } 1134 1135 while (!TAILQ_EMPTY(&seq->tasks)) { 1136 task = TAILQ_FIRST(&seq->tasks); 1137 TAILQ_REMOVE(&seq->tasks, task, seq_link); 1138 cb_fn = task->step_cb_fn; 1139 cb_arg = task->step_cb_arg; 1140 TAILQ_INSERT_HEAD(&ch->task_pool, task, link); 1141 if (cb_fn != NULL) { 1142 cb_fn(cb_arg); 1143 } 1144 } 1145 } 1146 1147 static void 1148 accel_sequence_complete(struct spdk_accel_sequence *seq) 1149 { 1150 SPDK_DEBUGLOG(accel, "Completed sequence: %p with status: %d\n", seq, seq->status); 1151 1152 accel_update_stats(seq->ch, sequence_executed); 1153 if (spdk_unlikely(seq->status != 0)) { 1154 accel_update_stats(seq->ch, sequence_failed); 1155 } 1156 1157 /* First notify all users that appended operations to this sequence */ 1158 accel_sequence_complete_tasks(seq); 1159 1160 /* Then notify the user that finished the sequence */ 1161 seq->cb_fn(seq->cb_arg, seq->status); 1162 1163 accel_sequence_put(seq); 1164 } 1165 1166 static void 1167 accel_update_virt_iov(struct iovec *diov, struct iovec *siov, struct accel_buffer *accel_buf) 1168 { 1169 uintptr_t offset; 1170 1171 offset = (uintptr_t)siov->iov_base & ACCEL_BUFFER_OFFSET_MASK; 1172 assert(offset < accel_buf->len); 1173 1174 diov->iov_base = (char *)accel_buf->buf + offset; 1175 diov->iov_len = siov->iov_len; 1176 } 1177 1178 static void 1179 accel_sequence_set_virtbuf(struct spdk_accel_sequence *seq, struct accel_buffer *buf) 1180 { 1181 struct spdk_accel_task *task; 1182 struct iovec *iov; 1183 1184 /* Now that we've allocated the actual data buffer for this accel_buffer, update all tasks 1185 * in a sequence that were using it. 1186 */ 1187 TAILQ_FOREACH(task, &seq->tasks, seq_link) { 1188 if (task->src_domain == g_accel_domain && task->src_domain_ctx == buf) { 1189 iov = &task->aux_iovs[SPDK_ACCEL_AXU_IOV_VIRT_SRC]; 1190 assert(task->s.iovcnt == 1); 1191 accel_update_virt_iov(iov, &task->s.iovs[0], buf); 1192 task->src_domain = NULL; 1193 task->s.iovs = iov; 1194 } 1195 if (task->dst_domain == g_accel_domain && task->dst_domain_ctx == buf) { 1196 iov = &task->aux_iovs[SPDK_ACCEL_AXU_IOV_VIRT_DST]; 1197 assert(task->d.iovcnt == 1); 1198 accel_update_virt_iov(iov, &task->d.iovs[0], buf); 1199 task->dst_domain = NULL; 1200 task->d.iovs = iov; 1201 } 1202 } 1203 } 1204 1205 static void accel_process_sequence(struct spdk_accel_sequence *seq); 1206 1207 static void 1208 accel_iobuf_get_virtbuf_cb(struct spdk_iobuf_entry *entry, void *buf) 1209 { 1210 struct accel_buffer *accel_buf; 1211 1212 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1213 1214 assert(accel_buf->seq != NULL); 1215 assert(accel_buf->buf == NULL); 1216 accel_buf->buf = buf; 1217 1218 assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF); 1219 accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF); 1220 accel_sequence_set_virtbuf(accel_buf->seq, accel_buf); 1221 accel_process_sequence(accel_buf->seq); 1222 } 1223 1224 static bool 1225 accel_sequence_alloc_buf(struct spdk_accel_sequence *seq, struct accel_buffer *buf, 1226 spdk_iobuf_get_cb cb_fn) 1227 { 1228 struct accel_io_channel *ch = seq->ch; 1229 1230 assert(buf->buf == NULL); 1231 assert(buf->seq == NULL); 1232 1233 buf->seq = seq; 1234 buf->buf = spdk_iobuf_get(&ch->iobuf, buf->len, &buf->iobuf, cb_fn); 1235 if (buf->buf == NULL) { 1236 return false; 1237 } 1238 1239 return true; 1240 } 1241 1242 static bool 1243 accel_sequence_check_virtbuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1244 { 1245 /* If a task doesn't have dst/src (e.g. fill, crc32), its dst/src domain should be set to 1246 * NULL */ 1247 if (task->src_domain == g_accel_domain) { 1248 if (!accel_sequence_alloc_buf(seq, task->src_domain_ctx, 1249 accel_iobuf_get_virtbuf_cb)) { 1250 return false; 1251 } 1252 1253 accel_sequence_set_virtbuf(seq, task->src_domain_ctx); 1254 } 1255 1256 if (task->dst_domain == g_accel_domain) { 1257 if (!accel_sequence_alloc_buf(seq, task->dst_domain_ctx, 1258 accel_iobuf_get_virtbuf_cb)) { 1259 return false; 1260 } 1261 1262 accel_sequence_set_virtbuf(seq, task->dst_domain_ctx); 1263 } 1264 1265 return true; 1266 } 1267 1268 static void 1269 accel_sequence_get_buf_cb(struct spdk_iobuf_entry *entry, void *buf) 1270 { 1271 struct accel_buffer *accel_buf; 1272 1273 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1274 1275 assert(accel_buf->seq != NULL); 1276 assert(accel_buf->buf == NULL); 1277 accel_buf->buf = buf; 1278 1279 accel_sequence_set_virtbuf(accel_buf->seq, accel_buf); 1280 accel_buf->cb_fn(accel_buf->seq, accel_buf->cb_ctx); 1281 } 1282 1283 bool 1284 spdk_accel_alloc_sequence_buf(struct spdk_accel_sequence *seq, void *buf, 1285 struct spdk_memory_domain *domain, void *domain_ctx, 1286 spdk_accel_sequence_get_buf_cb cb_fn, void *cb_ctx) 1287 { 1288 struct accel_buffer *accel_buf = domain_ctx; 1289 1290 assert(domain == g_accel_domain); 1291 accel_buf->cb_fn = cb_fn; 1292 accel_buf->cb_ctx = cb_ctx; 1293 1294 if (!accel_sequence_alloc_buf(seq, accel_buf, accel_sequence_get_buf_cb)) { 1295 return false; 1296 } 1297 1298 accel_sequence_set_virtbuf(seq, accel_buf); 1299 1300 return true; 1301 } 1302 1303 struct spdk_accel_task * 1304 spdk_accel_sequence_first_task(struct spdk_accel_sequence *seq) 1305 { 1306 return TAILQ_FIRST(&seq->tasks); 1307 } 1308 1309 struct spdk_accel_task * 1310 spdk_accel_sequence_next_task(struct spdk_accel_task *task) 1311 { 1312 return TAILQ_NEXT(task, seq_link); 1313 } 1314 1315 static inline uint64_t 1316 accel_get_iovlen(struct iovec *iovs, uint32_t iovcnt) 1317 { 1318 uint64_t result = 0; 1319 uint32_t i; 1320 1321 for (i = 0; i < iovcnt; ++i) { 1322 result += iovs[i].iov_len; 1323 } 1324 1325 return result; 1326 } 1327 1328 static inline void 1329 accel_set_bounce_buffer(struct spdk_accel_bounce_buffer *bounce, struct iovec **iovs, 1330 uint32_t *iovcnt, struct spdk_memory_domain **domain, void **domain_ctx, 1331 struct accel_buffer *buf) 1332 { 1333 bounce->orig_iovs = *iovs; 1334 bounce->orig_iovcnt = *iovcnt; 1335 bounce->orig_domain = *domain; 1336 bounce->orig_domain_ctx = *domain_ctx; 1337 bounce->iov.iov_base = buf->buf; 1338 bounce->iov.iov_len = buf->len; 1339 1340 *iovs = &bounce->iov; 1341 *iovcnt = 1; 1342 *domain = NULL; 1343 } 1344 1345 static void 1346 accel_iobuf_get_src_bounce_cb(struct spdk_iobuf_entry *entry, void *buf) 1347 { 1348 struct spdk_accel_task *task; 1349 struct accel_buffer *accel_buf; 1350 1351 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1352 assert(accel_buf->buf == NULL); 1353 accel_buf->buf = buf; 1354 1355 task = TAILQ_FIRST(&accel_buf->seq->tasks); 1356 assert(task != NULL); 1357 1358 assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF); 1359 accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF); 1360 accel_set_bounce_buffer(&task->bounce.s, &task->s.iovs, &task->s.iovcnt, &task->src_domain, 1361 &task->src_domain_ctx, accel_buf); 1362 accel_process_sequence(accel_buf->seq); 1363 } 1364 1365 static void 1366 accel_iobuf_get_dst_bounce_cb(struct spdk_iobuf_entry *entry, void *buf) 1367 { 1368 struct spdk_accel_task *task; 1369 struct accel_buffer *accel_buf; 1370 1371 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1372 assert(accel_buf->buf == NULL); 1373 accel_buf->buf = buf; 1374 1375 task = TAILQ_FIRST(&accel_buf->seq->tasks); 1376 assert(task != NULL); 1377 1378 assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF); 1379 accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF); 1380 accel_set_bounce_buffer(&task->bounce.d, &task->d.iovs, &task->d.iovcnt, &task->dst_domain, 1381 &task->dst_domain_ctx, accel_buf); 1382 accel_process_sequence(accel_buf->seq); 1383 } 1384 1385 static int 1386 accel_sequence_check_bouncebuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1387 { 1388 struct accel_buffer *buf; 1389 1390 if (task->src_domain != NULL) { 1391 /* By the time we're here, accel buffers should have been allocated */ 1392 assert(task->src_domain != g_accel_domain); 1393 1394 buf = accel_get_buf(seq->ch, accel_get_iovlen(task->s.iovs, task->s.iovcnt)); 1395 if (buf == NULL) { 1396 SPDK_ERRLOG("Couldn't allocate buffer descriptor\n"); 1397 return -ENOMEM; 1398 } 1399 1400 TAILQ_INSERT_TAIL(&seq->bounce_bufs, buf, link); 1401 if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_src_bounce_cb)) { 1402 return -EAGAIN; 1403 } 1404 1405 accel_set_bounce_buffer(&task->bounce.s, &task->s.iovs, &task->s.iovcnt, 1406 &task->src_domain, &task->src_domain_ctx, buf); 1407 } 1408 1409 if (task->dst_domain != NULL) { 1410 /* By the time we're here, accel buffers should have been allocated */ 1411 assert(task->dst_domain != g_accel_domain); 1412 1413 buf = accel_get_buf(seq->ch, accel_get_iovlen(task->d.iovs, task->d.iovcnt)); 1414 if (buf == NULL) { 1415 /* The src buffer will be released when a sequence is completed */ 1416 SPDK_ERRLOG("Couldn't allocate buffer descriptor\n"); 1417 return -ENOMEM; 1418 } 1419 1420 TAILQ_INSERT_TAIL(&seq->bounce_bufs, buf, link); 1421 if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_dst_bounce_cb)) { 1422 return -EAGAIN; 1423 } 1424 1425 accel_set_bounce_buffer(&task->bounce.d, &task->d.iovs, &task->d.iovcnt, 1426 &task->dst_domain, &task->dst_domain_ctx, buf); 1427 } 1428 1429 return 0; 1430 } 1431 1432 static void 1433 accel_task_pull_data_cb(void *ctx, int status) 1434 { 1435 struct spdk_accel_sequence *seq = ctx; 1436 1437 assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA); 1438 if (spdk_likely(status == 0)) { 1439 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK); 1440 } else { 1441 accel_sequence_set_fail(seq, status); 1442 } 1443 1444 accel_process_sequence(seq); 1445 } 1446 1447 static void 1448 accel_task_pull_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1449 { 1450 int rc; 1451 1452 assert(task->bounce.s.orig_iovs != NULL); 1453 assert(task->bounce.s.orig_domain != NULL); 1454 assert(task->bounce.s.orig_domain != g_accel_domain); 1455 assert(!g_modules_opc[task->op_code].supports_memory_domains); 1456 1457 rc = spdk_memory_domain_pull_data(task->bounce.s.orig_domain, 1458 task->bounce.s.orig_domain_ctx, 1459 task->bounce.s.orig_iovs, task->bounce.s.orig_iovcnt, 1460 task->s.iovs, task->s.iovcnt, 1461 accel_task_pull_data_cb, seq); 1462 if (spdk_unlikely(rc != 0)) { 1463 SPDK_ERRLOG("Failed to pull data from memory domain: %s, rc: %d\n", 1464 spdk_memory_domain_get_dma_device_id(task->bounce.s.orig_domain), rc); 1465 accel_sequence_set_fail(seq, rc); 1466 } 1467 } 1468 1469 static void 1470 accel_task_push_data_cb(void *ctx, int status) 1471 { 1472 struct spdk_accel_sequence *seq = ctx; 1473 1474 assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA); 1475 if (spdk_likely(status == 0)) { 1476 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK); 1477 } else { 1478 accel_sequence_set_fail(seq, status); 1479 } 1480 1481 accel_process_sequence(seq); 1482 } 1483 1484 static void 1485 accel_task_push_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1486 { 1487 int rc; 1488 1489 assert(task->bounce.d.orig_iovs != NULL); 1490 assert(task->bounce.d.orig_domain != NULL); 1491 assert(task->bounce.d.orig_domain != g_accel_domain); 1492 assert(!g_modules_opc[task->op_code].supports_memory_domains); 1493 1494 rc = spdk_memory_domain_push_data(task->bounce.d.orig_domain, 1495 task->bounce.d.orig_domain_ctx, 1496 task->bounce.d.orig_iovs, task->bounce.d.orig_iovcnt, 1497 task->d.iovs, task->d.iovcnt, 1498 accel_task_push_data_cb, seq); 1499 if (spdk_unlikely(rc != 0)) { 1500 SPDK_ERRLOG("Failed to push data to memory domain: %s, rc: %d\n", 1501 spdk_memory_domain_get_dma_device_id(task->bounce.s.orig_domain), rc); 1502 accel_sequence_set_fail(seq, rc); 1503 } 1504 } 1505 1506 static void 1507 accel_process_sequence(struct spdk_accel_sequence *seq) 1508 { 1509 struct accel_io_channel *accel_ch = seq->ch; 1510 struct spdk_accel_task *task; 1511 enum accel_sequence_state state; 1512 int rc; 1513 1514 /* Prevent recursive calls to this function */ 1515 if (spdk_unlikely(seq->in_process_sequence)) { 1516 return; 1517 } 1518 seq->in_process_sequence = true; 1519 1520 task = TAILQ_FIRST(&seq->tasks); 1521 assert(task != NULL); 1522 1523 do { 1524 state = seq->state; 1525 switch (state) { 1526 case ACCEL_SEQUENCE_STATE_INIT: 1527 if (g_accel_driver != NULL) { 1528 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_EXEC); 1529 break; 1530 } 1531 /* Fall through */ 1532 case ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF: 1533 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF); 1534 if (!accel_sequence_check_virtbuf(seq, task)) { 1535 /* We couldn't allocate a buffer, wait until one is available */ 1536 break; 1537 } 1538 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF); 1539 /* Fall through */ 1540 case ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF: 1541 /* If a module supports memory domains, we don't need to allocate bounce 1542 * buffers */ 1543 if (g_modules_opc[task->op_code].supports_memory_domains) { 1544 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK); 1545 break; 1546 } 1547 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF); 1548 rc = accel_sequence_check_bouncebuf(seq, task); 1549 if (rc != 0) { 1550 /* We couldn't allocate a buffer, wait until one is available */ 1551 if (rc == -EAGAIN) { 1552 break; 1553 } 1554 accel_sequence_set_fail(seq, rc); 1555 break; 1556 } 1557 if (task->bounce.s.orig_iovs != NULL) { 1558 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PULL_DATA); 1559 break; 1560 } 1561 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK); 1562 /* Fall through */ 1563 case ACCEL_SEQUENCE_STATE_EXEC_TASK: 1564 SPDK_DEBUGLOG(accel, "Executing %s operation, sequence: %p\n", 1565 g_opcode_strings[task->op_code], seq); 1566 1567 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_TASK); 1568 rc = accel_submit_task(accel_ch, task); 1569 if (spdk_unlikely(rc != 0)) { 1570 SPDK_ERRLOG("Failed to submit %s operation, sequence: %p\n", 1571 g_opcode_strings[task->op_code], seq); 1572 accel_sequence_set_fail(seq, rc); 1573 } 1574 break; 1575 case ACCEL_SEQUENCE_STATE_PULL_DATA: 1576 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA); 1577 accel_task_pull_data(seq, task); 1578 break; 1579 case ACCEL_SEQUENCE_STATE_COMPLETE_TASK: 1580 if (task->bounce.d.orig_iovs != NULL) { 1581 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PUSH_DATA); 1582 break; 1583 } 1584 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK); 1585 break; 1586 case ACCEL_SEQUENCE_STATE_PUSH_DATA: 1587 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA); 1588 accel_task_push_data(seq, task); 1589 break; 1590 case ACCEL_SEQUENCE_STATE_NEXT_TASK: 1591 TAILQ_REMOVE(&seq->tasks, task, seq_link); 1592 TAILQ_INSERT_TAIL(&seq->completed, task, seq_link); 1593 /* Check if there are any remaining tasks */ 1594 task = TAILQ_FIRST(&seq->tasks); 1595 if (task == NULL) { 1596 /* Immediately return here to make sure we don't touch the sequence 1597 * after it's completed */ 1598 accel_sequence_complete(seq); 1599 return; 1600 } 1601 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_INIT); 1602 break; 1603 case ACCEL_SEQUENCE_STATE_DRIVER_EXEC: 1604 assert(!TAILQ_EMPTY(&seq->tasks)); 1605 1606 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASK); 1607 rc = g_accel_driver->execute_sequence(seq); 1608 if (spdk_unlikely(rc != 0)) { 1609 SPDK_ERRLOG("Failed to execute sequence: %p using driver: %s\n", 1610 seq, g_accel_driver->name); 1611 accel_sequence_set_fail(seq, rc); 1612 } 1613 break; 1614 case ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE: 1615 task = TAILQ_FIRST(&seq->tasks); 1616 if (task == NULL) { 1617 /* Immediately return here to make sure we don't touch the sequence 1618 * after it's completed */ 1619 accel_sequence_complete(seq); 1620 return; 1621 } 1622 /* We don't want to execute the next task through the driver, so we 1623 * explicitly omit the INIT state here */ 1624 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF); 1625 break; 1626 case ACCEL_SEQUENCE_STATE_ERROR: 1627 /* Immediately return here to make sure we don't touch the sequence 1628 * after it's completed */ 1629 assert(seq->status != 0); 1630 accel_sequence_complete(seq); 1631 return; 1632 case ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF: 1633 case ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF: 1634 case ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA: 1635 case ACCEL_SEQUENCE_STATE_AWAIT_TASK: 1636 case ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA: 1637 case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASK: 1638 break; 1639 default: 1640 assert(0 && "bad state"); 1641 break; 1642 } 1643 } while (seq->state != state); 1644 1645 seq->in_process_sequence = false; 1646 } 1647 1648 static void 1649 accel_sequence_task_cb(void *cb_arg, int status) 1650 { 1651 struct spdk_accel_sequence *seq = cb_arg; 1652 struct spdk_accel_task *task = TAILQ_FIRST(&seq->tasks); 1653 struct accel_io_channel *accel_ch = seq->ch; 1654 1655 /* spdk_accel_task_complete() puts the task back to the task pool, but we don't want to do 1656 * that if a task is part of a sequence. Removing the task from that pool here is the 1657 * easiest way to prevent this, even though it is a bit hacky. 1658 */ 1659 assert(task != NULL); 1660 TAILQ_REMOVE(&accel_ch->task_pool, task, link); 1661 1662 switch (seq->state) { 1663 case ACCEL_SEQUENCE_STATE_AWAIT_TASK: 1664 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_COMPLETE_TASK); 1665 if (spdk_unlikely(status != 0)) { 1666 SPDK_ERRLOG("Failed to execute %s operation, sequence: %p\n", 1667 g_opcode_strings[task->op_code], seq); 1668 accel_sequence_set_fail(seq, status); 1669 } 1670 1671 accel_process_sequence(seq); 1672 break; 1673 case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASK: 1674 assert(g_accel_driver != NULL); 1675 /* Immediately remove the task from the outstanding list to make sure the next call 1676 * to spdk_accel_sequence_first_task() doesn't return it */ 1677 TAILQ_REMOVE(&seq->tasks, task, seq_link); 1678 TAILQ_INSERT_TAIL(&seq->completed, task, seq_link); 1679 1680 if (spdk_unlikely(status != 0)) { 1681 SPDK_ERRLOG("Failed to execute %s operation, sequence: %p through " 1682 "driver: %s\n", g_opcode_strings[task->op_code], seq, 1683 g_accel_driver->name); 1684 /* Update status without using accel_sequence_set_fail() to avoid changing 1685 * seq's state to ERROR until driver calls spdk_accel_sequence_continue() */ 1686 seq->status = status; 1687 } 1688 break; 1689 default: 1690 assert(0 && "bad state"); 1691 break; 1692 } 1693 } 1694 1695 void 1696 spdk_accel_sequence_continue(struct spdk_accel_sequence *seq) 1697 { 1698 assert(g_accel_driver != NULL); 1699 assert(seq->state == ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASK); 1700 1701 if (spdk_likely(seq->status == 0)) { 1702 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE); 1703 } else { 1704 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR); 1705 } 1706 1707 accel_process_sequence(seq); 1708 } 1709 1710 static bool 1711 accel_compare_iovs(struct iovec *iova, uint32_t iovacnt, struct iovec *iovb, uint32_t iovbcnt) 1712 { 1713 /* For now, just do a dumb check that the iovecs arrays are exactly the same */ 1714 if (iovacnt != iovbcnt) { 1715 return false; 1716 } 1717 1718 return memcmp(iova, iovb, sizeof(*iova) * iovacnt) == 0; 1719 } 1720 1721 static void 1722 accel_sequence_merge_tasks(struct spdk_accel_sequence *seq, struct spdk_accel_task *task, 1723 struct spdk_accel_task **next_task) 1724 { 1725 struct spdk_accel_task *next = *next_task; 1726 1727 switch (task->op_code) { 1728 case ACCEL_OPC_COPY: 1729 /* We only allow changing src of operations that actually have a src, e.g. we never 1730 * do it for fill. Theoretically, it is possible, but we'd have to be careful to 1731 * change the src of the operation after fill (which in turn could also be a fill). 1732 * So, for the sake of simplicity, skip this type of operations for now. 1733 */ 1734 if (next->op_code != ACCEL_OPC_DECOMPRESS && 1735 next->op_code != ACCEL_OPC_COPY && 1736 next->op_code != ACCEL_OPC_ENCRYPT && 1737 next->op_code != ACCEL_OPC_DECRYPT) { 1738 break; 1739 } 1740 if (task->dst_domain != next->src_domain) { 1741 break; 1742 } 1743 if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt, 1744 next->s.iovs, next->s.iovcnt)) { 1745 break; 1746 } 1747 next->s.iovs = task->s.iovs; 1748 next->s.iovcnt = task->s.iovcnt; 1749 next->src_domain = task->src_domain; 1750 TAILQ_REMOVE(&seq->tasks, task, seq_link); 1751 TAILQ_INSERT_TAIL(&seq->completed, task, seq_link); 1752 break; 1753 case ACCEL_OPC_DECOMPRESS: 1754 case ACCEL_OPC_FILL: 1755 case ACCEL_OPC_ENCRYPT: 1756 case ACCEL_OPC_DECRYPT: 1757 /* We can only merge tasks when one of them is a copy */ 1758 if (next->op_code != ACCEL_OPC_COPY) { 1759 break; 1760 } 1761 if (task->dst_domain != next->src_domain) { 1762 break; 1763 } 1764 if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt, 1765 next->s.iovs, next->s.iovcnt)) { 1766 break; 1767 } 1768 task->d.iovs = next->d.iovs; 1769 task->d.iovcnt = next->d.iovcnt; 1770 task->dst_domain = next->dst_domain; 1771 /* We're removing next_task from the tasks queue, so we need to update its pointer, 1772 * so that the TAILQ_FOREACH_SAFE() loop below works correctly */ 1773 *next_task = TAILQ_NEXT(next, seq_link); 1774 TAILQ_REMOVE(&seq->tasks, next, seq_link); 1775 TAILQ_INSERT_TAIL(&seq->completed, next, seq_link); 1776 break; 1777 default: 1778 assert(0 && "bad opcode"); 1779 break; 1780 } 1781 } 1782 1783 int 1784 spdk_accel_sequence_finish(struct spdk_accel_sequence *seq, 1785 spdk_accel_completion_cb cb_fn, void *cb_arg) 1786 { 1787 struct spdk_accel_task *task, *next; 1788 1789 /* Try to remove any copy operations if possible */ 1790 TAILQ_FOREACH_SAFE(task, &seq->tasks, seq_link, next) { 1791 if (next == NULL) { 1792 break; 1793 } 1794 accel_sequence_merge_tasks(seq, task, &next); 1795 } 1796 1797 seq->cb_fn = cb_fn; 1798 seq->cb_arg = cb_arg; 1799 1800 accel_process_sequence(seq); 1801 1802 return 0; 1803 } 1804 1805 void 1806 spdk_accel_sequence_reverse(struct spdk_accel_sequence *seq) 1807 { 1808 struct accel_sequence_tasks tasks = TAILQ_HEAD_INITIALIZER(tasks); 1809 struct spdk_accel_task *task; 1810 1811 assert(TAILQ_EMPTY(&seq->completed)); 1812 TAILQ_SWAP(&tasks, &seq->tasks, spdk_accel_task, seq_link); 1813 1814 while (!TAILQ_EMPTY(&tasks)) { 1815 task = TAILQ_FIRST(&tasks); 1816 TAILQ_REMOVE(&tasks, task, seq_link); 1817 TAILQ_INSERT_HEAD(&seq->tasks, task, seq_link); 1818 } 1819 } 1820 1821 void 1822 spdk_accel_sequence_abort(struct spdk_accel_sequence *seq) 1823 { 1824 if (seq == NULL) { 1825 return; 1826 } 1827 1828 accel_sequence_complete_tasks(seq); 1829 accel_sequence_put(seq); 1830 } 1831 1832 struct spdk_memory_domain * 1833 spdk_accel_get_memory_domain(void) 1834 { 1835 return g_accel_domain; 1836 } 1837 1838 static struct spdk_accel_module_if * 1839 _module_find_by_name(const char *name) 1840 { 1841 struct spdk_accel_module_if *accel_module = NULL; 1842 1843 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 1844 if (strcmp(name, accel_module->name) == 0) { 1845 break; 1846 } 1847 } 1848 1849 return accel_module; 1850 } 1851 1852 static inline struct spdk_accel_crypto_key * 1853 _accel_crypto_key_get(const char *name) 1854 { 1855 struct spdk_accel_crypto_key *key; 1856 1857 assert(spdk_spin_held(&g_keyring_spin)); 1858 1859 TAILQ_FOREACH(key, &g_keyring, link) { 1860 if (strcmp(name, key->param.key_name) == 0) { 1861 return key; 1862 } 1863 } 1864 1865 return NULL; 1866 } 1867 1868 static void 1869 accel_crypto_key_free_mem(struct spdk_accel_crypto_key *key) 1870 { 1871 if (key->param.hex_key) { 1872 spdk_memset_s(key->param.hex_key, key->key_size * 2, 0, key->key_size * 2); 1873 free(key->param.hex_key); 1874 } 1875 if (key->param.hex_key2) { 1876 spdk_memset_s(key->param.hex_key2, key->key2_size * 2, 0, key->key2_size * 2); 1877 free(key->param.hex_key2); 1878 } 1879 free(key->param.key_name); 1880 free(key->param.cipher); 1881 if (key->key) { 1882 spdk_memset_s(key->key, key->key_size, 0, key->key_size); 1883 free(key->key); 1884 } 1885 if (key->key2) { 1886 spdk_memset_s(key->key2, key->key2_size, 0, key->key2_size); 1887 free(key->key2); 1888 } 1889 free(key); 1890 } 1891 1892 static void 1893 accel_crypto_key_destroy_unsafe(struct spdk_accel_crypto_key *key) 1894 { 1895 assert(key->module_if); 1896 assert(key->module_if->crypto_key_deinit); 1897 1898 key->module_if->crypto_key_deinit(key); 1899 accel_crypto_key_free_mem(key); 1900 } 1901 1902 /* 1903 * This function mitigates a timing side channel which could be caused by using strcmp() 1904 * Please refer to chapter "Mitigating Information Leakage Based on Variable Timing" in 1905 * the article [1] for more details 1906 * [1] https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/secure-coding/mitigate-timing-side-channel-crypto-implementation.html 1907 */ 1908 static bool 1909 accel_aes_xts_keys_equal(const char *k1, size_t k1_len, const char *k2, size_t k2_len) 1910 { 1911 size_t i; 1912 volatile size_t x = k1_len ^ k2_len; 1913 1914 for (i = 0; ((i < k1_len) & (i < k2_len)); i++) { 1915 x |= k1[i] ^ k2[i]; 1916 } 1917 1918 return x == 0; 1919 } 1920 1921 int 1922 spdk_accel_crypto_key_create(const struct spdk_accel_crypto_key_create_param *param) 1923 { 1924 struct spdk_accel_module_if *module; 1925 struct spdk_accel_crypto_key *key; 1926 size_t hex_key_size, hex_key2_size; 1927 int rc; 1928 1929 if (!param || !param->hex_key || !param->cipher || !param->key_name) { 1930 return -EINVAL; 1931 } 1932 1933 if (g_modules_opc[ACCEL_OPC_ENCRYPT].module != g_modules_opc[ACCEL_OPC_DECRYPT].module) { 1934 /* hardly ever possible, but let's check and warn the user */ 1935 SPDK_ERRLOG("Different accel modules are used for encryption and decryption\n"); 1936 } 1937 module = g_modules_opc[ACCEL_OPC_ENCRYPT].module; 1938 1939 if (!module) { 1940 SPDK_ERRLOG("No accel module found assigned for crypto operation\n"); 1941 return -ENOENT; 1942 } 1943 if (!module->crypto_key_init) { 1944 SPDK_ERRLOG("Accel module \"%s\" doesn't support crypto operations\n", module->name); 1945 return -ENOTSUP; 1946 } 1947 1948 key = calloc(1, sizeof(*key)); 1949 if (!key) { 1950 return -ENOMEM; 1951 } 1952 1953 key->param.key_name = strdup(param->key_name); 1954 if (!key->param.key_name) { 1955 rc = -ENOMEM; 1956 goto error; 1957 } 1958 1959 key->param.cipher = strdup(param->cipher); 1960 if (!key->param.cipher) { 1961 rc = -ENOMEM; 1962 goto error; 1963 } 1964 1965 hex_key_size = strnlen(param->hex_key, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 1966 if (hex_key_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) { 1967 SPDK_ERRLOG("key1 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 1968 rc = -EINVAL; 1969 goto error; 1970 } 1971 key->param.hex_key = strdup(param->hex_key); 1972 if (!key->param.hex_key) { 1973 rc = -ENOMEM; 1974 goto error; 1975 } 1976 1977 key->key_size = hex_key_size / 2; 1978 key->key = spdk_unhexlify(key->param.hex_key); 1979 if (!key->key) { 1980 SPDK_ERRLOG("Failed to unhexlify key1\n"); 1981 rc = -EINVAL; 1982 goto error; 1983 } 1984 1985 if (param->hex_key2) { 1986 hex_key2_size = strnlen(param->hex_key2, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 1987 if (hex_key2_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) { 1988 SPDK_ERRLOG("key2 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 1989 rc = -EINVAL; 1990 goto error; 1991 } 1992 key->param.hex_key2 = strdup(param->hex_key2); 1993 if (!key->param.hex_key2) { 1994 rc = -ENOMEM; 1995 goto error; 1996 } 1997 1998 key->key2_size = hex_key2_size / 2; 1999 key->key2 = spdk_unhexlify(key->param.hex_key2); 2000 if (!key->key2) { 2001 SPDK_ERRLOG("Failed to unhexlify key2\n"); 2002 rc = -EINVAL; 2003 goto error; 2004 } 2005 2006 if (accel_aes_xts_keys_equal(key->key, key->key_size, key->key2, key->key2_size)) { 2007 SPDK_ERRLOG("Identical keys are not secure\n"); 2008 rc = -EINVAL; 2009 goto error; 2010 } 2011 } 2012 2013 key->module_if = module; 2014 2015 spdk_spin_lock(&g_keyring_spin); 2016 if (_accel_crypto_key_get(param->key_name)) { 2017 rc = -EEXIST; 2018 } else { 2019 rc = module->crypto_key_init(key); 2020 if (!rc) { 2021 TAILQ_INSERT_TAIL(&g_keyring, key, link); 2022 } 2023 } 2024 spdk_spin_unlock(&g_keyring_spin); 2025 2026 if (rc) { 2027 goto error; 2028 } 2029 2030 return 0; 2031 2032 error: 2033 accel_crypto_key_free_mem(key); 2034 return rc; 2035 } 2036 2037 int 2038 spdk_accel_crypto_key_destroy(struct spdk_accel_crypto_key *key) 2039 { 2040 if (!key || !key->module_if) { 2041 return -EINVAL; 2042 } 2043 2044 spdk_spin_lock(&g_keyring_spin); 2045 if (!_accel_crypto_key_get(key->param.key_name)) { 2046 spdk_spin_unlock(&g_keyring_spin); 2047 return -ENOENT; 2048 } 2049 TAILQ_REMOVE(&g_keyring, key, link); 2050 spdk_spin_unlock(&g_keyring_spin); 2051 2052 accel_crypto_key_destroy_unsafe(key); 2053 2054 return 0; 2055 } 2056 2057 struct spdk_accel_crypto_key * 2058 spdk_accel_crypto_key_get(const char *name) 2059 { 2060 struct spdk_accel_crypto_key *key; 2061 2062 spdk_spin_lock(&g_keyring_spin); 2063 key = _accel_crypto_key_get(name); 2064 spdk_spin_unlock(&g_keyring_spin); 2065 2066 return key; 2067 } 2068 2069 /* Helper function when accel modules register with the framework. */ 2070 void 2071 spdk_accel_module_list_add(struct spdk_accel_module_if *accel_module) 2072 { 2073 if (_module_find_by_name(accel_module->name)) { 2074 SPDK_NOTICELOG("Accel module %s already registered\n", accel_module->name); 2075 assert(false); 2076 return; 2077 } 2078 2079 /* Make sure that the software module is at the head of the list, this 2080 * will assure that all opcodes are later assigned to software first and 2081 * then updated to HW modules as they are registered. 2082 */ 2083 if (strcmp(accel_module->name, "software") == 0) { 2084 TAILQ_INSERT_HEAD(&spdk_accel_module_list, accel_module, tailq); 2085 } else { 2086 TAILQ_INSERT_TAIL(&spdk_accel_module_list, accel_module, tailq); 2087 } 2088 2089 if (accel_module->get_ctx_size && accel_module->get_ctx_size() > g_max_accel_module_size) { 2090 g_max_accel_module_size = accel_module->get_ctx_size(); 2091 } 2092 } 2093 2094 /* Framework level channel create callback. */ 2095 static int 2096 accel_create_channel(void *io_device, void *ctx_buf) 2097 { 2098 struct accel_io_channel *accel_ch = ctx_buf; 2099 struct spdk_accel_task *accel_task; 2100 struct spdk_accel_sequence *seq; 2101 struct accel_buffer *buf; 2102 uint8_t *task_mem; 2103 int i = 0, j, rc; 2104 2105 accel_ch->task_pool_base = calloc(MAX_TASKS_PER_CHANNEL, g_max_accel_module_size); 2106 if (accel_ch->task_pool_base == NULL) { 2107 return -ENOMEM; 2108 } 2109 2110 accel_ch->seq_pool_base = calloc(MAX_TASKS_PER_CHANNEL, sizeof(struct spdk_accel_sequence)); 2111 if (accel_ch->seq_pool_base == NULL) { 2112 goto err; 2113 } 2114 2115 accel_ch->buf_pool_base = calloc(MAX_TASKS_PER_CHANNEL, sizeof(struct accel_buffer)); 2116 if (accel_ch->buf_pool_base == NULL) { 2117 goto err; 2118 } 2119 2120 TAILQ_INIT(&accel_ch->task_pool); 2121 TAILQ_INIT(&accel_ch->seq_pool); 2122 TAILQ_INIT(&accel_ch->buf_pool); 2123 task_mem = accel_ch->task_pool_base; 2124 for (i = 0 ; i < MAX_TASKS_PER_CHANNEL; i++) { 2125 accel_task = (struct spdk_accel_task *)task_mem; 2126 seq = &accel_ch->seq_pool_base[i]; 2127 buf = &accel_ch->buf_pool_base[i]; 2128 TAILQ_INSERT_TAIL(&accel_ch->task_pool, accel_task, link); 2129 TAILQ_INSERT_TAIL(&accel_ch->seq_pool, seq, link); 2130 TAILQ_INSERT_TAIL(&accel_ch->buf_pool, buf, link); 2131 task_mem += g_max_accel_module_size; 2132 } 2133 2134 /* Assign modules and get IO channels for each */ 2135 for (i = 0; i < ACCEL_OPC_LAST; i++) { 2136 accel_ch->module_ch[i] = g_modules_opc[i].module->get_io_channel(); 2137 /* This can happen if idxd runs out of channels. */ 2138 if (accel_ch->module_ch[i] == NULL) { 2139 goto err; 2140 } 2141 } 2142 2143 rc = spdk_iobuf_channel_init(&accel_ch->iobuf, "accel", g_opts.small_cache_size, 2144 g_opts.large_cache_size); 2145 if (rc != 0) { 2146 SPDK_ERRLOG("Failed to initialize iobuf accel channel\n"); 2147 goto err; 2148 } 2149 2150 return 0; 2151 err: 2152 for (j = 0; j < i; j++) { 2153 spdk_put_io_channel(accel_ch->module_ch[j]); 2154 } 2155 free(accel_ch->task_pool_base); 2156 free(accel_ch->seq_pool_base); 2157 free(accel_ch->buf_pool_base); 2158 2159 return -ENOMEM; 2160 } 2161 2162 /* Framework level channel destroy callback. */ 2163 static void 2164 accel_destroy_channel(void *io_device, void *ctx_buf) 2165 { 2166 struct accel_io_channel *accel_ch = ctx_buf; 2167 int i; 2168 2169 spdk_iobuf_channel_fini(&accel_ch->iobuf); 2170 2171 for (i = 0; i < ACCEL_OPC_LAST; i++) { 2172 assert(accel_ch->module_ch[i] != NULL); 2173 spdk_put_io_channel(accel_ch->module_ch[i]); 2174 accel_ch->module_ch[i] = NULL; 2175 } 2176 2177 free(accel_ch->task_pool_base); 2178 free(accel_ch->seq_pool_base); 2179 free(accel_ch->buf_pool_base); 2180 } 2181 2182 struct spdk_io_channel * 2183 spdk_accel_get_io_channel(void) 2184 { 2185 return spdk_get_io_channel(&spdk_accel_module_list); 2186 } 2187 2188 static void 2189 accel_module_initialize(void) 2190 { 2191 struct spdk_accel_module_if *accel_module; 2192 2193 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 2194 accel_module->module_init(); 2195 } 2196 } 2197 2198 static void 2199 accel_module_init_opcode(enum accel_opcode opcode) 2200 { 2201 struct accel_module *module = &g_modules_opc[opcode]; 2202 struct spdk_accel_module_if *module_if = module->module; 2203 2204 if (module_if->get_memory_domains != NULL) { 2205 module->supports_memory_domains = module_if->get_memory_domains(NULL, 0) > 0; 2206 } 2207 } 2208 2209 int 2210 spdk_accel_initialize(void) 2211 { 2212 enum accel_opcode op; 2213 struct spdk_accel_module_if *accel_module = NULL; 2214 int rc; 2215 2216 rc = spdk_memory_domain_create(&g_accel_domain, SPDK_DMA_DEVICE_TYPE_ACCEL, NULL, 2217 "SPDK_ACCEL_DMA_DEVICE"); 2218 if (rc != 0) { 2219 SPDK_ERRLOG("Failed to create accel memory domain\n"); 2220 return rc; 2221 } 2222 2223 spdk_spin_init(&g_keyring_spin); 2224 2225 g_modules_started = true; 2226 accel_module_initialize(); 2227 2228 /* Create our priority global map of opcodes to modules, we populate starting 2229 * with the software module (guaranteed to be first on the list) and then 2230 * updating opcodes with HW modules that have been initialized. 2231 * NOTE: all opcodes must be supported by software in the event that no HW 2232 * modules are initialized to support the operation. 2233 */ 2234 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 2235 for (op = 0; op < ACCEL_OPC_LAST; op++) { 2236 if (accel_module->supports_opcode(op)) { 2237 g_modules_opc[op].module = accel_module; 2238 SPDK_DEBUGLOG(accel, "OPC 0x%x now assigned to %s\n", op, accel_module->name); 2239 } 2240 } 2241 } 2242 2243 /* Now lets check for overrides and apply all that exist */ 2244 for (op = 0; op < ACCEL_OPC_LAST; op++) { 2245 if (g_modules_opc_override[op] != NULL) { 2246 accel_module = _module_find_by_name(g_modules_opc_override[op]); 2247 if (accel_module == NULL) { 2248 SPDK_ERRLOG("Invalid module name of %s\n", g_modules_opc_override[op]); 2249 rc = -EINVAL; 2250 goto error; 2251 } 2252 if (accel_module->supports_opcode(op) == false) { 2253 SPDK_ERRLOG("Module %s does not support op code %d\n", accel_module->name, op); 2254 rc = -EINVAL; 2255 goto error; 2256 } 2257 g_modules_opc[op].module = accel_module; 2258 } 2259 } 2260 2261 if (g_modules_opc[ACCEL_OPC_ENCRYPT].module != g_modules_opc[ACCEL_OPC_DECRYPT].module) { 2262 SPDK_ERRLOG("Different accel modules are assigned to encrypt and decrypt operations"); 2263 rc = -EINVAL; 2264 goto error; 2265 } 2266 2267 for (op = 0; op < ACCEL_OPC_LAST; op++) { 2268 assert(g_modules_opc[op].module != NULL); 2269 accel_module_init_opcode(op); 2270 } 2271 2272 rc = spdk_iobuf_register_module("accel"); 2273 if (rc != 0) { 2274 SPDK_ERRLOG("Failed to register accel iobuf module\n"); 2275 goto error; 2276 } 2277 2278 /* 2279 * We need a unique identifier for the accel framework, so use the 2280 * spdk_accel_module_list address for this purpose. 2281 */ 2282 spdk_io_device_register(&spdk_accel_module_list, accel_create_channel, accel_destroy_channel, 2283 sizeof(struct accel_io_channel), "accel"); 2284 2285 return 0; 2286 error: 2287 spdk_memory_domain_destroy(g_accel_domain); 2288 2289 return rc; 2290 } 2291 2292 static void 2293 accel_module_finish_cb(void) 2294 { 2295 spdk_accel_fini_cb cb_fn = g_fini_cb_fn; 2296 2297 spdk_memory_domain_destroy(g_accel_domain); 2298 2299 cb_fn(g_fini_cb_arg); 2300 g_fini_cb_fn = NULL; 2301 g_fini_cb_arg = NULL; 2302 } 2303 2304 static void 2305 accel_write_overridden_opc(struct spdk_json_write_ctx *w, const char *opc_str, 2306 const char *module_str) 2307 { 2308 spdk_json_write_object_begin(w); 2309 spdk_json_write_named_string(w, "method", "accel_assign_opc"); 2310 spdk_json_write_named_object_begin(w, "params"); 2311 spdk_json_write_named_string(w, "opname", opc_str); 2312 spdk_json_write_named_string(w, "module", module_str); 2313 spdk_json_write_object_end(w); 2314 spdk_json_write_object_end(w); 2315 } 2316 2317 static void 2318 __accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key) 2319 { 2320 spdk_json_write_named_string(w, "name", key->param.key_name); 2321 spdk_json_write_named_string(w, "cipher", key->param.cipher); 2322 spdk_json_write_named_string(w, "key", key->param.hex_key); 2323 if (key->param.hex_key2) { 2324 spdk_json_write_named_string(w, "key2", key->param.hex_key2); 2325 } 2326 } 2327 2328 void 2329 _accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key) 2330 { 2331 spdk_json_write_object_begin(w); 2332 __accel_crypto_key_dump_param(w, key); 2333 spdk_json_write_object_end(w); 2334 } 2335 2336 static void 2337 _accel_crypto_key_write_config_json(struct spdk_json_write_ctx *w, 2338 struct spdk_accel_crypto_key *key) 2339 { 2340 spdk_json_write_object_begin(w); 2341 spdk_json_write_named_string(w, "method", "accel_crypto_key_create"); 2342 spdk_json_write_named_object_begin(w, "params"); 2343 __accel_crypto_key_dump_param(w, key); 2344 spdk_json_write_object_end(w); 2345 spdk_json_write_object_end(w); 2346 } 2347 2348 static void 2349 accel_write_options(struct spdk_json_write_ctx *w) 2350 { 2351 spdk_json_write_object_begin(w); 2352 spdk_json_write_named_string(w, "method", "accel_set_options"); 2353 spdk_json_write_named_object_begin(w, "params"); 2354 spdk_json_write_named_uint32(w, "small_cache_size", g_opts.small_cache_size); 2355 spdk_json_write_named_uint32(w, "large_cache_size", g_opts.large_cache_size); 2356 spdk_json_write_object_end(w); 2357 spdk_json_write_object_end(w); 2358 } 2359 2360 static void 2361 _accel_crypto_keys_write_config_json(struct spdk_json_write_ctx *w, bool full_dump) 2362 { 2363 struct spdk_accel_crypto_key *key; 2364 2365 spdk_spin_lock(&g_keyring_spin); 2366 TAILQ_FOREACH(key, &g_keyring, link) { 2367 if (full_dump) { 2368 _accel_crypto_key_write_config_json(w, key); 2369 } else { 2370 _accel_crypto_key_dump_param(w, key); 2371 } 2372 } 2373 spdk_spin_unlock(&g_keyring_spin); 2374 } 2375 2376 void 2377 _accel_crypto_keys_dump_param(struct spdk_json_write_ctx *w) 2378 { 2379 _accel_crypto_keys_write_config_json(w, false); 2380 } 2381 2382 void 2383 spdk_accel_write_config_json(struct spdk_json_write_ctx *w) 2384 { 2385 struct spdk_accel_module_if *accel_module; 2386 int i; 2387 2388 spdk_json_write_array_begin(w); 2389 accel_write_options(w); 2390 2391 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 2392 if (accel_module->write_config_json) { 2393 accel_module->write_config_json(w); 2394 } 2395 } 2396 for (i = 0; i < ACCEL_OPC_LAST; i++) { 2397 if (g_modules_opc_override[i]) { 2398 accel_write_overridden_opc(w, g_opcode_strings[i], g_modules_opc_override[i]); 2399 } 2400 } 2401 2402 _accel_crypto_keys_write_config_json(w, true); 2403 2404 spdk_json_write_array_end(w); 2405 } 2406 2407 void 2408 spdk_accel_module_finish(void) 2409 { 2410 if (!g_accel_module) { 2411 g_accel_module = TAILQ_FIRST(&spdk_accel_module_list); 2412 } else { 2413 g_accel_module = TAILQ_NEXT(g_accel_module, tailq); 2414 } 2415 2416 if (!g_accel_module) { 2417 spdk_spin_destroy(&g_keyring_spin); 2418 accel_module_finish_cb(); 2419 return; 2420 } 2421 2422 if (g_accel_module->module_fini) { 2423 spdk_thread_send_msg(spdk_get_thread(), g_accel_module->module_fini, NULL); 2424 } else { 2425 spdk_accel_module_finish(); 2426 } 2427 } 2428 2429 void 2430 spdk_accel_finish(spdk_accel_fini_cb cb_fn, void *cb_arg) 2431 { 2432 struct spdk_accel_crypto_key *key, *key_tmp; 2433 enum accel_opcode op; 2434 2435 assert(cb_fn != NULL); 2436 2437 g_fini_cb_fn = cb_fn; 2438 g_fini_cb_arg = cb_arg; 2439 2440 spdk_spin_lock(&g_keyring_spin); 2441 TAILQ_FOREACH_SAFE(key, &g_keyring, link, key_tmp) { 2442 accel_crypto_key_destroy_unsafe(key); 2443 } 2444 spdk_spin_unlock(&g_keyring_spin); 2445 2446 for (op = 0; op < ACCEL_OPC_LAST; op++) { 2447 if (g_modules_opc_override[op] != NULL) { 2448 free(g_modules_opc_override[op]); 2449 g_modules_opc_override[op] = NULL; 2450 } 2451 g_modules_opc[op].module = NULL; 2452 } 2453 2454 spdk_io_device_unregister(&spdk_accel_module_list, NULL); 2455 spdk_accel_module_finish(); 2456 } 2457 2458 static struct spdk_accel_driver * 2459 accel_find_driver(const char *name) 2460 { 2461 struct spdk_accel_driver *driver; 2462 2463 TAILQ_FOREACH(driver, &g_accel_drivers, tailq) { 2464 if (strcmp(driver->name, name) == 0) { 2465 return driver; 2466 } 2467 } 2468 2469 return NULL; 2470 } 2471 2472 int 2473 spdk_accel_set_driver(const char *name) 2474 { 2475 struct spdk_accel_driver *driver; 2476 2477 driver = accel_find_driver(name); 2478 if (driver == NULL) { 2479 SPDK_ERRLOG("Couldn't find driver named '%s'\n", name); 2480 return -ENODEV; 2481 } 2482 2483 g_accel_driver = driver; 2484 2485 return 0; 2486 } 2487 2488 void 2489 spdk_accel_driver_register(struct spdk_accel_driver *driver) 2490 { 2491 if (accel_find_driver(driver->name)) { 2492 SPDK_ERRLOG("Driver named '%s' has already been registered\n", driver->name); 2493 assert(0); 2494 return; 2495 } 2496 2497 TAILQ_INSERT_TAIL(&g_accel_drivers, driver, tailq); 2498 } 2499 2500 int 2501 spdk_accel_set_opts(const struct spdk_accel_opts *opts) 2502 { 2503 if (opts->size > sizeof(*opts)) { 2504 return -EINVAL; 2505 } 2506 2507 memcpy(&g_opts, opts, opts->size); 2508 2509 return 0; 2510 } 2511 2512 void 2513 spdk_accel_get_opts(struct spdk_accel_opts *opts) 2514 { 2515 size_t size = opts->size; 2516 2517 assert(size <= sizeof(*opts)); 2518 2519 memcpy(opts, &g_opts, spdk_min(sizeof(*opts), size)); 2520 opts->size = size; 2521 } 2522 2523 struct accel_get_stats_ctx { 2524 struct accel_stats stats; 2525 accel_get_stats_cb cb_fn; 2526 void *cb_arg; 2527 }; 2528 2529 static void 2530 accel_get_channel_stats_done(struct spdk_io_channel_iter *iter, int status) 2531 { 2532 struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter); 2533 2534 ctx->cb_fn(&ctx->stats, ctx->cb_arg); 2535 free(ctx); 2536 } 2537 2538 static void 2539 accel_get_channel_stats(struct spdk_io_channel_iter *iter) 2540 { 2541 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(iter); 2542 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 2543 struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter); 2544 struct accel_stats *stats = &ctx->stats; 2545 int i; 2546 2547 stats->sequence_executed += accel_ch->stats.sequence_executed; 2548 stats->sequence_failed += accel_ch->stats.sequence_failed; 2549 for (i = 0; i < ACCEL_OPC_LAST; ++i) { 2550 stats->operations[i].executed += accel_ch->stats.operations[i].executed; 2551 stats->operations[i].failed += accel_ch->stats.operations[i].failed; 2552 } 2553 2554 spdk_for_each_channel_continue(iter, 0); 2555 } 2556 2557 int 2558 accel_get_stats(accel_get_stats_cb cb_fn, void *cb_arg) 2559 { 2560 struct accel_get_stats_ctx *ctx; 2561 2562 ctx = calloc(1, sizeof(*ctx)); 2563 if (ctx == NULL) { 2564 return -ENOMEM; 2565 } 2566 2567 ctx->cb_fn = cb_fn; 2568 ctx->cb_arg = cb_arg; 2569 2570 spdk_for_each_channel(&spdk_accel_module_list, accel_get_channel_stats, ctx, 2571 accel_get_channel_stats_done); 2572 2573 return 0; 2574 } 2575 2576 SPDK_LOG_REGISTER_COMPONENT(accel) 2577