1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES. 4 * All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/accel_module.h" 10 11 #include "accel_internal.h" 12 13 #include "spdk/dma.h" 14 #include "spdk/env.h" 15 #include "spdk/likely.h" 16 #include "spdk/log.h" 17 #include "spdk/thread.h" 18 #include "spdk/json.h" 19 #include "spdk/crc32.h" 20 #include "spdk/util.h" 21 #include "spdk/hexlify.h" 22 23 /* Accelerator Framework: The following provides a top level 24 * generic API for the accelerator functions defined here. Modules, 25 * such as the one in /module/accel/ioat, supply the implementation 26 * with the exception of the pure software implementation contained 27 * later in this file. 28 */ 29 30 #define ALIGN_4K 0x1000 31 #define MAX_TASKS_PER_CHANNEL 0x800 32 #define ACCEL_SMALL_CACHE_SIZE 128 33 #define ACCEL_LARGE_CACHE_SIZE 16 34 /* Set MSB, so we don't return NULL pointers as buffers */ 35 #define ACCEL_BUFFER_BASE ((void *)(1ull << 63)) 36 #define ACCEL_BUFFER_OFFSET_MASK ((uintptr_t)ACCEL_BUFFER_BASE - 1) 37 38 #define ACCEL_CRYPTO_TWEAK_MODE_DEFAULT SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA 39 40 struct accel_module { 41 struct spdk_accel_module_if *module; 42 bool supports_memory_domains; 43 }; 44 45 /* Largest context size for all accel modules */ 46 static size_t g_max_accel_module_size = sizeof(struct spdk_accel_task); 47 48 static struct spdk_accel_module_if *g_accel_module = NULL; 49 static spdk_accel_fini_cb g_fini_cb_fn = NULL; 50 static void *g_fini_cb_arg = NULL; 51 static bool g_modules_started = false; 52 static struct spdk_memory_domain *g_accel_domain; 53 54 /* Global list of registered accelerator modules */ 55 static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list = 56 TAILQ_HEAD_INITIALIZER(spdk_accel_module_list); 57 58 /* Crypto keyring */ 59 static TAILQ_HEAD(, spdk_accel_crypto_key) g_keyring = TAILQ_HEAD_INITIALIZER(g_keyring); 60 static struct spdk_spinlock g_keyring_spin; 61 62 /* Global array mapping capabilities to modules */ 63 static struct accel_module g_modules_opc[SPDK_ACCEL_OPC_LAST] = {}; 64 static char *g_modules_opc_override[SPDK_ACCEL_OPC_LAST] = {}; 65 TAILQ_HEAD(, spdk_accel_driver) g_accel_drivers = TAILQ_HEAD_INITIALIZER(g_accel_drivers); 66 static struct spdk_accel_driver *g_accel_driver; 67 static struct spdk_accel_opts g_opts = { 68 .small_cache_size = ACCEL_SMALL_CACHE_SIZE, 69 .large_cache_size = ACCEL_LARGE_CACHE_SIZE, 70 .task_count = MAX_TASKS_PER_CHANNEL, 71 .sequence_count = MAX_TASKS_PER_CHANNEL, 72 .buf_count = MAX_TASKS_PER_CHANNEL, 73 }; 74 static struct accel_stats g_stats; 75 static struct spdk_spinlock g_stats_lock; 76 77 static const char *g_opcode_strings[SPDK_ACCEL_OPC_LAST] = { 78 "copy", "fill", "dualcast", "compare", "crc32c", "copy_crc32c", 79 "compress", "decompress", "encrypt", "decrypt", "xor" 80 }; 81 82 enum accel_sequence_state { 83 ACCEL_SEQUENCE_STATE_INIT, 84 ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF, 85 ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF, 86 ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF, 87 ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF, 88 ACCEL_SEQUENCE_STATE_PULL_DATA, 89 ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA, 90 ACCEL_SEQUENCE_STATE_EXEC_TASK, 91 ACCEL_SEQUENCE_STATE_AWAIT_TASK, 92 ACCEL_SEQUENCE_STATE_COMPLETE_TASK, 93 ACCEL_SEQUENCE_STATE_NEXT_TASK, 94 ACCEL_SEQUENCE_STATE_PUSH_DATA, 95 ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA, 96 ACCEL_SEQUENCE_STATE_DRIVER_EXEC, 97 ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASK, 98 ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE, 99 ACCEL_SEQUENCE_STATE_ERROR, 100 ACCEL_SEQUENCE_STATE_MAX, 101 }; 102 103 static const char *g_seq_states[] 104 __attribute__((unused)) = { 105 [ACCEL_SEQUENCE_STATE_INIT] = "init", 106 [ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF] = "check-virtbuf", 107 [ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF] = "await-virtbuf", 108 [ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF] = "check-bouncebuf", 109 [ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF] = "await-bouncebuf", 110 [ACCEL_SEQUENCE_STATE_PULL_DATA] = "pull-data", 111 [ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA] = "await-pull-data", 112 [ACCEL_SEQUENCE_STATE_EXEC_TASK] = "exec-task", 113 [ACCEL_SEQUENCE_STATE_AWAIT_TASK] = "await-task", 114 [ACCEL_SEQUENCE_STATE_COMPLETE_TASK] = "complete-task", 115 [ACCEL_SEQUENCE_STATE_NEXT_TASK] = "next-task", 116 [ACCEL_SEQUENCE_STATE_PUSH_DATA] = "push-data", 117 [ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA] = "await-push-data", 118 [ACCEL_SEQUENCE_STATE_DRIVER_EXEC] = "driver-exec", 119 [ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASK] = "driver-await-task", 120 [ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE] = "driver-complete", 121 [ACCEL_SEQUENCE_STATE_ERROR] = "error", 122 [ACCEL_SEQUENCE_STATE_MAX] = "", 123 }; 124 125 #define ACCEL_SEQUENCE_STATE_STRING(s) \ 126 (((s) >= ACCEL_SEQUENCE_STATE_INIT && (s) < ACCEL_SEQUENCE_STATE_MAX) \ 127 ? g_seq_states[s] : "unknown") 128 129 struct accel_buffer { 130 struct spdk_accel_sequence *seq; 131 void *buf; 132 uint64_t len; 133 struct spdk_iobuf_entry iobuf; 134 spdk_accel_sequence_get_buf_cb cb_fn; 135 void *cb_ctx; 136 TAILQ_ENTRY(accel_buffer) link; 137 }; 138 139 struct accel_io_channel { 140 struct spdk_io_channel *module_ch[SPDK_ACCEL_OPC_LAST]; 141 struct spdk_io_channel *driver_channel; 142 void *task_pool_base; 143 struct spdk_accel_sequence *seq_pool_base; 144 struct accel_buffer *buf_pool_base; 145 TAILQ_HEAD(, spdk_accel_task) task_pool; 146 TAILQ_HEAD(, spdk_accel_sequence) seq_pool; 147 TAILQ_HEAD(, accel_buffer) buf_pool; 148 struct spdk_iobuf_channel iobuf; 149 struct accel_stats stats; 150 }; 151 152 TAILQ_HEAD(accel_sequence_tasks, spdk_accel_task); 153 154 struct spdk_accel_sequence { 155 struct accel_io_channel *ch; 156 struct accel_sequence_tasks tasks; 157 struct accel_sequence_tasks completed; 158 TAILQ_HEAD(, accel_buffer) bounce_bufs; 159 int status; 160 /* state uses enum accel_sequence_state */ 161 uint8_t state; 162 bool in_process_sequence; 163 spdk_accel_completion_cb cb_fn; 164 void *cb_arg; 165 TAILQ_ENTRY(spdk_accel_sequence) link; 166 }; 167 168 #define accel_update_stats(ch, event, v) \ 169 do { \ 170 (ch)->stats.event += (v); \ 171 } while (0) 172 173 #define accel_update_task_stats(ch, task, event, v) \ 174 accel_update_stats(ch, operations[(task)->op_code].event, v) 175 176 static inline void 177 accel_sequence_set_state(struct spdk_accel_sequence *seq, enum accel_sequence_state state) 178 { 179 SPDK_DEBUGLOG(accel, "seq=%p, setting state: %s -> %s\n", seq, 180 ACCEL_SEQUENCE_STATE_STRING(seq->state), ACCEL_SEQUENCE_STATE_STRING(state)); 181 assert(seq->state != ACCEL_SEQUENCE_STATE_ERROR || state == ACCEL_SEQUENCE_STATE_ERROR); 182 seq->state = state; 183 } 184 185 static void 186 accel_sequence_set_fail(struct spdk_accel_sequence *seq, int status) 187 { 188 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR); 189 assert(status != 0); 190 seq->status = status; 191 } 192 193 int 194 spdk_accel_get_opc_module_name(enum spdk_accel_opcode opcode, const char **module_name) 195 { 196 if (opcode >= SPDK_ACCEL_OPC_LAST) { 197 /* invalid opcode */ 198 return -EINVAL; 199 } 200 201 if (g_modules_opc[opcode].module) { 202 *module_name = g_modules_opc[opcode].module->name; 203 } else { 204 return -ENOENT; 205 } 206 207 return 0; 208 } 209 210 void 211 _accel_for_each_module(struct module_info *info, _accel_for_each_module_fn fn) 212 { 213 struct spdk_accel_module_if *accel_module; 214 enum spdk_accel_opcode opcode; 215 int j = 0; 216 217 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 218 for (opcode = 0; opcode < SPDK_ACCEL_OPC_LAST; opcode++) { 219 if (accel_module->supports_opcode(opcode)) { 220 info->ops[j] = opcode; 221 j++; 222 } 223 } 224 info->name = accel_module->name; 225 info->num_ops = j; 226 fn(info); 227 j = 0; 228 } 229 } 230 231 int 232 _accel_get_opc_name(enum spdk_accel_opcode opcode, const char **opcode_name) 233 { 234 int rc = 0; 235 236 if (opcode < SPDK_ACCEL_OPC_LAST) { 237 *opcode_name = g_opcode_strings[opcode]; 238 } else { 239 /* invalid opcode */ 240 rc = -EINVAL; 241 } 242 243 return rc; 244 } 245 246 int 247 spdk_accel_assign_opc(enum spdk_accel_opcode opcode, const char *name) 248 { 249 char *copy; 250 251 if (g_modules_started == true) { 252 /* we don't allow re-assignment once things have started */ 253 return -EINVAL; 254 } 255 256 if (opcode >= SPDK_ACCEL_OPC_LAST) { 257 /* invalid opcode */ 258 return -EINVAL; 259 } 260 261 copy = strdup(name); 262 if (copy == NULL) { 263 return -ENOMEM; 264 } 265 266 /* module selection will be validated after the framework starts. */ 267 g_modules_opc_override[opcode] = copy; 268 269 return 0; 270 } 271 272 void 273 spdk_accel_task_complete(struct spdk_accel_task *accel_task, int status) 274 { 275 struct accel_io_channel *accel_ch = accel_task->accel_ch; 276 spdk_accel_completion_cb cb_fn = accel_task->cb_fn; 277 void *cb_arg = accel_task->cb_arg; 278 279 /* We should put the accel_task into the list firstly in order to avoid 280 * the accel task list is exhausted when there is recursive call to 281 * allocate accel_task in user's call back function (cb_fn) 282 */ 283 TAILQ_INSERT_HEAD(&accel_ch->task_pool, accel_task, link); 284 accel_task->seq = NULL; 285 286 accel_update_task_stats(accel_ch, accel_task, executed, 1); 287 accel_update_task_stats(accel_ch, accel_task, num_bytes, accel_task->nbytes); 288 if (spdk_unlikely(status != 0)) { 289 accel_update_task_stats(accel_ch, accel_task, failed, 1); 290 } 291 292 cb_fn(cb_arg, status); 293 } 294 295 inline static struct spdk_accel_task * 296 _get_task(struct accel_io_channel *accel_ch, spdk_accel_completion_cb cb_fn, void *cb_arg) 297 { 298 struct spdk_accel_task *accel_task; 299 300 accel_task = TAILQ_FIRST(&accel_ch->task_pool); 301 if (spdk_unlikely(accel_task == NULL)) { 302 accel_update_stats(accel_ch, retry.task, 1); 303 return NULL; 304 } 305 306 TAILQ_REMOVE(&accel_ch->task_pool, accel_task, link); 307 accel_task->link.tqe_next = NULL; 308 accel_task->link.tqe_prev = NULL; 309 310 accel_task->cb_fn = cb_fn; 311 accel_task->cb_arg = cb_arg; 312 accel_task->accel_ch = accel_ch; 313 accel_task->s.iovs = NULL; 314 accel_task->d.iovs = NULL; 315 316 return accel_task; 317 } 318 319 static inline int 320 accel_submit_task(struct accel_io_channel *accel_ch, struct spdk_accel_task *task) 321 { 322 struct spdk_io_channel *module_ch = accel_ch->module_ch[task->op_code]; 323 struct spdk_accel_module_if *module = g_modules_opc[task->op_code].module; 324 int rc; 325 326 rc = module->submit_tasks(module_ch, task); 327 if (spdk_unlikely(rc != 0)) { 328 accel_update_task_stats(accel_ch, task, failed, 1); 329 } 330 331 return rc; 332 } 333 334 static inline uint64_t 335 accel_get_iovlen(struct iovec *iovs, uint32_t iovcnt) 336 { 337 uint64_t result = 0; 338 uint32_t i; 339 340 for (i = 0; i < iovcnt; ++i) { 341 result += iovs[i].iov_len; 342 } 343 344 return result; 345 } 346 347 /* Accel framework public API for copy function */ 348 int 349 spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, 350 uint64_t nbytes, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) 351 { 352 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 353 struct spdk_accel_task *accel_task; 354 355 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 356 if (spdk_unlikely(accel_task == NULL)) { 357 return -ENOMEM; 358 } 359 360 accel_task->s.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_SRC]; 361 accel_task->d.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 362 accel_task->d.iovs[0].iov_base = dst; 363 accel_task->d.iovs[0].iov_len = nbytes; 364 accel_task->d.iovcnt = 1; 365 accel_task->s.iovs[0].iov_base = src; 366 accel_task->s.iovs[0].iov_len = nbytes; 367 accel_task->s.iovcnt = 1; 368 accel_task->nbytes = nbytes; 369 accel_task->op_code = SPDK_ACCEL_OPC_COPY; 370 accel_task->flags = flags; 371 accel_task->src_domain = NULL; 372 accel_task->dst_domain = NULL; 373 accel_task->step_cb_fn = NULL; 374 375 return accel_submit_task(accel_ch, accel_task); 376 } 377 378 /* Accel framework public API for dual cast copy function */ 379 int 380 spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, 381 void *dst2, void *src, uint64_t nbytes, int flags, 382 spdk_accel_completion_cb cb_fn, void *cb_arg) 383 { 384 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 385 struct spdk_accel_task *accel_task; 386 387 if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) { 388 SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n"); 389 return -EINVAL; 390 } 391 392 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 393 if (spdk_unlikely(accel_task == NULL)) { 394 return -ENOMEM; 395 } 396 397 accel_task->s.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_SRC]; 398 accel_task->d.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 399 accel_task->d2.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST2]; 400 accel_task->d.iovs[0].iov_base = dst1; 401 accel_task->d.iovs[0].iov_len = nbytes; 402 accel_task->d.iovcnt = 1; 403 accel_task->d2.iovs[0].iov_base = dst2; 404 accel_task->d2.iovs[0].iov_len = nbytes; 405 accel_task->d2.iovcnt = 1; 406 accel_task->s.iovs[0].iov_base = src; 407 accel_task->s.iovs[0].iov_len = nbytes; 408 accel_task->s.iovcnt = 1; 409 accel_task->nbytes = nbytes; 410 accel_task->flags = flags; 411 accel_task->op_code = SPDK_ACCEL_OPC_DUALCAST; 412 accel_task->src_domain = NULL; 413 accel_task->dst_domain = NULL; 414 accel_task->step_cb_fn = NULL; 415 416 return accel_submit_task(accel_ch, accel_task); 417 } 418 419 /* Accel framework public API for compare function */ 420 int 421 spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, 422 void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn, 423 void *cb_arg) 424 { 425 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 426 struct spdk_accel_task *accel_task; 427 428 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 429 if (spdk_unlikely(accel_task == NULL)) { 430 return -ENOMEM; 431 } 432 433 accel_task->s.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_SRC]; 434 accel_task->s2.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_SRC2]; 435 accel_task->s.iovs[0].iov_base = src1; 436 accel_task->s.iovs[0].iov_len = nbytes; 437 accel_task->s.iovcnt = 1; 438 accel_task->s2.iovs[0].iov_base = src2; 439 accel_task->s2.iovs[0].iov_len = nbytes; 440 accel_task->s2.iovcnt = 1; 441 accel_task->nbytes = nbytes; 442 accel_task->op_code = SPDK_ACCEL_OPC_COMPARE; 443 accel_task->src_domain = NULL; 444 accel_task->dst_domain = NULL; 445 accel_task->step_cb_fn = NULL; 446 447 return accel_submit_task(accel_ch, accel_task); 448 } 449 450 /* Accel framework public API for fill function */ 451 int 452 spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, 453 uint8_t fill, uint64_t nbytes, int flags, 454 spdk_accel_completion_cb cb_fn, void *cb_arg) 455 { 456 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 457 struct spdk_accel_task *accel_task; 458 459 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 460 if (spdk_unlikely(accel_task == NULL)) { 461 return -ENOMEM; 462 } 463 464 accel_task->d.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 465 accel_task->d.iovs[0].iov_base = dst; 466 accel_task->d.iovs[0].iov_len = nbytes; 467 accel_task->d.iovcnt = 1; 468 accel_task->nbytes = nbytes; 469 memset(&accel_task->fill_pattern, fill, sizeof(uint64_t)); 470 accel_task->flags = flags; 471 accel_task->op_code = SPDK_ACCEL_OPC_FILL; 472 accel_task->src_domain = NULL; 473 accel_task->dst_domain = NULL; 474 accel_task->step_cb_fn = NULL; 475 476 return accel_submit_task(accel_ch, accel_task); 477 } 478 479 /* Accel framework public API for CRC-32C function */ 480 int 481 spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, 482 void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn, 483 void *cb_arg) 484 { 485 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 486 struct spdk_accel_task *accel_task; 487 488 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 489 if (spdk_unlikely(accel_task == NULL)) { 490 return -ENOMEM; 491 } 492 493 accel_task->s.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_SRC]; 494 accel_task->s.iovs[0].iov_base = src; 495 accel_task->s.iovs[0].iov_len = nbytes; 496 accel_task->s.iovcnt = 1; 497 accel_task->nbytes = nbytes; 498 accel_task->crc_dst = crc_dst; 499 accel_task->seed = seed; 500 accel_task->op_code = SPDK_ACCEL_OPC_CRC32C; 501 accel_task->src_domain = NULL; 502 accel_task->dst_domain = NULL; 503 accel_task->step_cb_fn = NULL; 504 505 return accel_submit_task(accel_ch, accel_task); 506 } 507 508 /* Accel framework public API for chained CRC-32C function */ 509 int 510 spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, 511 struct iovec *iov, uint32_t iov_cnt, uint32_t seed, 512 spdk_accel_completion_cb cb_fn, void *cb_arg) 513 { 514 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 515 struct spdk_accel_task *accel_task; 516 517 if (iov == NULL) { 518 SPDK_ERRLOG("iov should not be NULL"); 519 return -EINVAL; 520 } 521 522 if (!iov_cnt) { 523 SPDK_ERRLOG("iovcnt should not be zero value\n"); 524 return -EINVAL; 525 } 526 527 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 528 if (spdk_unlikely(accel_task == NULL)) { 529 SPDK_ERRLOG("no memory\n"); 530 assert(0); 531 return -ENOMEM; 532 } 533 534 accel_task->s.iovs = iov; 535 accel_task->s.iovcnt = iov_cnt; 536 accel_task->nbytes = accel_get_iovlen(iov, iov_cnt); 537 accel_task->crc_dst = crc_dst; 538 accel_task->seed = seed; 539 accel_task->op_code = SPDK_ACCEL_OPC_CRC32C; 540 accel_task->src_domain = NULL; 541 accel_task->dst_domain = NULL; 542 accel_task->step_cb_fn = NULL; 543 544 return accel_submit_task(accel_ch, accel_task); 545 } 546 547 /* Accel framework public API for copy with CRC-32C function */ 548 int 549 spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, 550 void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes, 551 int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) 552 { 553 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 554 struct spdk_accel_task *accel_task; 555 556 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 557 if (spdk_unlikely(accel_task == NULL)) { 558 return -ENOMEM; 559 } 560 561 accel_task->s.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_SRC]; 562 accel_task->d.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 563 accel_task->d.iovs[0].iov_base = dst; 564 accel_task->d.iovs[0].iov_len = nbytes; 565 accel_task->d.iovcnt = 1; 566 accel_task->s.iovs[0].iov_base = src; 567 accel_task->s.iovs[0].iov_len = nbytes; 568 accel_task->s.iovcnt = 1; 569 accel_task->nbytes = nbytes; 570 accel_task->crc_dst = crc_dst; 571 accel_task->seed = seed; 572 accel_task->flags = flags; 573 accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C; 574 accel_task->src_domain = NULL; 575 accel_task->dst_domain = NULL; 576 accel_task->step_cb_fn = NULL; 577 578 return accel_submit_task(accel_ch, accel_task); 579 } 580 581 /* Accel framework public API for chained copy + CRC-32C function */ 582 int 583 spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, 584 struct iovec *src_iovs, uint32_t iov_cnt, uint32_t *crc_dst, 585 uint32_t seed, int flags, spdk_accel_completion_cb cb_fn, void *cb_arg) 586 { 587 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 588 struct spdk_accel_task *accel_task; 589 uint64_t nbytes; 590 591 if (src_iovs == NULL) { 592 SPDK_ERRLOG("iov should not be NULL"); 593 return -EINVAL; 594 } 595 596 if (!iov_cnt) { 597 SPDK_ERRLOG("iovcnt should not be zero value\n"); 598 return -EINVAL; 599 } 600 601 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 602 if (spdk_unlikely(accel_task == NULL)) { 603 SPDK_ERRLOG("no memory\n"); 604 assert(0); 605 return -ENOMEM; 606 } 607 608 nbytes = accel_get_iovlen(src_iovs, iov_cnt); 609 accel_task->d.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 610 accel_task->d.iovs[0].iov_base = dst; 611 accel_task->d.iovs[0].iov_len = nbytes; 612 accel_task->d.iovcnt = 1; 613 accel_task->s.iovs = src_iovs; 614 accel_task->s.iovcnt = iov_cnt; 615 accel_task->nbytes = nbytes; 616 accel_task->crc_dst = crc_dst; 617 accel_task->seed = seed; 618 accel_task->flags = flags; 619 accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C; 620 accel_task->src_domain = NULL; 621 accel_task->dst_domain = NULL; 622 accel_task->step_cb_fn = NULL; 623 624 return accel_submit_task(accel_ch, accel_task); 625 } 626 627 int 628 spdk_accel_submit_compress(struct spdk_io_channel *ch, void *dst, uint64_t nbytes, 629 struct iovec *src_iovs, size_t src_iovcnt, uint32_t *output_size, int flags, 630 spdk_accel_completion_cb cb_fn, void *cb_arg) 631 { 632 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 633 struct spdk_accel_task *accel_task; 634 635 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 636 if (spdk_unlikely(accel_task == NULL)) { 637 return -ENOMEM; 638 } 639 640 accel_task->d.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 641 accel_task->d.iovs[0].iov_base = dst; 642 accel_task->d.iovs[0].iov_len = nbytes; 643 accel_task->d.iovcnt = 1; 644 accel_task->output_size = output_size; 645 accel_task->s.iovs = src_iovs; 646 accel_task->s.iovcnt = src_iovcnt; 647 accel_task->nbytes = nbytes; 648 accel_task->flags = flags; 649 accel_task->op_code = SPDK_ACCEL_OPC_COMPRESS; 650 accel_task->src_domain = NULL; 651 accel_task->dst_domain = NULL; 652 accel_task->step_cb_fn = NULL; 653 654 return accel_submit_task(accel_ch, accel_task); 655 } 656 657 int 658 spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_iovs, 659 size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt, 660 uint32_t *output_size, int flags, spdk_accel_completion_cb cb_fn, 661 void *cb_arg) 662 { 663 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 664 struct spdk_accel_task *accel_task; 665 666 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 667 if (spdk_unlikely(accel_task == NULL)) { 668 return -ENOMEM; 669 } 670 671 accel_task->output_size = output_size; 672 accel_task->s.iovs = src_iovs; 673 accel_task->s.iovcnt = src_iovcnt; 674 accel_task->d.iovs = dst_iovs; 675 accel_task->d.iovcnt = dst_iovcnt; 676 accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 677 accel_task->flags = flags; 678 accel_task->op_code = SPDK_ACCEL_OPC_DECOMPRESS; 679 accel_task->src_domain = NULL; 680 accel_task->dst_domain = NULL; 681 accel_task->step_cb_fn = NULL; 682 683 return accel_submit_task(accel_ch, accel_task); 684 } 685 686 int 687 spdk_accel_submit_encrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key, 688 struct iovec *dst_iovs, uint32_t dst_iovcnt, 689 struct iovec *src_iovs, uint32_t src_iovcnt, 690 uint64_t iv, uint32_t block_size, int flags, 691 spdk_accel_completion_cb cb_fn, void *cb_arg) 692 { 693 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 694 struct spdk_accel_task *accel_task; 695 696 if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) { 697 return -EINVAL; 698 } 699 700 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 701 if (spdk_unlikely(accel_task == NULL)) { 702 return -ENOMEM; 703 } 704 705 accel_task->crypto_key = key; 706 accel_task->s.iovs = src_iovs; 707 accel_task->s.iovcnt = src_iovcnt; 708 accel_task->d.iovs = dst_iovs; 709 accel_task->d.iovcnt = dst_iovcnt; 710 accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 711 accel_task->iv = iv; 712 accel_task->block_size = block_size; 713 accel_task->flags = flags; 714 accel_task->op_code = SPDK_ACCEL_OPC_ENCRYPT; 715 accel_task->src_domain = NULL; 716 accel_task->dst_domain = NULL; 717 accel_task->step_cb_fn = NULL; 718 719 return accel_submit_task(accel_ch, accel_task); 720 } 721 722 int 723 spdk_accel_submit_decrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key, 724 struct iovec *dst_iovs, uint32_t dst_iovcnt, 725 struct iovec *src_iovs, uint32_t src_iovcnt, 726 uint64_t iv, uint32_t block_size, int flags, 727 spdk_accel_completion_cb cb_fn, void *cb_arg) 728 { 729 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 730 struct spdk_accel_task *accel_task; 731 732 if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) { 733 return -EINVAL; 734 } 735 736 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 737 if (spdk_unlikely(accel_task == NULL)) { 738 return -ENOMEM; 739 } 740 741 accel_task->crypto_key = key; 742 accel_task->s.iovs = src_iovs; 743 accel_task->s.iovcnt = src_iovcnt; 744 accel_task->d.iovs = dst_iovs; 745 accel_task->d.iovcnt = dst_iovcnt; 746 accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 747 accel_task->iv = iv; 748 accel_task->block_size = block_size; 749 accel_task->flags = flags; 750 accel_task->op_code = SPDK_ACCEL_OPC_DECRYPT; 751 accel_task->src_domain = NULL; 752 accel_task->dst_domain = NULL; 753 accel_task->step_cb_fn = NULL; 754 755 return accel_submit_task(accel_ch, accel_task); 756 } 757 758 int 759 spdk_accel_submit_xor(struct spdk_io_channel *ch, void *dst, void **sources, uint32_t nsrcs, 760 uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg) 761 { 762 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 763 struct spdk_accel_task *accel_task; 764 765 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 766 if (spdk_unlikely(accel_task == NULL)) { 767 return -ENOMEM; 768 } 769 770 accel_task->nsrcs.srcs = sources; 771 accel_task->nsrcs.cnt = nsrcs; 772 accel_task->d.iovs = &accel_task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 773 accel_task->d.iovs[0].iov_base = dst; 774 accel_task->d.iovs[0].iov_len = nbytes; 775 accel_task->d.iovcnt = 1; 776 accel_task->nbytes = nbytes; 777 accel_task->op_code = SPDK_ACCEL_OPC_XOR; 778 accel_task->src_domain = NULL; 779 accel_task->dst_domain = NULL; 780 accel_task->step_cb_fn = NULL; 781 782 return accel_submit_task(accel_ch, accel_task); 783 } 784 785 static inline struct accel_buffer * 786 accel_get_buf(struct accel_io_channel *ch, uint64_t len) 787 { 788 struct accel_buffer *buf; 789 790 buf = TAILQ_FIRST(&ch->buf_pool); 791 if (spdk_unlikely(buf == NULL)) { 792 accel_update_stats(ch, retry.bufdesc, 1); 793 return NULL; 794 } 795 796 TAILQ_REMOVE(&ch->buf_pool, buf, link); 797 buf->len = len; 798 buf->buf = NULL; 799 buf->seq = NULL; 800 buf->cb_fn = NULL; 801 802 return buf; 803 } 804 805 static inline void 806 accel_put_buf(struct accel_io_channel *ch, struct accel_buffer *buf) 807 { 808 if (buf->buf != NULL) { 809 spdk_iobuf_put(&ch->iobuf, buf->buf, buf->len); 810 } 811 812 TAILQ_INSERT_HEAD(&ch->buf_pool, buf, link); 813 } 814 815 static inline struct spdk_accel_sequence * 816 accel_sequence_get(struct accel_io_channel *ch) 817 { 818 struct spdk_accel_sequence *seq; 819 820 seq = TAILQ_FIRST(&ch->seq_pool); 821 if (spdk_unlikely(seq == NULL)) { 822 accel_update_stats(ch, retry.sequence, 1); 823 return NULL; 824 } 825 826 TAILQ_REMOVE(&ch->seq_pool, seq, link); 827 828 TAILQ_INIT(&seq->tasks); 829 TAILQ_INIT(&seq->completed); 830 TAILQ_INIT(&seq->bounce_bufs); 831 832 seq->ch = ch; 833 seq->status = 0; 834 seq->state = ACCEL_SEQUENCE_STATE_INIT; 835 seq->in_process_sequence = false; 836 837 return seq; 838 } 839 840 static inline void 841 accel_sequence_put(struct spdk_accel_sequence *seq) 842 { 843 struct accel_io_channel *ch = seq->ch; 844 struct accel_buffer *buf; 845 846 while (!TAILQ_EMPTY(&seq->bounce_bufs)) { 847 buf = TAILQ_FIRST(&seq->bounce_bufs); 848 TAILQ_REMOVE(&seq->bounce_bufs, buf, link); 849 accel_put_buf(seq->ch, buf); 850 } 851 852 assert(TAILQ_EMPTY(&seq->tasks)); 853 assert(TAILQ_EMPTY(&seq->completed)); 854 seq->ch = NULL; 855 856 TAILQ_INSERT_HEAD(&ch->seq_pool, seq, link); 857 } 858 859 static void accel_sequence_task_cb(void *cb_arg, int status); 860 861 static inline struct spdk_accel_task * 862 accel_sequence_get_task(struct accel_io_channel *ch, struct spdk_accel_sequence *seq, 863 spdk_accel_step_cb cb_fn, void *cb_arg) 864 { 865 struct spdk_accel_task *task; 866 867 task = _get_task(ch, accel_sequence_task_cb, seq); 868 if (spdk_unlikely(task == NULL)) { 869 return task; 870 } 871 872 task->step_cb_fn = cb_fn; 873 task->step_cb_arg = cb_arg; 874 task->seq = seq; 875 876 return task; 877 } 878 879 int 880 spdk_accel_append_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 881 struct iovec *dst_iovs, uint32_t dst_iovcnt, 882 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 883 struct iovec *src_iovs, uint32_t src_iovcnt, 884 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 885 int flags, spdk_accel_step_cb cb_fn, void *cb_arg) 886 { 887 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 888 struct spdk_accel_task *task; 889 struct spdk_accel_sequence *seq = *pseq; 890 891 if (seq == NULL) { 892 seq = accel_sequence_get(accel_ch); 893 if (spdk_unlikely(seq == NULL)) { 894 return -ENOMEM; 895 } 896 } 897 898 assert(seq->ch == accel_ch); 899 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 900 if (spdk_unlikely(task == NULL)) { 901 if (*pseq == NULL) { 902 accel_sequence_put(seq); 903 } 904 905 return -ENOMEM; 906 } 907 908 task->dst_domain = dst_domain; 909 task->dst_domain_ctx = dst_domain_ctx; 910 task->d.iovs = dst_iovs; 911 task->d.iovcnt = dst_iovcnt; 912 task->src_domain = src_domain; 913 task->src_domain_ctx = src_domain_ctx; 914 task->s.iovs = src_iovs; 915 task->s.iovcnt = src_iovcnt; 916 task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 917 task->flags = flags; 918 task->op_code = SPDK_ACCEL_OPC_COPY; 919 920 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 921 *pseq = seq; 922 923 return 0; 924 } 925 926 int 927 spdk_accel_append_fill(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 928 void *buf, uint64_t len, 929 struct spdk_memory_domain *domain, void *domain_ctx, uint8_t pattern, 930 int flags, spdk_accel_step_cb cb_fn, void *cb_arg) 931 { 932 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 933 struct spdk_accel_task *task; 934 struct spdk_accel_sequence *seq = *pseq; 935 936 if (seq == NULL) { 937 seq = accel_sequence_get(accel_ch); 938 if (spdk_unlikely(seq == NULL)) { 939 return -ENOMEM; 940 } 941 } 942 943 assert(seq->ch == accel_ch); 944 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 945 if (spdk_unlikely(task == NULL)) { 946 if (*pseq == NULL) { 947 accel_sequence_put(seq); 948 } 949 950 return -ENOMEM; 951 } 952 953 memset(&task->fill_pattern, pattern, sizeof(uint64_t)); 954 955 task->d.iovs = &task->aux_iovs[SPDK_ACCEL_AUX_IOV_DST]; 956 task->d.iovs[0].iov_base = buf; 957 task->d.iovs[0].iov_len = len; 958 task->d.iovcnt = 1; 959 task->nbytes = len; 960 task->src_domain = NULL; 961 task->dst_domain = domain; 962 task->dst_domain_ctx = domain_ctx; 963 task->flags = flags; 964 task->op_code = SPDK_ACCEL_OPC_FILL; 965 966 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 967 *pseq = seq; 968 969 return 0; 970 } 971 972 int 973 spdk_accel_append_decompress(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 974 struct iovec *dst_iovs, size_t dst_iovcnt, 975 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 976 struct iovec *src_iovs, size_t src_iovcnt, 977 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 978 int flags, spdk_accel_step_cb cb_fn, void *cb_arg) 979 { 980 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 981 struct spdk_accel_task *task; 982 struct spdk_accel_sequence *seq = *pseq; 983 984 if (seq == NULL) { 985 seq = accel_sequence_get(accel_ch); 986 if (spdk_unlikely(seq == NULL)) { 987 return -ENOMEM; 988 } 989 } 990 991 assert(seq->ch == accel_ch); 992 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 993 if (spdk_unlikely(task == NULL)) { 994 if (*pseq == NULL) { 995 accel_sequence_put(seq); 996 } 997 998 return -ENOMEM; 999 } 1000 1001 /* TODO: support output_size for chaining */ 1002 task->output_size = NULL; 1003 task->dst_domain = dst_domain; 1004 task->dst_domain_ctx = dst_domain_ctx; 1005 task->d.iovs = dst_iovs; 1006 task->d.iovcnt = dst_iovcnt; 1007 task->src_domain = src_domain; 1008 task->src_domain_ctx = src_domain_ctx; 1009 task->s.iovs = src_iovs; 1010 task->s.iovcnt = src_iovcnt; 1011 task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 1012 task->flags = flags; 1013 task->op_code = SPDK_ACCEL_OPC_DECOMPRESS; 1014 1015 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1016 *pseq = seq; 1017 1018 return 0; 1019 } 1020 1021 int 1022 spdk_accel_append_encrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1023 struct spdk_accel_crypto_key *key, 1024 struct iovec *dst_iovs, uint32_t dst_iovcnt, 1025 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 1026 struct iovec *src_iovs, uint32_t src_iovcnt, 1027 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 1028 uint64_t iv, uint32_t block_size, int flags, 1029 spdk_accel_step_cb cb_fn, void *cb_arg) 1030 { 1031 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1032 struct spdk_accel_task *task; 1033 struct spdk_accel_sequence *seq = *pseq; 1034 1035 assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size); 1036 1037 if (seq == NULL) { 1038 seq = accel_sequence_get(accel_ch); 1039 if (spdk_unlikely(seq == NULL)) { 1040 return -ENOMEM; 1041 } 1042 } 1043 1044 assert(seq->ch == accel_ch); 1045 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1046 if (spdk_unlikely(task == NULL)) { 1047 if (*pseq == NULL) { 1048 accel_sequence_put(seq); 1049 } 1050 1051 return -ENOMEM; 1052 } 1053 1054 task->crypto_key = key; 1055 task->src_domain = src_domain; 1056 task->src_domain_ctx = src_domain_ctx; 1057 task->s.iovs = src_iovs; 1058 task->s.iovcnt = src_iovcnt; 1059 task->dst_domain = dst_domain; 1060 task->dst_domain_ctx = dst_domain_ctx; 1061 task->d.iovs = dst_iovs; 1062 task->d.iovcnt = dst_iovcnt; 1063 task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 1064 task->iv = iv; 1065 task->block_size = block_size; 1066 task->flags = flags; 1067 task->op_code = SPDK_ACCEL_OPC_ENCRYPT; 1068 1069 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1070 *pseq = seq; 1071 1072 return 0; 1073 } 1074 1075 int 1076 spdk_accel_append_decrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1077 struct spdk_accel_crypto_key *key, 1078 struct iovec *dst_iovs, uint32_t dst_iovcnt, 1079 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 1080 struct iovec *src_iovs, uint32_t src_iovcnt, 1081 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 1082 uint64_t iv, uint32_t block_size, int flags, 1083 spdk_accel_step_cb cb_fn, void *cb_arg) 1084 { 1085 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1086 struct spdk_accel_task *task; 1087 struct spdk_accel_sequence *seq = *pseq; 1088 1089 assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size); 1090 1091 if (seq == NULL) { 1092 seq = accel_sequence_get(accel_ch); 1093 if (spdk_unlikely(seq == NULL)) { 1094 return -ENOMEM; 1095 } 1096 } 1097 1098 assert(seq->ch == accel_ch); 1099 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1100 if (spdk_unlikely(task == NULL)) { 1101 if (*pseq == NULL) { 1102 accel_sequence_put(seq); 1103 } 1104 1105 return -ENOMEM; 1106 } 1107 1108 task->crypto_key = key; 1109 task->src_domain = src_domain; 1110 task->src_domain_ctx = src_domain_ctx; 1111 task->s.iovs = src_iovs; 1112 task->s.iovcnt = src_iovcnt; 1113 task->dst_domain = dst_domain; 1114 task->dst_domain_ctx = dst_domain_ctx; 1115 task->d.iovs = dst_iovs; 1116 task->d.iovcnt = dst_iovcnt; 1117 task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 1118 task->iv = iv; 1119 task->block_size = block_size; 1120 task->flags = flags; 1121 task->op_code = SPDK_ACCEL_OPC_DECRYPT; 1122 1123 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1124 *pseq = seq; 1125 1126 return 0; 1127 } 1128 1129 int 1130 spdk_accel_append_crc32c(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1131 uint32_t *dst, struct iovec *iovs, uint32_t iovcnt, 1132 struct spdk_memory_domain *domain, void *domain_ctx, 1133 uint32_t seed, spdk_accel_step_cb cb_fn, void *cb_arg) 1134 { 1135 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1136 struct spdk_accel_task *task; 1137 struct spdk_accel_sequence *seq = *pseq; 1138 1139 if (seq == NULL) { 1140 seq = accel_sequence_get(accel_ch); 1141 if (spdk_unlikely(seq == NULL)) { 1142 return -ENOMEM; 1143 } 1144 } 1145 1146 assert(seq->ch == accel_ch); 1147 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1148 if (spdk_unlikely(task == NULL)) { 1149 if (*pseq == NULL) { 1150 accel_sequence_put(seq); 1151 } 1152 1153 return -ENOMEM; 1154 } 1155 1156 task->s.iovs = iovs; 1157 task->s.iovcnt = iovcnt; 1158 task->src_domain = domain; 1159 task->src_domain_ctx = domain_ctx; 1160 task->nbytes = accel_get_iovlen(iovs, iovcnt); 1161 task->crc_dst = dst; 1162 task->seed = seed; 1163 task->op_code = SPDK_ACCEL_OPC_CRC32C; 1164 task->dst_domain = NULL; 1165 1166 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1167 *pseq = seq; 1168 1169 return 0; 1170 } 1171 1172 int 1173 spdk_accel_get_buf(struct spdk_io_channel *ch, uint64_t len, void **buf, 1174 struct spdk_memory_domain **domain, void **domain_ctx) 1175 { 1176 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1177 struct accel_buffer *accel_buf; 1178 1179 accel_buf = accel_get_buf(accel_ch, len); 1180 if (spdk_unlikely(accel_buf == NULL)) { 1181 return -ENOMEM; 1182 } 1183 1184 /* We always return the same pointer and identify the buffers through domain_ctx */ 1185 *buf = ACCEL_BUFFER_BASE; 1186 *domain_ctx = accel_buf; 1187 *domain = g_accel_domain; 1188 1189 return 0; 1190 } 1191 1192 void 1193 spdk_accel_put_buf(struct spdk_io_channel *ch, void *buf, 1194 struct spdk_memory_domain *domain, void *domain_ctx) 1195 { 1196 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1197 struct accel_buffer *accel_buf = domain_ctx; 1198 1199 assert(domain == g_accel_domain); 1200 assert(buf == ACCEL_BUFFER_BASE); 1201 1202 accel_put_buf(accel_ch, accel_buf); 1203 } 1204 1205 static void 1206 accel_sequence_complete_tasks(struct spdk_accel_sequence *seq) 1207 { 1208 struct spdk_accel_task *task; 1209 struct accel_io_channel *ch = seq->ch; 1210 spdk_accel_step_cb cb_fn; 1211 void *cb_arg; 1212 1213 while (!TAILQ_EMPTY(&seq->completed)) { 1214 task = TAILQ_FIRST(&seq->completed); 1215 TAILQ_REMOVE(&seq->completed, task, seq_link); 1216 cb_fn = task->step_cb_fn; 1217 cb_arg = task->step_cb_arg; 1218 TAILQ_INSERT_HEAD(&ch->task_pool, task, link); 1219 if (cb_fn != NULL) { 1220 cb_fn(cb_arg); 1221 } 1222 } 1223 1224 while (!TAILQ_EMPTY(&seq->tasks)) { 1225 task = TAILQ_FIRST(&seq->tasks); 1226 TAILQ_REMOVE(&seq->tasks, task, seq_link); 1227 cb_fn = task->step_cb_fn; 1228 cb_arg = task->step_cb_arg; 1229 TAILQ_INSERT_HEAD(&ch->task_pool, task, link); 1230 if (cb_fn != NULL) { 1231 cb_fn(cb_arg); 1232 } 1233 } 1234 } 1235 1236 static void 1237 accel_sequence_complete(struct spdk_accel_sequence *seq) 1238 { 1239 SPDK_DEBUGLOG(accel, "Completed sequence: %p with status: %d\n", seq, seq->status); 1240 1241 accel_update_stats(seq->ch, sequence_executed, 1); 1242 if (spdk_unlikely(seq->status != 0)) { 1243 accel_update_stats(seq->ch, sequence_failed, 1); 1244 } 1245 1246 /* First notify all users that appended operations to this sequence */ 1247 accel_sequence_complete_tasks(seq); 1248 1249 /* Then notify the user that finished the sequence */ 1250 seq->cb_fn(seq->cb_arg, seq->status); 1251 1252 accel_sequence_put(seq); 1253 } 1254 1255 static void 1256 accel_update_virt_iov(struct iovec *diov, struct iovec *siov, struct accel_buffer *accel_buf) 1257 { 1258 uintptr_t offset; 1259 1260 offset = (uintptr_t)siov->iov_base & ACCEL_BUFFER_OFFSET_MASK; 1261 assert(offset < accel_buf->len); 1262 1263 diov->iov_base = (char *)accel_buf->buf + offset; 1264 diov->iov_len = siov->iov_len; 1265 } 1266 1267 static void 1268 accel_sequence_set_virtbuf(struct spdk_accel_sequence *seq, struct accel_buffer *buf) 1269 { 1270 struct spdk_accel_task *task; 1271 struct iovec *iov; 1272 1273 /* Now that we've allocated the actual data buffer for this accel_buffer, update all tasks 1274 * in a sequence that were using it. 1275 */ 1276 TAILQ_FOREACH(task, &seq->tasks, seq_link) { 1277 if (task->src_domain == g_accel_domain && task->src_domain_ctx == buf) { 1278 iov = &task->aux_iovs[SPDK_ACCEL_AXU_IOV_VIRT_SRC]; 1279 assert(task->s.iovcnt == 1); 1280 accel_update_virt_iov(iov, &task->s.iovs[0], buf); 1281 task->src_domain = NULL; 1282 task->s.iovs = iov; 1283 } 1284 if (task->dst_domain == g_accel_domain && task->dst_domain_ctx == buf) { 1285 iov = &task->aux_iovs[SPDK_ACCEL_AXU_IOV_VIRT_DST]; 1286 assert(task->d.iovcnt == 1); 1287 accel_update_virt_iov(iov, &task->d.iovs[0], buf); 1288 task->dst_domain = NULL; 1289 task->d.iovs = iov; 1290 } 1291 } 1292 } 1293 1294 static void accel_process_sequence(struct spdk_accel_sequence *seq); 1295 1296 static void 1297 accel_iobuf_get_virtbuf_cb(struct spdk_iobuf_entry *entry, void *buf) 1298 { 1299 struct accel_buffer *accel_buf; 1300 1301 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1302 1303 assert(accel_buf->seq != NULL); 1304 assert(accel_buf->buf == NULL); 1305 accel_buf->buf = buf; 1306 1307 assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF); 1308 accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF); 1309 accel_sequence_set_virtbuf(accel_buf->seq, accel_buf); 1310 accel_process_sequence(accel_buf->seq); 1311 } 1312 1313 static bool 1314 accel_sequence_alloc_buf(struct spdk_accel_sequence *seq, struct accel_buffer *buf, 1315 spdk_iobuf_get_cb cb_fn) 1316 { 1317 struct accel_io_channel *ch = seq->ch; 1318 1319 assert(buf->buf == NULL); 1320 assert(buf->seq == NULL); 1321 1322 buf->seq = seq; 1323 buf->buf = spdk_iobuf_get(&ch->iobuf, buf->len, &buf->iobuf, cb_fn); 1324 if (buf->buf == NULL) { 1325 accel_update_stats(ch, retry.iobuf, 1); 1326 return false; 1327 } 1328 1329 return true; 1330 } 1331 1332 static bool 1333 accel_sequence_check_virtbuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1334 { 1335 /* If a task doesn't have dst/src (e.g. fill, crc32), its dst/src domain should be set to 1336 * NULL */ 1337 if (task->src_domain == g_accel_domain) { 1338 if (!accel_sequence_alloc_buf(seq, task->src_domain_ctx, 1339 accel_iobuf_get_virtbuf_cb)) { 1340 return false; 1341 } 1342 1343 accel_sequence_set_virtbuf(seq, task->src_domain_ctx); 1344 } 1345 1346 if (task->dst_domain == g_accel_domain) { 1347 if (!accel_sequence_alloc_buf(seq, task->dst_domain_ctx, 1348 accel_iobuf_get_virtbuf_cb)) { 1349 return false; 1350 } 1351 1352 accel_sequence_set_virtbuf(seq, task->dst_domain_ctx); 1353 } 1354 1355 return true; 1356 } 1357 1358 static void 1359 accel_sequence_get_buf_cb(struct spdk_iobuf_entry *entry, void *buf) 1360 { 1361 struct accel_buffer *accel_buf; 1362 1363 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1364 1365 assert(accel_buf->seq != NULL); 1366 assert(accel_buf->buf == NULL); 1367 accel_buf->buf = buf; 1368 1369 accel_sequence_set_virtbuf(accel_buf->seq, accel_buf); 1370 accel_buf->cb_fn(accel_buf->seq, accel_buf->cb_ctx); 1371 } 1372 1373 bool 1374 spdk_accel_alloc_sequence_buf(struct spdk_accel_sequence *seq, void *buf, 1375 struct spdk_memory_domain *domain, void *domain_ctx, 1376 spdk_accel_sequence_get_buf_cb cb_fn, void *cb_ctx) 1377 { 1378 struct accel_buffer *accel_buf = domain_ctx; 1379 1380 assert(domain == g_accel_domain); 1381 accel_buf->cb_fn = cb_fn; 1382 accel_buf->cb_ctx = cb_ctx; 1383 1384 if (!accel_sequence_alloc_buf(seq, accel_buf, accel_sequence_get_buf_cb)) { 1385 return false; 1386 } 1387 1388 accel_sequence_set_virtbuf(seq, accel_buf); 1389 1390 return true; 1391 } 1392 1393 struct spdk_accel_task * 1394 spdk_accel_sequence_first_task(struct spdk_accel_sequence *seq) 1395 { 1396 return TAILQ_FIRST(&seq->tasks); 1397 } 1398 1399 struct spdk_accel_task * 1400 spdk_accel_sequence_next_task(struct spdk_accel_task *task) 1401 { 1402 return TAILQ_NEXT(task, seq_link); 1403 } 1404 1405 static inline void 1406 accel_set_bounce_buffer(struct spdk_accel_bounce_buffer *bounce, struct iovec **iovs, 1407 uint32_t *iovcnt, struct spdk_memory_domain **domain, void **domain_ctx, 1408 struct accel_buffer *buf) 1409 { 1410 bounce->orig_iovs = *iovs; 1411 bounce->orig_iovcnt = *iovcnt; 1412 bounce->orig_domain = *domain; 1413 bounce->orig_domain_ctx = *domain_ctx; 1414 bounce->iov.iov_base = buf->buf; 1415 bounce->iov.iov_len = buf->len; 1416 1417 *iovs = &bounce->iov; 1418 *iovcnt = 1; 1419 *domain = NULL; 1420 } 1421 1422 static void 1423 accel_iobuf_get_src_bounce_cb(struct spdk_iobuf_entry *entry, void *buf) 1424 { 1425 struct spdk_accel_task *task; 1426 struct accel_buffer *accel_buf; 1427 1428 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1429 assert(accel_buf->buf == NULL); 1430 accel_buf->buf = buf; 1431 1432 task = TAILQ_FIRST(&accel_buf->seq->tasks); 1433 assert(task != NULL); 1434 1435 assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF); 1436 accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF); 1437 accel_set_bounce_buffer(&task->bounce.s, &task->s.iovs, &task->s.iovcnt, &task->src_domain, 1438 &task->src_domain_ctx, accel_buf); 1439 accel_process_sequence(accel_buf->seq); 1440 } 1441 1442 static void 1443 accel_iobuf_get_dst_bounce_cb(struct spdk_iobuf_entry *entry, void *buf) 1444 { 1445 struct spdk_accel_task *task; 1446 struct accel_buffer *accel_buf; 1447 1448 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1449 assert(accel_buf->buf == NULL); 1450 accel_buf->buf = buf; 1451 1452 task = TAILQ_FIRST(&accel_buf->seq->tasks); 1453 assert(task != NULL); 1454 1455 assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF); 1456 accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF); 1457 accel_set_bounce_buffer(&task->bounce.d, &task->d.iovs, &task->d.iovcnt, &task->dst_domain, 1458 &task->dst_domain_ctx, accel_buf); 1459 accel_process_sequence(accel_buf->seq); 1460 } 1461 1462 static int 1463 accel_sequence_check_bouncebuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1464 { 1465 struct accel_buffer *buf; 1466 1467 if (task->src_domain != NULL) { 1468 /* By the time we're here, accel buffers should have been allocated */ 1469 assert(task->src_domain != g_accel_domain); 1470 1471 buf = accel_get_buf(seq->ch, accel_get_iovlen(task->s.iovs, task->s.iovcnt)); 1472 if (buf == NULL) { 1473 SPDK_ERRLOG("Couldn't allocate buffer descriptor\n"); 1474 return -ENOMEM; 1475 } 1476 1477 TAILQ_INSERT_TAIL(&seq->bounce_bufs, buf, link); 1478 if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_src_bounce_cb)) { 1479 return -EAGAIN; 1480 } 1481 1482 accel_set_bounce_buffer(&task->bounce.s, &task->s.iovs, &task->s.iovcnt, 1483 &task->src_domain, &task->src_domain_ctx, buf); 1484 } 1485 1486 if (task->dst_domain != NULL) { 1487 /* By the time we're here, accel buffers should have been allocated */ 1488 assert(task->dst_domain != g_accel_domain); 1489 1490 buf = accel_get_buf(seq->ch, accel_get_iovlen(task->d.iovs, task->d.iovcnt)); 1491 if (buf == NULL) { 1492 /* The src buffer will be released when a sequence is completed */ 1493 SPDK_ERRLOG("Couldn't allocate buffer descriptor\n"); 1494 return -ENOMEM; 1495 } 1496 1497 TAILQ_INSERT_TAIL(&seq->bounce_bufs, buf, link); 1498 if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_dst_bounce_cb)) { 1499 return -EAGAIN; 1500 } 1501 1502 accel_set_bounce_buffer(&task->bounce.d, &task->d.iovs, &task->d.iovcnt, 1503 &task->dst_domain, &task->dst_domain_ctx, buf); 1504 } 1505 1506 return 0; 1507 } 1508 1509 static void 1510 accel_task_pull_data_cb(void *ctx, int status) 1511 { 1512 struct spdk_accel_sequence *seq = ctx; 1513 1514 assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA); 1515 if (spdk_likely(status == 0)) { 1516 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK); 1517 } else { 1518 accel_sequence_set_fail(seq, status); 1519 } 1520 1521 accel_process_sequence(seq); 1522 } 1523 1524 static void 1525 accel_task_pull_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1526 { 1527 int rc; 1528 1529 assert(task->bounce.s.orig_iovs != NULL); 1530 assert(task->bounce.s.orig_domain != NULL); 1531 assert(task->bounce.s.orig_domain != g_accel_domain); 1532 assert(!g_modules_opc[task->op_code].supports_memory_domains); 1533 1534 rc = spdk_memory_domain_pull_data(task->bounce.s.orig_domain, 1535 task->bounce.s.orig_domain_ctx, 1536 task->bounce.s.orig_iovs, task->bounce.s.orig_iovcnt, 1537 task->s.iovs, task->s.iovcnt, 1538 accel_task_pull_data_cb, seq); 1539 if (spdk_unlikely(rc != 0)) { 1540 SPDK_ERRLOG("Failed to pull data from memory domain: %s, rc: %d\n", 1541 spdk_memory_domain_get_dma_device_id(task->bounce.s.orig_domain), rc); 1542 accel_sequence_set_fail(seq, rc); 1543 } 1544 } 1545 1546 static void 1547 accel_task_push_data_cb(void *ctx, int status) 1548 { 1549 struct spdk_accel_sequence *seq = ctx; 1550 1551 assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA); 1552 if (spdk_likely(status == 0)) { 1553 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK); 1554 } else { 1555 accel_sequence_set_fail(seq, status); 1556 } 1557 1558 accel_process_sequence(seq); 1559 } 1560 1561 static void 1562 accel_task_push_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1563 { 1564 int rc; 1565 1566 assert(task->bounce.d.orig_iovs != NULL); 1567 assert(task->bounce.d.orig_domain != NULL); 1568 assert(task->bounce.d.orig_domain != g_accel_domain); 1569 assert(!g_modules_opc[task->op_code].supports_memory_domains); 1570 1571 rc = spdk_memory_domain_push_data(task->bounce.d.orig_domain, 1572 task->bounce.d.orig_domain_ctx, 1573 task->bounce.d.orig_iovs, task->bounce.d.orig_iovcnt, 1574 task->d.iovs, task->d.iovcnt, 1575 accel_task_push_data_cb, seq); 1576 if (spdk_unlikely(rc != 0)) { 1577 SPDK_ERRLOG("Failed to push data to memory domain: %s, rc: %d\n", 1578 spdk_memory_domain_get_dma_device_id(task->bounce.s.orig_domain), rc); 1579 accel_sequence_set_fail(seq, rc); 1580 } 1581 } 1582 1583 static void 1584 accel_process_sequence(struct spdk_accel_sequence *seq) 1585 { 1586 struct accel_io_channel *accel_ch = seq->ch; 1587 struct spdk_accel_task *task; 1588 enum accel_sequence_state state; 1589 int rc; 1590 1591 /* Prevent recursive calls to this function */ 1592 if (spdk_unlikely(seq->in_process_sequence)) { 1593 return; 1594 } 1595 seq->in_process_sequence = true; 1596 1597 task = TAILQ_FIRST(&seq->tasks); 1598 do { 1599 state = seq->state; 1600 switch (state) { 1601 case ACCEL_SEQUENCE_STATE_INIT: 1602 if (g_accel_driver != NULL) { 1603 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_EXEC); 1604 break; 1605 } 1606 /* Fall through */ 1607 case ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF: 1608 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF); 1609 if (!accel_sequence_check_virtbuf(seq, task)) { 1610 /* We couldn't allocate a buffer, wait until one is available */ 1611 break; 1612 } 1613 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF); 1614 /* Fall through */ 1615 case ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF: 1616 /* If a module supports memory domains, we don't need to allocate bounce 1617 * buffers */ 1618 if (g_modules_opc[task->op_code].supports_memory_domains) { 1619 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK); 1620 break; 1621 } 1622 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF); 1623 rc = accel_sequence_check_bouncebuf(seq, task); 1624 if (spdk_unlikely(rc != 0)) { 1625 /* We couldn't allocate a buffer, wait until one is available */ 1626 if (rc == -EAGAIN) { 1627 break; 1628 } 1629 accel_sequence_set_fail(seq, rc); 1630 break; 1631 } 1632 if (task->s.iovs == &task->bounce.s.iov) { 1633 assert(task->bounce.s.orig_iovs); 1634 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PULL_DATA); 1635 break; 1636 } 1637 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK); 1638 /* Fall through */ 1639 case ACCEL_SEQUENCE_STATE_EXEC_TASK: 1640 SPDK_DEBUGLOG(accel, "Executing %s operation, sequence: %p\n", 1641 g_opcode_strings[task->op_code], seq); 1642 1643 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_TASK); 1644 rc = accel_submit_task(accel_ch, task); 1645 if (spdk_unlikely(rc != 0)) { 1646 SPDK_ERRLOG("Failed to submit %s operation, sequence: %p\n", 1647 g_opcode_strings[task->op_code], seq); 1648 accel_sequence_set_fail(seq, rc); 1649 } 1650 break; 1651 case ACCEL_SEQUENCE_STATE_PULL_DATA: 1652 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA); 1653 accel_task_pull_data(seq, task); 1654 break; 1655 case ACCEL_SEQUENCE_STATE_COMPLETE_TASK: 1656 if (task->d.iovs == &task->bounce.d.iov) { 1657 assert(task->bounce.d.orig_iovs); 1658 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PUSH_DATA); 1659 break; 1660 } 1661 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK); 1662 break; 1663 case ACCEL_SEQUENCE_STATE_PUSH_DATA: 1664 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA); 1665 accel_task_push_data(seq, task); 1666 break; 1667 case ACCEL_SEQUENCE_STATE_NEXT_TASK: 1668 TAILQ_REMOVE(&seq->tasks, task, seq_link); 1669 TAILQ_INSERT_TAIL(&seq->completed, task, seq_link); 1670 /* Check if there are any remaining tasks */ 1671 task = TAILQ_FIRST(&seq->tasks); 1672 if (task == NULL) { 1673 /* Immediately return here to make sure we don't touch the sequence 1674 * after it's completed */ 1675 accel_sequence_complete(seq); 1676 return; 1677 } 1678 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_INIT); 1679 break; 1680 case ACCEL_SEQUENCE_STATE_DRIVER_EXEC: 1681 assert(!TAILQ_EMPTY(&seq->tasks)); 1682 1683 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASK); 1684 rc = g_accel_driver->execute_sequence(accel_ch->driver_channel, seq); 1685 if (spdk_unlikely(rc != 0)) { 1686 SPDK_ERRLOG("Failed to execute sequence: %p using driver: %s\n", 1687 seq, g_accel_driver->name); 1688 accel_sequence_set_fail(seq, rc); 1689 } 1690 break; 1691 case ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE: 1692 /* Get the task again, as the driver might have completed some tasks 1693 * synchronously */ 1694 task = TAILQ_FIRST(&seq->tasks); 1695 if (task == NULL) { 1696 /* Immediately return here to make sure we don't touch the sequence 1697 * after it's completed */ 1698 accel_sequence_complete(seq); 1699 return; 1700 } 1701 /* We don't want to execute the next task through the driver, so we 1702 * explicitly omit the INIT state here */ 1703 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF); 1704 break; 1705 case ACCEL_SEQUENCE_STATE_ERROR: 1706 /* Immediately return here to make sure we don't touch the sequence 1707 * after it's completed */ 1708 assert(seq->status != 0); 1709 accel_sequence_complete(seq); 1710 return; 1711 case ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF: 1712 case ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF: 1713 case ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA: 1714 case ACCEL_SEQUENCE_STATE_AWAIT_TASK: 1715 case ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA: 1716 case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASK: 1717 break; 1718 default: 1719 assert(0 && "bad state"); 1720 break; 1721 } 1722 } while (seq->state != state); 1723 1724 seq->in_process_sequence = false; 1725 } 1726 1727 static void 1728 accel_sequence_task_cb(void *cb_arg, int status) 1729 { 1730 struct spdk_accel_sequence *seq = cb_arg; 1731 struct spdk_accel_task *task = TAILQ_FIRST(&seq->tasks); 1732 struct accel_io_channel *accel_ch = seq->ch; 1733 1734 /* spdk_accel_task_complete() puts the task back to the task pool, but we don't want to do 1735 * that if a task is part of a sequence. Removing the task from that pool here is the 1736 * easiest way to prevent this, even though it is a bit hacky. 1737 */ 1738 assert(task != NULL); 1739 TAILQ_REMOVE(&accel_ch->task_pool, task, link); 1740 1741 switch (seq->state) { 1742 case ACCEL_SEQUENCE_STATE_AWAIT_TASK: 1743 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_COMPLETE_TASK); 1744 if (spdk_unlikely(status != 0)) { 1745 SPDK_ERRLOG("Failed to execute %s operation, sequence: %p\n", 1746 g_opcode_strings[task->op_code], seq); 1747 accel_sequence_set_fail(seq, status); 1748 } 1749 1750 accel_process_sequence(seq); 1751 break; 1752 case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASK: 1753 assert(g_accel_driver != NULL); 1754 /* Immediately remove the task from the outstanding list to make sure the next call 1755 * to spdk_accel_sequence_first_task() doesn't return it */ 1756 TAILQ_REMOVE(&seq->tasks, task, seq_link); 1757 TAILQ_INSERT_TAIL(&seq->completed, task, seq_link); 1758 1759 if (spdk_unlikely(status != 0)) { 1760 SPDK_ERRLOG("Failed to execute %s operation, sequence: %p through " 1761 "driver: %s\n", g_opcode_strings[task->op_code], seq, 1762 g_accel_driver->name); 1763 /* Update status without using accel_sequence_set_fail() to avoid changing 1764 * seq's state to ERROR until driver calls spdk_accel_sequence_continue() */ 1765 seq->status = status; 1766 } 1767 break; 1768 default: 1769 assert(0 && "bad state"); 1770 break; 1771 } 1772 } 1773 1774 void 1775 spdk_accel_sequence_continue(struct spdk_accel_sequence *seq) 1776 { 1777 assert(g_accel_driver != NULL); 1778 assert(seq->state == ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASK); 1779 1780 if (spdk_likely(seq->status == 0)) { 1781 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE); 1782 } else { 1783 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR); 1784 } 1785 1786 accel_process_sequence(seq); 1787 } 1788 1789 static bool 1790 accel_compare_iovs(struct iovec *iova, uint32_t iovacnt, struct iovec *iovb, uint32_t iovbcnt) 1791 { 1792 /* For now, just do a dumb check that the iovecs arrays are exactly the same */ 1793 if (iovacnt != iovbcnt) { 1794 return false; 1795 } 1796 1797 return memcmp(iova, iovb, sizeof(*iova) * iovacnt) == 0; 1798 } 1799 1800 static bool 1801 accel_task_set_dstbuf(struct spdk_accel_task *task, struct spdk_accel_task *next) 1802 { 1803 struct spdk_accel_task *prev; 1804 1805 switch (task->op_code) { 1806 case SPDK_ACCEL_OPC_DECOMPRESS: 1807 case SPDK_ACCEL_OPC_FILL: 1808 case SPDK_ACCEL_OPC_ENCRYPT: 1809 case SPDK_ACCEL_OPC_DECRYPT: 1810 if (task->dst_domain != next->src_domain) { 1811 return false; 1812 } 1813 if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt, 1814 next->s.iovs, next->s.iovcnt)) { 1815 return false; 1816 } 1817 task->d.iovs = next->d.iovs; 1818 task->d.iovcnt = next->d.iovcnt; 1819 task->dst_domain = next->dst_domain; 1820 task->dst_domain_ctx = next->dst_domain_ctx; 1821 break; 1822 case SPDK_ACCEL_OPC_CRC32C: 1823 /* crc32 is special, because it doesn't have a dst buffer */ 1824 if (task->src_domain != next->src_domain) { 1825 return false; 1826 } 1827 if (!accel_compare_iovs(task->s.iovs, task->s.iovcnt, 1828 next->s.iovs, next->s.iovcnt)) { 1829 return false; 1830 } 1831 /* We can only change crc32's buffer if we can change previous task's buffer */ 1832 prev = TAILQ_PREV(task, accel_sequence_tasks, seq_link); 1833 if (prev == NULL) { 1834 return false; 1835 } 1836 if (!accel_task_set_dstbuf(prev, next)) { 1837 return false; 1838 } 1839 task->s.iovs = next->d.iovs; 1840 task->s.iovcnt = next->d.iovcnt; 1841 task->src_domain = next->dst_domain; 1842 task->src_domain_ctx = next->dst_domain_ctx; 1843 break; 1844 default: 1845 return false; 1846 } 1847 1848 return true; 1849 } 1850 1851 static void 1852 accel_sequence_merge_tasks(struct spdk_accel_sequence *seq, struct spdk_accel_task *task, 1853 struct spdk_accel_task **next_task) 1854 { 1855 struct spdk_accel_task *next = *next_task; 1856 1857 switch (task->op_code) { 1858 case SPDK_ACCEL_OPC_COPY: 1859 /* We only allow changing src of operations that actually have a src, e.g. we never 1860 * do it for fill. Theoretically, it is possible, but we'd have to be careful to 1861 * change the src of the operation after fill (which in turn could also be a fill). 1862 * So, for the sake of simplicity, skip this type of operations for now. 1863 */ 1864 if (next->op_code != SPDK_ACCEL_OPC_DECOMPRESS && 1865 next->op_code != SPDK_ACCEL_OPC_COPY && 1866 next->op_code != SPDK_ACCEL_OPC_ENCRYPT && 1867 next->op_code != SPDK_ACCEL_OPC_DECRYPT && 1868 next->op_code != SPDK_ACCEL_OPC_CRC32C) { 1869 break; 1870 } 1871 if (task->dst_domain != next->src_domain) { 1872 break; 1873 } 1874 if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt, 1875 next->s.iovs, next->s.iovcnt)) { 1876 break; 1877 } 1878 next->s.iovs = task->s.iovs; 1879 next->s.iovcnt = task->s.iovcnt; 1880 next->src_domain = task->src_domain; 1881 next->src_domain_ctx = task->src_domain_ctx; 1882 TAILQ_REMOVE(&seq->tasks, task, seq_link); 1883 TAILQ_INSERT_TAIL(&seq->completed, task, seq_link); 1884 break; 1885 case SPDK_ACCEL_OPC_DECOMPRESS: 1886 case SPDK_ACCEL_OPC_FILL: 1887 case SPDK_ACCEL_OPC_ENCRYPT: 1888 case SPDK_ACCEL_OPC_DECRYPT: 1889 case SPDK_ACCEL_OPC_CRC32C: 1890 /* We can only merge tasks when one of them is a copy */ 1891 if (next->op_code != SPDK_ACCEL_OPC_COPY) { 1892 break; 1893 } 1894 if (!accel_task_set_dstbuf(task, next)) { 1895 break; 1896 } 1897 /* We're removing next_task from the tasks queue, so we need to update its pointer, 1898 * so that the TAILQ_FOREACH_SAFE() loop below works correctly */ 1899 *next_task = TAILQ_NEXT(next, seq_link); 1900 TAILQ_REMOVE(&seq->tasks, next, seq_link); 1901 TAILQ_INSERT_TAIL(&seq->completed, next, seq_link); 1902 break; 1903 default: 1904 assert(0 && "bad opcode"); 1905 break; 1906 } 1907 } 1908 1909 void 1910 spdk_accel_sequence_finish(struct spdk_accel_sequence *seq, 1911 spdk_accel_completion_cb cb_fn, void *cb_arg) 1912 { 1913 struct spdk_accel_task *task, *next; 1914 1915 /* Try to remove any copy operations if possible */ 1916 TAILQ_FOREACH_SAFE(task, &seq->tasks, seq_link, next) { 1917 if (next == NULL) { 1918 break; 1919 } 1920 accel_sequence_merge_tasks(seq, task, &next); 1921 } 1922 1923 seq->cb_fn = cb_fn; 1924 seq->cb_arg = cb_arg; 1925 1926 accel_process_sequence(seq); 1927 } 1928 1929 void 1930 spdk_accel_sequence_reverse(struct spdk_accel_sequence *seq) 1931 { 1932 struct accel_sequence_tasks tasks = TAILQ_HEAD_INITIALIZER(tasks); 1933 struct spdk_accel_task *task; 1934 1935 assert(TAILQ_EMPTY(&seq->completed)); 1936 TAILQ_SWAP(&tasks, &seq->tasks, spdk_accel_task, seq_link); 1937 1938 while (!TAILQ_EMPTY(&tasks)) { 1939 task = TAILQ_FIRST(&tasks); 1940 TAILQ_REMOVE(&tasks, task, seq_link); 1941 TAILQ_INSERT_HEAD(&seq->tasks, task, seq_link); 1942 } 1943 } 1944 1945 void 1946 spdk_accel_sequence_abort(struct spdk_accel_sequence *seq) 1947 { 1948 if (seq == NULL) { 1949 return; 1950 } 1951 1952 accel_sequence_complete_tasks(seq); 1953 accel_sequence_put(seq); 1954 } 1955 1956 struct spdk_memory_domain * 1957 spdk_accel_get_memory_domain(void) 1958 { 1959 return g_accel_domain; 1960 } 1961 1962 static struct spdk_accel_module_if * 1963 _module_find_by_name(const char *name) 1964 { 1965 struct spdk_accel_module_if *accel_module = NULL; 1966 1967 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 1968 if (strcmp(name, accel_module->name) == 0) { 1969 break; 1970 } 1971 } 1972 1973 return accel_module; 1974 } 1975 1976 static inline struct spdk_accel_crypto_key * 1977 _accel_crypto_key_get(const char *name) 1978 { 1979 struct spdk_accel_crypto_key *key; 1980 1981 assert(spdk_spin_held(&g_keyring_spin)); 1982 1983 TAILQ_FOREACH(key, &g_keyring, link) { 1984 if (strcmp(name, key->param.key_name) == 0) { 1985 return key; 1986 } 1987 } 1988 1989 return NULL; 1990 } 1991 1992 static void 1993 accel_crypto_key_free_mem(struct spdk_accel_crypto_key *key) 1994 { 1995 if (key->param.hex_key) { 1996 spdk_memset_s(key->param.hex_key, key->key_size * 2, 0, key->key_size * 2); 1997 free(key->param.hex_key); 1998 } 1999 if (key->param.hex_key2) { 2000 spdk_memset_s(key->param.hex_key2, key->key2_size * 2, 0, key->key2_size * 2); 2001 free(key->param.hex_key2); 2002 } 2003 free(key->param.tweak_mode); 2004 free(key->param.key_name); 2005 free(key->param.cipher); 2006 if (key->key) { 2007 spdk_memset_s(key->key, key->key_size, 0, key->key_size); 2008 free(key->key); 2009 } 2010 if (key->key2) { 2011 spdk_memset_s(key->key2, key->key2_size, 0, key->key2_size); 2012 free(key->key2); 2013 } 2014 free(key); 2015 } 2016 2017 static void 2018 accel_crypto_key_destroy_unsafe(struct spdk_accel_crypto_key *key) 2019 { 2020 assert(key->module_if); 2021 assert(key->module_if->crypto_key_deinit); 2022 2023 key->module_if->crypto_key_deinit(key); 2024 accel_crypto_key_free_mem(key); 2025 } 2026 2027 /* 2028 * This function mitigates a timing side channel which could be caused by using strcmp() 2029 * Please refer to chapter "Mitigating Information Leakage Based on Variable Timing" in 2030 * the article [1] for more details 2031 * [1] https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/secure-coding/mitigate-timing-side-channel-crypto-implementation.html 2032 */ 2033 static bool 2034 accel_aes_xts_keys_equal(const char *k1, size_t k1_len, const char *k2, size_t k2_len) 2035 { 2036 size_t i; 2037 volatile size_t x = k1_len ^ k2_len; 2038 2039 for (i = 0; ((i < k1_len) & (i < k2_len)); i++) { 2040 x |= k1[i] ^ k2[i]; 2041 } 2042 2043 return x == 0; 2044 } 2045 2046 static const char *g_tweak_modes[] = { 2047 [SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA] = "SIMPLE_LBA", 2048 [SPDK_ACCEL_CRYPTO_TWEAK_MODE_JOIN_NEG_LBA_WITH_LBA] = "JOIN_NEG_LBA_WITH_LBA", 2049 [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_FULL_LBA] = "INCR_512_FULL_LBA", 2050 [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_UPPER_LBA] = "INCR_512_UPPER_LBA", 2051 }; 2052 2053 static const char *g_ciphers[] = { 2054 [SPDK_ACCEL_CIPHER_AES_CBC] = "AES_CBC", 2055 [SPDK_ACCEL_CIPHER_AES_XTS] = "AES_XTS", 2056 }; 2057 2058 int 2059 spdk_accel_crypto_key_create(const struct spdk_accel_crypto_key_create_param *param) 2060 { 2061 struct spdk_accel_module_if *module; 2062 struct spdk_accel_crypto_key *key; 2063 size_t hex_key_size, hex_key2_size; 2064 bool found = false; 2065 size_t i; 2066 int rc; 2067 2068 if (!param || !param->hex_key || !param->cipher || !param->key_name) { 2069 return -EINVAL; 2070 } 2071 2072 if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) { 2073 /* hardly ever possible, but let's check and warn the user */ 2074 SPDK_ERRLOG("Different accel modules are used for encryption and decryption\n"); 2075 } 2076 module = g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module; 2077 2078 if (!module) { 2079 SPDK_ERRLOG("No accel module found assigned for crypto operation\n"); 2080 return -ENOENT; 2081 } 2082 2083 if (!module->crypto_key_init || !module->crypto_supports_cipher) { 2084 SPDK_ERRLOG("Module %s doesn't support crypto operations\n", module->name); 2085 return -ENOTSUP; 2086 } 2087 2088 key = calloc(1, sizeof(*key)); 2089 if (!key) { 2090 return -ENOMEM; 2091 } 2092 2093 key->param.key_name = strdup(param->key_name); 2094 if (!key->param.key_name) { 2095 rc = -ENOMEM; 2096 goto error; 2097 } 2098 2099 for (i = 0; i < SPDK_COUNTOF(g_ciphers); ++i) { 2100 assert(g_ciphers[i]); 2101 2102 if (strncmp(param->cipher, g_ciphers[i], strlen(g_ciphers[i])) == 0) { 2103 key->cipher = i; 2104 found = true; 2105 break; 2106 } 2107 } 2108 2109 if (!found) { 2110 SPDK_ERRLOG("Failed to parse cipher\n"); 2111 rc = -EINVAL; 2112 goto error; 2113 } 2114 2115 key->param.cipher = strdup(param->cipher); 2116 if (!key->param.cipher) { 2117 rc = -ENOMEM; 2118 goto error; 2119 } 2120 2121 hex_key_size = strnlen(param->hex_key, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 2122 if (hex_key_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) { 2123 SPDK_ERRLOG("key1 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 2124 rc = -EINVAL; 2125 goto error; 2126 } 2127 2128 if (hex_key_size == 0) { 2129 SPDK_ERRLOG("key1 size cannot be 0\n"); 2130 rc = -EINVAL; 2131 goto error; 2132 } 2133 2134 key->param.hex_key = strdup(param->hex_key); 2135 if (!key->param.hex_key) { 2136 rc = -ENOMEM; 2137 goto error; 2138 } 2139 2140 key->key_size = hex_key_size / 2; 2141 key->key = spdk_unhexlify(key->param.hex_key); 2142 if (!key->key) { 2143 SPDK_ERRLOG("Failed to unhexlify key1\n"); 2144 rc = -EINVAL; 2145 goto error; 2146 } 2147 2148 if (param->hex_key2) { 2149 hex_key2_size = strnlen(param->hex_key2, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 2150 if (hex_key2_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) { 2151 SPDK_ERRLOG("key2 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 2152 rc = -EINVAL; 2153 goto error; 2154 } 2155 2156 if (hex_key2_size == 0) { 2157 SPDK_ERRLOG("key2 size cannot be 0\n"); 2158 rc = -EINVAL; 2159 goto error; 2160 } 2161 2162 key->param.hex_key2 = strdup(param->hex_key2); 2163 if (!key->param.hex_key2) { 2164 rc = -ENOMEM; 2165 goto error; 2166 } 2167 2168 key->key2_size = hex_key2_size / 2; 2169 key->key2 = spdk_unhexlify(key->param.hex_key2); 2170 if (!key->key2) { 2171 SPDK_ERRLOG("Failed to unhexlify key2\n"); 2172 rc = -EINVAL; 2173 goto error; 2174 } 2175 } 2176 2177 key->tweak_mode = ACCEL_CRYPTO_TWEAK_MODE_DEFAULT; 2178 if (param->tweak_mode) { 2179 found = false; 2180 2181 key->param.tweak_mode = strdup(param->tweak_mode); 2182 if (!key->param.tweak_mode) { 2183 rc = -ENOMEM; 2184 goto error; 2185 } 2186 2187 for (i = 0; i < SPDK_COUNTOF(g_tweak_modes); ++i) { 2188 assert(g_tweak_modes[i]); 2189 2190 if (strncmp(param->tweak_mode, g_tweak_modes[i], strlen(g_tweak_modes[i])) == 0) { 2191 key->tweak_mode = i; 2192 found = true; 2193 break; 2194 } 2195 } 2196 2197 if (!found) { 2198 SPDK_ERRLOG("Failed to parse tweak mode\n"); 2199 rc = -EINVAL; 2200 goto error; 2201 } 2202 } 2203 2204 if ((!module->crypto_supports_tweak_mode && key->tweak_mode != ACCEL_CRYPTO_TWEAK_MODE_DEFAULT) || 2205 (module->crypto_supports_tweak_mode && !module->crypto_supports_tweak_mode(key->tweak_mode))) { 2206 SPDK_ERRLOG("Module %s doesn't support %s tweak mode\n", module->name, 2207 g_tweak_modes[key->tweak_mode]); 2208 rc = -EINVAL; 2209 goto error; 2210 } 2211 2212 if (!module->crypto_supports_cipher(key->cipher, key->key_size)) { 2213 SPDK_ERRLOG("Module %s doesn't support %s cipher with %zu key size\n", module->name, 2214 g_ciphers[key->cipher], key->key_size); 2215 rc = -EINVAL; 2216 goto error; 2217 } 2218 2219 if (key->cipher == SPDK_ACCEL_CIPHER_AES_XTS) { 2220 if (!key->key2) { 2221 SPDK_ERRLOG("%s key2 is missing\n", g_ciphers[key->cipher]); 2222 rc = -EINVAL; 2223 goto error; 2224 } 2225 2226 if (key->key_size != key->key2_size) { 2227 SPDK_ERRLOG("%s key size %zu is not equal to key2 size %zu\n", g_ciphers[key->cipher], 2228 key->key_size, 2229 key->key2_size); 2230 rc = -EINVAL; 2231 goto error; 2232 } 2233 2234 if (accel_aes_xts_keys_equal(key->key, key->key_size, key->key2, key->key2_size)) { 2235 SPDK_ERRLOG("%s identical keys are not secure\n", g_ciphers[key->cipher]); 2236 rc = -EINVAL; 2237 goto error; 2238 } 2239 } 2240 2241 if (key->cipher == SPDK_ACCEL_CIPHER_AES_CBC) { 2242 if (key->key2_size) { 2243 SPDK_ERRLOG("%s doesn't use key2\n", g_ciphers[key->cipher]); 2244 rc = -EINVAL; 2245 goto error; 2246 } 2247 } 2248 2249 key->module_if = module; 2250 2251 spdk_spin_lock(&g_keyring_spin); 2252 if (_accel_crypto_key_get(param->key_name)) { 2253 rc = -EEXIST; 2254 } else { 2255 rc = module->crypto_key_init(key); 2256 if (rc) { 2257 SPDK_ERRLOG("Module %s failed to initialize crypto key\n", module->name); 2258 } else { 2259 TAILQ_INSERT_TAIL(&g_keyring, key, link); 2260 } 2261 } 2262 spdk_spin_unlock(&g_keyring_spin); 2263 2264 if (rc) { 2265 goto error; 2266 } 2267 2268 return 0; 2269 2270 error: 2271 accel_crypto_key_free_mem(key); 2272 return rc; 2273 } 2274 2275 int 2276 spdk_accel_crypto_key_destroy(struct spdk_accel_crypto_key *key) 2277 { 2278 if (!key || !key->module_if) { 2279 return -EINVAL; 2280 } 2281 2282 spdk_spin_lock(&g_keyring_spin); 2283 if (!_accel_crypto_key_get(key->param.key_name)) { 2284 spdk_spin_unlock(&g_keyring_spin); 2285 return -ENOENT; 2286 } 2287 TAILQ_REMOVE(&g_keyring, key, link); 2288 spdk_spin_unlock(&g_keyring_spin); 2289 2290 accel_crypto_key_destroy_unsafe(key); 2291 2292 return 0; 2293 } 2294 2295 struct spdk_accel_crypto_key * 2296 spdk_accel_crypto_key_get(const char *name) 2297 { 2298 struct spdk_accel_crypto_key *key; 2299 2300 spdk_spin_lock(&g_keyring_spin); 2301 key = _accel_crypto_key_get(name); 2302 spdk_spin_unlock(&g_keyring_spin); 2303 2304 return key; 2305 } 2306 2307 /* Helper function when accel modules register with the framework. */ 2308 void 2309 spdk_accel_module_list_add(struct spdk_accel_module_if *accel_module) 2310 { 2311 if (_module_find_by_name(accel_module->name)) { 2312 SPDK_NOTICELOG("Module %s already registered\n", accel_module->name); 2313 assert(false); 2314 return; 2315 } 2316 2317 /* Make sure that the software module is at the head of the list, this 2318 * will assure that all opcodes are later assigned to software first and 2319 * then updated to HW modules as they are registered. 2320 */ 2321 if (strcmp(accel_module->name, "software") == 0) { 2322 TAILQ_INSERT_HEAD(&spdk_accel_module_list, accel_module, tailq); 2323 } else { 2324 TAILQ_INSERT_TAIL(&spdk_accel_module_list, accel_module, tailq); 2325 } 2326 2327 if (accel_module->get_ctx_size && accel_module->get_ctx_size() > g_max_accel_module_size) { 2328 g_max_accel_module_size = accel_module->get_ctx_size(); 2329 } 2330 } 2331 2332 /* Framework level channel create callback. */ 2333 static int 2334 accel_create_channel(void *io_device, void *ctx_buf) 2335 { 2336 struct accel_io_channel *accel_ch = ctx_buf; 2337 struct spdk_accel_task *accel_task; 2338 struct spdk_accel_sequence *seq; 2339 struct accel_buffer *buf; 2340 uint8_t *task_mem; 2341 uint32_t i = 0, j; 2342 int rc; 2343 2344 accel_ch->task_pool_base = calloc(g_opts.task_count, g_max_accel_module_size); 2345 if (accel_ch->task_pool_base == NULL) { 2346 return -ENOMEM; 2347 } 2348 2349 accel_ch->seq_pool_base = calloc(g_opts.sequence_count, sizeof(struct spdk_accel_sequence)); 2350 if (accel_ch->seq_pool_base == NULL) { 2351 goto err; 2352 } 2353 2354 accel_ch->buf_pool_base = calloc(g_opts.buf_count, sizeof(struct accel_buffer)); 2355 if (accel_ch->buf_pool_base == NULL) { 2356 goto err; 2357 } 2358 2359 TAILQ_INIT(&accel_ch->task_pool); 2360 TAILQ_INIT(&accel_ch->seq_pool); 2361 TAILQ_INIT(&accel_ch->buf_pool); 2362 2363 task_mem = accel_ch->task_pool_base; 2364 for (i = 0; i < g_opts.task_count; i++) { 2365 accel_task = (struct spdk_accel_task *)task_mem; 2366 TAILQ_INSERT_TAIL(&accel_ch->task_pool, accel_task, link); 2367 task_mem += g_max_accel_module_size; 2368 } 2369 for (i = 0; i < g_opts.sequence_count; i++) { 2370 seq = &accel_ch->seq_pool_base[i]; 2371 TAILQ_INSERT_TAIL(&accel_ch->seq_pool, seq, link); 2372 } 2373 for (i = 0; i < g_opts.buf_count; i++) { 2374 buf = &accel_ch->buf_pool_base[i]; 2375 TAILQ_INSERT_TAIL(&accel_ch->buf_pool, buf, link); 2376 } 2377 2378 /* Assign modules and get IO channels for each */ 2379 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 2380 accel_ch->module_ch[i] = g_modules_opc[i].module->get_io_channel(); 2381 /* This can happen if idxd runs out of channels. */ 2382 if (accel_ch->module_ch[i] == NULL) { 2383 SPDK_ERRLOG("Module %s failed to get io channel\n", g_modules_opc[i].module->name); 2384 goto err; 2385 } 2386 } 2387 2388 if (g_accel_driver != NULL) { 2389 accel_ch->driver_channel = g_accel_driver->get_io_channel(); 2390 if (accel_ch->driver_channel == NULL) { 2391 SPDK_ERRLOG("Failed to get driver's IO channel\n"); 2392 goto err; 2393 } 2394 } 2395 2396 rc = spdk_iobuf_channel_init(&accel_ch->iobuf, "accel", g_opts.small_cache_size, 2397 g_opts.large_cache_size); 2398 if (rc != 0) { 2399 SPDK_ERRLOG("Failed to initialize iobuf accel channel\n"); 2400 goto err; 2401 } 2402 2403 return 0; 2404 err: 2405 if (accel_ch->driver_channel != NULL) { 2406 spdk_put_io_channel(accel_ch->driver_channel); 2407 } 2408 for (j = 0; j < i; j++) { 2409 spdk_put_io_channel(accel_ch->module_ch[j]); 2410 } 2411 free(accel_ch->task_pool_base); 2412 free(accel_ch->seq_pool_base); 2413 free(accel_ch->buf_pool_base); 2414 2415 return -ENOMEM; 2416 } 2417 2418 static void 2419 accel_add_stats(struct accel_stats *total, struct accel_stats *stats) 2420 { 2421 int i; 2422 2423 total->sequence_executed += stats->sequence_executed; 2424 total->sequence_failed += stats->sequence_failed; 2425 total->retry.task += stats->retry.task; 2426 total->retry.sequence += stats->retry.sequence; 2427 total->retry.iobuf += stats->retry.iobuf; 2428 total->retry.bufdesc += stats->retry.bufdesc; 2429 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2430 total->operations[i].executed += stats->operations[i].executed; 2431 total->operations[i].failed += stats->operations[i].failed; 2432 total->operations[i].num_bytes += stats->operations[i].num_bytes; 2433 } 2434 } 2435 2436 /* Framework level channel destroy callback. */ 2437 static void 2438 accel_destroy_channel(void *io_device, void *ctx_buf) 2439 { 2440 struct accel_io_channel *accel_ch = ctx_buf; 2441 int i; 2442 2443 spdk_iobuf_channel_fini(&accel_ch->iobuf); 2444 2445 if (accel_ch->driver_channel != NULL) { 2446 spdk_put_io_channel(accel_ch->driver_channel); 2447 } 2448 2449 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 2450 assert(accel_ch->module_ch[i] != NULL); 2451 spdk_put_io_channel(accel_ch->module_ch[i]); 2452 accel_ch->module_ch[i] = NULL; 2453 } 2454 2455 /* Update global stats to make sure channel's stats aren't lost after a channel is gone */ 2456 spdk_spin_lock(&g_stats_lock); 2457 accel_add_stats(&g_stats, &accel_ch->stats); 2458 spdk_spin_unlock(&g_stats_lock); 2459 2460 free(accel_ch->task_pool_base); 2461 free(accel_ch->seq_pool_base); 2462 free(accel_ch->buf_pool_base); 2463 } 2464 2465 struct spdk_io_channel * 2466 spdk_accel_get_io_channel(void) 2467 { 2468 return spdk_get_io_channel(&spdk_accel_module_list); 2469 } 2470 2471 static int 2472 accel_module_initialize(void) 2473 { 2474 struct spdk_accel_module_if *accel_module, *tmp_module; 2475 int rc = 0, module_rc; 2476 2477 TAILQ_FOREACH_SAFE(accel_module, &spdk_accel_module_list, tailq, tmp_module) { 2478 module_rc = accel_module->module_init(); 2479 if (module_rc) { 2480 SPDK_ERRLOG("Module %s initialization failed with %d\n", accel_module->name, module_rc); 2481 TAILQ_REMOVE(&spdk_accel_module_list, accel_module, tailq); 2482 if (!rc) { 2483 rc = module_rc; 2484 } 2485 } 2486 2487 SPDK_DEBUGLOG(accel, "Module %s initialized.\n", accel_module->name); 2488 } 2489 2490 return rc; 2491 } 2492 2493 static void 2494 accel_module_init_opcode(enum spdk_accel_opcode opcode) 2495 { 2496 struct accel_module *module = &g_modules_opc[opcode]; 2497 struct spdk_accel_module_if *module_if = module->module; 2498 2499 if (module_if->get_memory_domains != NULL) { 2500 module->supports_memory_domains = module_if->get_memory_domains(NULL, 0) > 0; 2501 } 2502 } 2503 2504 int 2505 spdk_accel_initialize(void) 2506 { 2507 enum spdk_accel_opcode op; 2508 struct spdk_accel_module_if *accel_module = NULL; 2509 int rc; 2510 2511 /* 2512 * We need a unique identifier for the accel framework, so use the 2513 * spdk_accel_module_list address for this purpose. 2514 */ 2515 spdk_io_device_register(&spdk_accel_module_list, accel_create_channel, accel_destroy_channel, 2516 sizeof(struct accel_io_channel), "accel"); 2517 2518 spdk_spin_init(&g_keyring_spin); 2519 spdk_spin_init(&g_stats_lock); 2520 2521 rc = spdk_memory_domain_create(&g_accel_domain, SPDK_DMA_DEVICE_TYPE_ACCEL, NULL, 2522 "SPDK_ACCEL_DMA_DEVICE"); 2523 if (rc != 0) { 2524 SPDK_ERRLOG("Failed to create accel memory domain\n"); 2525 return rc; 2526 } 2527 2528 g_modules_started = true; 2529 rc = accel_module_initialize(); 2530 if (rc) { 2531 return rc; 2532 } 2533 2534 /* Create our priority global map of opcodes to modules, we populate starting 2535 * with the software module (guaranteed to be first on the list) and then 2536 * updating opcodes with HW modules that have been initialized. 2537 * NOTE: all opcodes must be supported by software in the event that no HW 2538 * modules are initialized to support the operation. 2539 */ 2540 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 2541 for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) { 2542 if (accel_module->supports_opcode(op)) { 2543 g_modules_opc[op].module = accel_module; 2544 SPDK_DEBUGLOG(accel, "OPC 0x%x now assigned to %s\n", op, accel_module->name); 2545 } 2546 } 2547 } 2548 2549 /* Now lets check for overrides and apply all that exist */ 2550 for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) { 2551 if (g_modules_opc_override[op] != NULL) { 2552 accel_module = _module_find_by_name(g_modules_opc_override[op]); 2553 if (accel_module == NULL) { 2554 SPDK_ERRLOG("Invalid module name of %s\n", g_modules_opc_override[op]); 2555 return -EINVAL; 2556 } 2557 if (accel_module->supports_opcode(op) == false) { 2558 SPDK_ERRLOG("Module %s does not support op code %d\n", accel_module->name, op); 2559 return -EINVAL; 2560 } 2561 g_modules_opc[op].module = accel_module; 2562 } 2563 } 2564 2565 if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) { 2566 SPDK_ERRLOG("Different accel modules are assigned to encrypt and decrypt operations"); 2567 return -EINVAL; 2568 } 2569 2570 for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) { 2571 assert(g_modules_opc[op].module != NULL); 2572 accel_module_init_opcode(op); 2573 } 2574 2575 rc = spdk_iobuf_register_module("accel"); 2576 if (rc != 0) { 2577 SPDK_ERRLOG("Failed to register accel iobuf module\n"); 2578 return rc; 2579 } 2580 2581 return 0; 2582 } 2583 2584 static void 2585 accel_module_finish_cb(void) 2586 { 2587 spdk_accel_fini_cb cb_fn = g_fini_cb_fn; 2588 2589 cb_fn(g_fini_cb_arg); 2590 g_fini_cb_fn = NULL; 2591 g_fini_cb_arg = NULL; 2592 } 2593 2594 static void 2595 accel_write_overridden_opc(struct spdk_json_write_ctx *w, const char *opc_str, 2596 const char *module_str) 2597 { 2598 spdk_json_write_object_begin(w); 2599 spdk_json_write_named_string(w, "method", "accel_assign_opc"); 2600 spdk_json_write_named_object_begin(w, "params"); 2601 spdk_json_write_named_string(w, "opname", opc_str); 2602 spdk_json_write_named_string(w, "module", module_str); 2603 spdk_json_write_object_end(w); 2604 spdk_json_write_object_end(w); 2605 } 2606 2607 static void 2608 __accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key) 2609 { 2610 spdk_json_write_named_string(w, "name", key->param.key_name); 2611 spdk_json_write_named_string(w, "cipher", key->param.cipher); 2612 spdk_json_write_named_string(w, "key", key->param.hex_key); 2613 if (key->param.hex_key2) { 2614 spdk_json_write_named_string(w, "key2", key->param.hex_key2); 2615 } 2616 2617 if (key->param.tweak_mode) { 2618 spdk_json_write_named_string(w, "tweak_mode", key->param.tweak_mode); 2619 } 2620 } 2621 2622 void 2623 _accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key) 2624 { 2625 spdk_json_write_object_begin(w); 2626 __accel_crypto_key_dump_param(w, key); 2627 spdk_json_write_object_end(w); 2628 } 2629 2630 static void 2631 _accel_crypto_key_write_config_json(struct spdk_json_write_ctx *w, 2632 struct spdk_accel_crypto_key *key) 2633 { 2634 spdk_json_write_object_begin(w); 2635 spdk_json_write_named_string(w, "method", "accel_crypto_key_create"); 2636 spdk_json_write_named_object_begin(w, "params"); 2637 __accel_crypto_key_dump_param(w, key); 2638 spdk_json_write_object_end(w); 2639 spdk_json_write_object_end(w); 2640 } 2641 2642 static void 2643 accel_write_options(struct spdk_json_write_ctx *w) 2644 { 2645 spdk_json_write_object_begin(w); 2646 spdk_json_write_named_string(w, "method", "accel_set_options"); 2647 spdk_json_write_named_object_begin(w, "params"); 2648 spdk_json_write_named_uint32(w, "small_cache_size", g_opts.small_cache_size); 2649 spdk_json_write_named_uint32(w, "large_cache_size", g_opts.large_cache_size); 2650 spdk_json_write_named_uint32(w, "task_count", g_opts.task_count); 2651 spdk_json_write_named_uint32(w, "sequence_count", g_opts.sequence_count); 2652 spdk_json_write_named_uint32(w, "buf_count", g_opts.buf_count); 2653 spdk_json_write_object_end(w); 2654 spdk_json_write_object_end(w); 2655 } 2656 2657 static void 2658 _accel_crypto_keys_write_config_json(struct spdk_json_write_ctx *w, bool full_dump) 2659 { 2660 struct spdk_accel_crypto_key *key; 2661 2662 spdk_spin_lock(&g_keyring_spin); 2663 TAILQ_FOREACH(key, &g_keyring, link) { 2664 if (full_dump) { 2665 _accel_crypto_key_write_config_json(w, key); 2666 } else { 2667 _accel_crypto_key_dump_param(w, key); 2668 } 2669 } 2670 spdk_spin_unlock(&g_keyring_spin); 2671 } 2672 2673 void 2674 _accel_crypto_keys_dump_param(struct spdk_json_write_ctx *w) 2675 { 2676 _accel_crypto_keys_write_config_json(w, false); 2677 } 2678 2679 void 2680 spdk_accel_write_config_json(struct spdk_json_write_ctx *w) 2681 { 2682 struct spdk_accel_module_if *accel_module; 2683 int i; 2684 2685 spdk_json_write_array_begin(w); 2686 accel_write_options(w); 2687 2688 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 2689 if (accel_module->write_config_json) { 2690 accel_module->write_config_json(w); 2691 } 2692 } 2693 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 2694 if (g_modules_opc_override[i]) { 2695 accel_write_overridden_opc(w, g_opcode_strings[i], g_modules_opc_override[i]); 2696 } 2697 } 2698 2699 _accel_crypto_keys_write_config_json(w, true); 2700 2701 spdk_json_write_array_end(w); 2702 } 2703 2704 void 2705 spdk_accel_module_finish(void) 2706 { 2707 if (!g_accel_module) { 2708 g_accel_module = TAILQ_FIRST(&spdk_accel_module_list); 2709 } else { 2710 g_accel_module = TAILQ_NEXT(g_accel_module, tailq); 2711 } 2712 2713 if (!g_accel_module) { 2714 spdk_spin_destroy(&g_keyring_spin); 2715 spdk_spin_destroy(&g_stats_lock); 2716 if (g_accel_domain) { 2717 spdk_memory_domain_destroy(g_accel_domain); 2718 g_accel_domain = NULL; 2719 } 2720 accel_module_finish_cb(); 2721 return; 2722 } 2723 2724 if (g_accel_module->module_fini) { 2725 spdk_thread_send_msg(spdk_get_thread(), g_accel_module->module_fini, NULL); 2726 } else { 2727 spdk_accel_module_finish(); 2728 } 2729 } 2730 2731 static void 2732 accel_io_device_unregister_cb(void *io_device) 2733 { 2734 struct spdk_accel_crypto_key *key, *key_tmp; 2735 enum spdk_accel_opcode op; 2736 2737 spdk_spin_lock(&g_keyring_spin); 2738 TAILQ_FOREACH_SAFE(key, &g_keyring, link, key_tmp) { 2739 accel_crypto_key_destroy_unsafe(key); 2740 } 2741 spdk_spin_unlock(&g_keyring_spin); 2742 2743 for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) { 2744 if (g_modules_opc_override[op] != NULL) { 2745 free(g_modules_opc_override[op]); 2746 g_modules_opc_override[op] = NULL; 2747 } 2748 g_modules_opc[op].module = NULL; 2749 } 2750 2751 spdk_accel_module_finish(); 2752 } 2753 2754 void 2755 spdk_accel_finish(spdk_accel_fini_cb cb_fn, void *cb_arg) 2756 { 2757 assert(cb_fn != NULL); 2758 2759 g_fini_cb_fn = cb_fn; 2760 g_fini_cb_arg = cb_arg; 2761 2762 spdk_io_device_unregister(&spdk_accel_module_list, accel_io_device_unregister_cb); 2763 } 2764 2765 static struct spdk_accel_driver * 2766 accel_find_driver(const char *name) 2767 { 2768 struct spdk_accel_driver *driver; 2769 2770 TAILQ_FOREACH(driver, &g_accel_drivers, tailq) { 2771 if (strcmp(driver->name, name) == 0) { 2772 return driver; 2773 } 2774 } 2775 2776 return NULL; 2777 } 2778 2779 int 2780 spdk_accel_set_driver(const char *name) 2781 { 2782 struct spdk_accel_driver *driver; 2783 2784 driver = accel_find_driver(name); 2785 if (driver == NULL) { 2786 SPDK_ERRLOG("Couldn't find driver named '%s'\n", name); 2787 return -ENODEV; 2788 } 2789 2790 g_accel_driver = driver; 2791 2792 return 0; 2793 } 2794 2795 void 2796 spdk_accel_driver_register(struct spdk_accel_driver *driver) 2797 { 2798 if (accel_find_driver(driver->name)) { 2799 SPDK_ERRLOG("Driver named '%s' has already been registered\n", driver->name); 2800 assert(0); 2801 return; 2802 } 2803 2804 TAILQ_INSERT_TAIL(&g_accel_drivers, driver, tailq); 2805 } 2806 2807 int 2808 spdk_accel_set_opts(const struct spdk_accel_opts *opts) 2809 { 2810 if (opts->size > sizeof(*opts)) { 2811 return -EINVAL; 2812 } 2813 2814 memcpy(&g_opts, opts, opts->size); 2815 2816 return 0; 2817 } 2818 2819 void 2820 spdk_accel_get_opts(struct spdk_accel_opts *opts) 2821 { 2822 size_t size = opts->size; 2823 2824 assert(size <= sizeof(*opts)); 2825 2826 memcpy(opts, &g_opts, spdk_min(sizeof(*opts), size)); 2827 opts->size = size; 2828 } 2829 2830 struct accel_get_stats_ctx { 2831 struct accel_stats stats; 2832 accel_get_stats_cb cb_fn; 2833 void *cb_arg; 2834 }; 2835 2836 static void 2837 accel_get_channel_stats_done(struct spdk_io_channel_iter *iter, int status) 2838 { 2839 struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter); 2840 2841 ctx->cb_fn(&ctx->stats, ctx->cb_arg); 2842 free(ctx); 2843 } 2844 2845 static void 2846 accel_get_channel_stats(struct spdk_io_channel_iter *iter) 2847 { 2848 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(iter); 2849 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 2850 struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter); 2851 2852 accel_add_stats(&ctx->stats, &accel_ch->stats); 2853 spdk_for_each_channel_continue(iter, 0); 2854 } 2855 2856 int 2857 accel_get_stats(accel_get_stats_cb cb_fn, void *cb_arg) 2858 { 2859 struct accel_get_stats_ctx *ctx; 2860 2861 ctx = calloc(1, sizeof(*ctx)); 2862 if (ctx == NULL) { 2863 return -ENOMEM; 2864 } 2865 2866 spdk_spin_lock(&g_stats_lock); 2867 accel_add_stats(&ctx->stats, &g_stats); 2868 spdk_spin_unlock(&g_stats_lock); 2869 2870 ctx->cb_fn = cb_fn; 2871 ctx->cb_arg = cb_arg; 2872 2873 spdk_for_each_channel(&spdk_accel_module_list, accel_get_channel_stats, ctx, 2874 accel_get_channel_stats_done); 2875 2876 return 0; 2877 } 2878 2879 void 2880 spdk_accel_get_opcode_stats(struct spdk_io_channel *ch, enum spdk_accel_opcode opcode, 2881 struct spdk_accel_opcode_stats *stats, size_t size) 2882 { 2883 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 2884 2885 #define FIELD_OK(field) \ 2886 offsetof(struct spdk_accel_opcode_stats, field) + sizeof(stats->field) <= size 2887 2888 #define SET_FIELD(field, value) \ 2889 if (FIELD_OK(field)) { \ 2890 stats->field = value; \ 2891 } 2892 2893 SET_FIELD(executed, accel_ch->stats.operations[opcode].executed); 2894 SET_FIELD(failed, accel_ch->stats.operations[opcode].failed); 2895 SET_FIELD(num_bytes, accel_ch->stats.operations[opcode].num_bytes); 2896 2897 #undef FIELD_OK 2898 #undef SET_FIELD 2899 } 2900 2901 uint8_t 2902 spdk_accel_get_buf_align(enum spdk_accel_opcode opcode, 2903 const struct spdk_accel_operation_exec_ctx *ctx) 2904 { 2905 struct spdk_accel_module_if *module = g_modules_opc[opcode].module; 2906 struct spdk_accel_opcode_info modinfo = {}, drvinfo = {}; 2907 2908 if (g_accel_driver != NULL && g_accel_driver->get_operation_info != NULL) { 2909 g_accel_driver->get_operation_info(opcode, ctx, &drvinfo); 2910 } 2911 2912 if (module->get_operation_info != NULL) { 2913 module->get_operation_info(opcode, ctx, &modinfo); 2914 } 2915 2916 /* If a driver is set, it'll execute most of the operations, while the rest will usually 2917 * fall back to accel_sw, which doesn't have any alignment requiremenets. However, to be 2918 * extra safe, return the max(driver, module) if a driver delegates some operations to a 2919 * hardware module. */ 2920 return spdk_max(modinfo.required_alignment, drvinfo.required_alignment); 2921 } 2922 2923 SPDK_LOG_REGISTER_COMPONENT(accel) 2924