1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES. 4 * All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/accel_module.h" 10 11 #include "accel_internal.h" 12 13 #include "spdk/dma.h" 14 #include "spdk/env.h" 15 #include "spdk/likely.h" 16 #include "spdk/log.h" 17 #include "spdk/thread.h" 18 #include "spdk/json.h" 19 #include "spdk/crc32.h" 20 #include "spdk/util.h" 21 #include "spdk/hexlify.h" 22 #include "spdk/string.h" 23 24 /* Accelerator Framework: The following provides a top level 25 * generic API for the accelerator functions defined here. Modules, 26 * such as the one in /module/accel/ioat, supply the implementation 27 * with the exception of the pure software implementation contained 28 * later in this file. 29 */ 30 31 #define ALIGN_4K 0x1000 32 #define MAX_TASKS_PER_CHANNEL 0x800 33 #define ACCEL_SMALL_CACHE_SIZE 128 34 #define ACCEL_LARGE_CACHE_SIZE 16 35 /* Set MSB, so we don't return NULL pointers as buffers */ 36 #define ACCEL_BUFFER_BASE ((void *)(1ull << 63)) 37 #define ACCEL_BUFFER_OFFSET_MASK ((uintptr_t)ACCEL_BUFFER_BASE - 1) 38 39 #define ACCEL_CRYPTO_TWEAK_MODE_DEFAULT SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA 40 41 struct accel_module { 42 struct spdk_accel_module_if *module; 43 bool supports_memory_domains; 44 }; 45 46 /* Largest context size for all accel modules */ 47 static size_t g_max_accel_module_size = sizeof(struct spdk_accel_task); 48 49 static struct spdk_accel_module_if *g_accel_module = NULL; 50 static spdk_accel_fini_cb g_fini_cb_fn = NULL; 51 static void *g_fini_cb_arg = NULL; 52 static bool g_modules_started = false; 53 static struct spdk_memory_domain *g_accel_domain; 54 55 /* Global list of registered accelerator modules */ 56 static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list = 57 TAILQ_HEAD_INITIALIZER(spdk_accel_module_list); 58 59 /* Crypto keyring */ 60 static TAILQ_HEAD(, spdk_accel_crypto_key) g_keyring = TAILQ_HEAD_INITIALIZER(g_keyring); 61 static struct spdk_spinlock g_keyring_spin; 62 63 /* Global array mapping capabilities to modules */ 64 static struct accel_module g_modules_opc[SPDK_ACCEL_OPC_LAST] = {}; 65 static char *g_modules_opc_override[SPDK_ACCEL_OPC_LAST] = {}; 66 TAILQ_HEAD(, spdk_accel_driver) g_accel_drivers = TAILQ_HEAD_INITIALIZER(g_accel_drivers); 67 static struct spdk_accel_driver *g_accel_driver; 68 static struct spdk_accel_opts g_opts = { 69 .small_cache_size = ACCEL_SMALL_CACHE_SIZE, 70 .large_cache_size = ACCEL_LARGE_CACHE_SIZE, 71 .task_count = MAX_TASKS_PER_CHANNEL, 72 .sequence_count = MAX_TASKS_PER_CHANNEL, 73 .buf_count = MAX_TASKS_PER_CHANNEL, 74 }; 75 static struct accel_stats g_stats; 76 static struct spdk_spinlock g_stats_lock; 77 78 static const char *g_opcode_strings[SPDK_ACCEL_OPC_LAST] = { 79 "copy", "fill", "dualcast", "compare", "crc32c", "copy_crc32c", 80 "compress", "decompress", "encrypt", "decrypt", "xor", 81 "dif_verify", "dif_verify_copy", "dif_generate", "dif_generate_copy" 82 }; 83 84 enum accel_sequence_state { 85 ACCEL_SEQUENCE_STATE_INIT, 86 ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF, 87 ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF, 88 ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF, 89 ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF, 90 ACCEL_SEQUENCE_STATE_PULL_DATA, 91 ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA, 92 ACCEL_SEQUENCE_STATE_EXEC_TASK, 93 ACCEL_SEQUENCE_STATE_AWAIT_TASK, 94 ACCEL_SEQUENCE_STATE_COMPLETE_TASK, 95 ACCEL_SEQUENCE_STATE_NEXT_TASK, 96 ACCEL_SEQUENCE_STATE_PUSH_DATA, 97 ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA, 98 ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS, 99 ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS, 100 ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS, 101 ACCEL_SEQUENCE_STATE_ERROR, 102 ACCEL_SEQUENCE_STATE_MAX, 103 }; 104 105 static const char *g_seq_states[] 106 __attribute__((unused)) = { 107 [ACCEL_SEQUENCE_STATE_INIT] = "init", 108 [ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF] = "check-virtbuf", 109 [ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF] = "await-virtbuf", 110 [ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF] = "check-bouncebuf", 111 [ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF] = "await-bouncebuf", 112 [ACCEL_SEQUENCE_STATE_PULL_DATA] = "pull-data", 113 [ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA] = "await-pull-data", 114 [ACCEL_SEQUENCE_STATE_EXEC_TASK] = "exec-task", 115 [ACCEL_SEQUENCE_STATE_AWAIT_TASK] = "await-task", 116 [ACCEL_SEQUENCE_STATE_COMPLETE_TASK] = "complete-task", 117 [ACCEL_SEQUENCE_STATE_NEXT_TASK] = "next-task", 118 [ACCEL_SEQUENCE_STATE_PUSH_DATA] = "push-data", 119 [ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA] = "await-push-data", 120 [ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS] = "driver-exec-tasks", 121 [ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS] = "driver-await-tasks", 122 [ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS] = "driver-complete-tasks", 123 [ACCEL_SEQUENCE_STATE_ERROR] = "error", 124 [ACCEL_SEQUENCE_STATE_MAX] = "", 125 }; 126 127 #define ACCEL_SEQUENCE_STATE_STRING(s) \ 128 (((s) >= ACCEL_SEQUENCE_STATE_INIT && (s) < ACCEL_SEQUENCE_STATE_MAX) \ 129 ? g_seq_states[s] : "unknown") 130 131 struct accel_buffer { 132 struct spdk_accel_sequence *seq; 133 void *buf; 134 uint64_t len; 135 struct spdk_iobuf_entry iobuf; 136 spdk_accel_sequence_get_buf_cb cb_fn; 137 void *cb_ctx; 138 SLIST_ENTRY(accel_buffer) link; 139 struct accel_io_channel *ch; 140 }; 141 142 struct accel_io_channel { 143 struct spdk_io_channel *module_ch[SPDK_ACCEL_OPC_LAST]; 144 struct spdk_io_channel *driver_channel; 145 void *task_pool_base; 146 struct spdk_accel_sequence *seq_pool_base; 147 struct accel_buffer *buf_pool_base; 148 struct spdk_accel_task_aux_data *task_aux_data_base; 149 STAILQ_HEAD(, spdk_accel_task) task_pool; 150 SLIST_HEAD(, spdk_accel_task_aux_data) task_aux_data_pool; 151 SLIST_HEAD(, spdk_accel_sequence) seq_pool; 152 SLIST_HEAD(, accel_buffer) buf_pool; 153 struct spdk_iobuf_channel iobuf; 154 struct accel_stats stats; 155 }; 156 157 TAILQ_HEAD(accel_sequence_tasks, spdk_accel_task); 158 159 struct spdk_accel_sequence { 160 struct accel_io_channel *ch; 161 struct accel_sequence_tasks tasks; 162 SLIST_HEAD(, accel_buffer) bounce_bufs; 163 int status; 164 /* state uses enum accel_sequence_state */ 165 uint8_t state; 166 bool in_process_sequence; 167 spdk_accel_completion_cb cb_fn; 168 void *cb_arg; 169 SLIST_ENTRY(spdk_accel_sequence) link; 170 }; 171 SPDK_STATIC_ASSERT(sizeof(struct spdk_accel_sequence) == 64, "invalid size"); 172 173 #define accel_update_stats(ch, event, v) \ 174 do { \ 175 (ch)->stats.event += (v); \ 176 } while (0) 177 178 #define accel_update_task_stats(ch, task, event, v) \ 179 accel_update_stats(ch, operations[(task)->op_code].event, v) 180 181 static inline void accel_sequence_task_cb(void *cb_arg, int status); 182 183 static inline void 184 accel_sequence_set_state(struct spdk_accel_sequence *seq, enum accel_sequence_state state) 185 { 186 SPDK_DEBUGLOG(accel, "seq=%p, setting state: %s -> %s\n", seq, 187 ACCEL_SEQUENCE_STATE_STRING(seq->state), ACCEL_SEQUENCE_STATE_STRING(state)); 188 assert(seq->state != ACCEL_SEQUENCE_STATE_ERROR || state == ACCEL_SEQUENCE_STATE_ERROR); 189 seq->state = state; 190 } 191 192 static void 193 accel_sequence_set_fail(struct spdk_accel_sequence *seq, int status) 194 { 195 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR); 196 assert(status != 0); 197 seq->status = status; 198 } 199 200 int 201 spdk_accel_get_opc_module_name(enum spdk_accel_opcode opcode, const char **module_name) 202 { 203 if (opcode >= SPDK_ACCEL_OPC_LAST) { 204 /* invalid opcode */ 205 return -EINVAL; 206 } 207 208 if (g_modules_opc[opcode].module) { 209 *module_name = g_modules_opc[opcode].module->name; 210 } else { 211 return -ENOENT; 212 } 213 214 return 0; 215 } 216 217 void 218 _accel_for_each_module(struct module_info *info, _accel_for_each_module_fn fn) 219 { 220 struct spdk_accel_module_if *accel_module; 221 enum spdk_accel_opcode opcode; 222 int j = 0; 223 224 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 225 for (opcode = 0; opcode < SPDK_ACCEL_OPC_LAST; opcode++) { 226 if (accel_module->supports_opcode(opcode)) { 227 info->ops[j] = opcode; 228 j++; 229 } 230 } 231 info->name = accel_module->name; 232 info->num_ops = j; 233 fn(info); 234 j = 0; 235 } 236 } 237 238 const char * 239 spdk_accel_get_opcode_name(enum spdk_accel_opcode opcode) 240 { 241 if (opcode < SPDK_ACCEL_OPC_LAST) { 242 return g_opcode_strings[opcode]; 243 } 244 245 return NULL; 246 } 247 248 int 249 spdk_accel_assign_opc(enum spdk_accel_opcode opcode, const char *name) 250 { 251 char *copy; 252 253 if (g_modules_started == true) { 254 /* we don't allow re-assignment once things have started */ 255 return -EINVAL; 256 } 257 258 if (opcode >= SPDK_ACCEL_OPC_LAST) { 259 /* invalid opcode */ 260 return -EINVAL; 261 } 262 263 copy = strdup(name); 264 if (copy == NULL) { 265 return -ENOMEM; 266 } 267 268 /* module selection will be validated after the framework starts. */ 269 free(g_modules_opc_override[opcode]); 270 g_modules_opc_override[opcode] = copy; 271 272 return 0; 273 } 274 275 void 276 spdk_accel_task_complete(struct spdk_accel_task *accel_task, int status) 277 { 278 struct accel_io_channel *accel_ch = accel_task->accel_ch; 279 spdk_accel_completion_cb cb_fn; 280 void *cb_arg; 281 282 accel_update_task_stats(accel_ch, accel_task, executed, 1); 283 accel_update_task_stats(accel_ch, accel_task, num_bytes, accel_task->nbytes); 284 if (spdk_unlikely(status != 0)) { 285 accel_update_task_stats(accel_ch, accel_task, failed, 1); 286 } 287 288 if (accel_task->seq) { 289 accel_sequence_task_cb(accel_task->seq, status); 290 return; 291 } 292 293 cb_fn = accel_task->cb_fn; 294 cb_arg = accel_task->cb_arg; 295 296 if (accel_task->has_aux) { 297 SLIST_INSERT_HEAD(&accel_ch->task_aux_data_pool, accel_task->aux, link); 298 accel_task->aux = NULL; 299 accel_task->has_aux = false; 300 } 301 302 /* We should put the accel_task into the list firstly in order to avoid 303 * the accel task list is exhausted when there is recursive call to 304 * allocate accel_task in user's call back function (cb_fn) 305 */ 306 STAILQ_INSERT_HEAD(&accel_ch->task_pool, accel_task, link); 307 308 cb_fn(cb_arg, status); 309 } 310 311 inline static struct spdk_accel_task * 312 _get_task(struct accel_io_channel *accel_ch, spdk_accel_completion_cb cb_fn, void *cb_arg) 313 { 314 struct spdk_accel_task *accel_task; 315 316 accel_task = STAILQ_FIRST(&accel_ch->task_pool); 317 if (spdk_unlikely(accel_task == NULL)) { 318 accel_update_stats(accel_ch, retry.task, 1); 319 return NULL; 320 } 321 322 STAILQ_REMOVE_HEAD(&accel_ch->task_pool, link); 323 accel_task->link.stqe_next = NULL; 324 325 accel_task->cb_fn = cb_fn; 326 accel_task->cb_arg = cb_arg; 327 accel_task->accel_ch = accel_ch; 328 accel_task->s.iovs = NULL; 329 accel_task->d.iovs = NULL; 330 331 return accel_task; 332 } 333 334 static inline int 335 accel_submit_task(struct accel_io_channel *accel_ch, struct spdk_accel_task *task) 336 { 337 struct spdk_io_channel *module_ch = accel_ch->module_ch[task->op_code]; 338 struct spdk_accel_module_if *module = g_modules_opc[task->op_code].module; 339 int rc; 340 341 rc = module->submit_tasks(module_ch, task); 342 if (spdk_unlikely(rc != 0)) { 343 accel_update_task_stats(accel_ch, task, failed, 1); 344 } 345 346 return rc; 347 } 348 349 static inline uint64_t 350 accel_get_iovlen(struct iovec *iovs, uint32_t iovcnt) 351 { 352 uint64_t result = 0; 353 uint32_t i; 354 355 for (i = 0; i < iovcnt; ++i) { 356 result += iovs[i].iov_len; 357 } 358 359 return result; 360 } 361 362 #define ACCEL_TASK_ALLOC_AUX_BUF(task) \ 363 do { \ 364 (task)->aux = SLIST_FIRST(&(task)->accel_ch->task_aux_data_pool); \ 365 if (spdk_unlikely(!(task)->aux)) { \ 366 SPDK_ERRLOG("Fatal problem, aux data was not allocated\n"); \ 367 STAILQ_INSERT_HEAD(&(task)->accel_ch->task_pool, (task), link); \ 368 assert(0); \ 369 return -ENOMEM; \ 370 } \ 371 SLIST_REMOVE_HEAD(&(task)->accel_ch->task_aux_data_pool, link); \ 372 (task)->has_aux = true; \ 373 } while (0) 374 375 /* Accel framework public API for copy function */ 376 int 377 spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, 378 uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg) 379 { 380 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 381 struct spdk_accel_task *accel_task; 382 383 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 384 if (spdk_unlikely(accel_task == NULL)) { 385 return -ENOMEM; 386 } 387 388 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 389 390 accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC]; 391 accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 392 accel_task->d.iovs[0].iov_base = dst; 393 accel_task->d.iovs[0].iov_len = nbytes; 394 accel_task->d.iovcnt = 1; 395 accel_task->s.iovs[0].iov_base = src; 396 accel_task->s.iovs[0].iov_len = nbytes; 397 accel_task->s.iovcnt = 1; 398 accel_task->nbytes = nbytes; 399 accel_task->op_code = SPDK_ACCEL_OPC_COPY; 400 accel_task->src_domain = NULL; 401 accel_task->dst_domain = NULL; 402 403 return accel_submit_task(accel_ch, accel_task); 404 } 405 406 /* Accel framework public API for dual cast copy function */ 407 int 408 spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, 409 void *dst2, void *src, uint64_t nbytes, 410 spdk_accel_completion_cb cb_fn, void *cb_arg) 411 { 412 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 413 struct spdk_accel_task *accel_task; 414 415 if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) { 416 SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n"); 417 return -EINVAL; 418 } 419 420 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 421 if (spdk_unlikely(accel_task == NULL)) { 422 return -ENOMEM; 423 } 424 425 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 426 427 accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC]; 428 accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 429 accel_task->d2.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST2]; 430 accel_task->d.iovs[0].iov_base = dst1; 431 accel_task->d.iovs[0].iov_len = nbytes; 432 accel_task->d.iovcnt = 1; 433 accel_task->d2.iovs[0].iov_base = dst2; 434 accel_task->d2.iovs[0].iov_len = nbytes; 435 accel_task->d2.iovcnt = 1; 436 accel_task->s.iovs[0].iov_base = src; 437 accel_task->s.iovs[0].iov_len = nbytes; 438 accel_task->s.iovcnt = 1; 439 accel_task->nbytes = nbytes; 440 accel_task->op_code = SPDK_ACCEL_OPC_DUALCAST; 441 accel_task->src_domain = NULL; 442 accel_task->dst_domain = NULL; 443 444 return accel_submit_task(accel_ch, accel_task); 445 } 446 447 /* Accel framework public API for compare function */ 448 449 int 450 spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, 451 void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn, 452 void *cb_arg) 453 { 454 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 455 struct spdk_accel_task *accel_task; 456 457 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 458 if (spdk_unlikely(accel_task == NULL)) { 459 return -ENOMEM; 460 } 461 462 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 463 464 accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC]; 465 accel_task->s2.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC2]; 466 accel_task->s.iovs[0].iov_base = src1; 467 accel_task->s.iovs[0].iov_len = nbytes; 468 accel_task->s.iovcnt = 1; 469 accel_task->s2.iovs[0].iov_base = src2; 470 accel_task->s2.iovs[0].iov_len = nbytes; 471 accel_task->s2.iovcnt = 1; 472 accel_task->nbytes = nbytes; 473 accel_task->op_code = SPDK_ACCEL_OPC_COMPARE; 474 accel_task->src_domain = NULL; 475 accel_task->dst_domain = NULL; 476 477 return accel_submit_task(accel_ch, accel_task); 478 } 479 480 /* Accel framework public API for fill function */ 481 int 482 spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, 483 uint8_t fill, uint64_t nbytes, 484 spdk_accel_completion_cb cb_fn, void *cb_arg) 485 { 486 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 487 struct spdk_accel_task *accel_task; 488 489 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 490 if (spdk_unlikely(accel_task == NULL)) { 491 return -ENOMEM; 492 } 493 494 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 495 496 accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 497 accel_task->d.iovs[0].iov_base = dst; 498 accel_task->d.iovs[0].iov_len = nbytes; 499 accel_task->d.iovcnt = 1; 500 accel_task->nbytes = nbytes; 501 memset(&accel_task->fill_pattern, fill, sizeof(uint64_t)); 502 accel_task->op_code = SPDK_ACCEL_OPC_FILL; 503 accel_task->src_domain = NULL; 504 accel_task->dst_domain = NULL; 505 506 return accel_submit_task(accel_ch, accel_task); 507 } 508 509 /* Accel framework public API for CRC-32C function */ 510 int 511 spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, 512 void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn, 513 void *cb_arg) 514 { 515 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 516 struct spdk_accel_task *accel_task; 517 518 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 519 if (spdk_unlikely(accel_task == NULL)) { 520 return -ENOMEM; 521 } 522 523 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 524 525 accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC]; 526 accel_task->s.iovs[0].iov_base = src; 527 accel_task->s.iovs[0].iov_len = nbytes; 528 accel_task->s.iovcnt = 1; 529 accel_task->nbytes = nbytes; 530 accel_task->crc_dst = crc_dst; 531 accel_task->seed = seed; 532 accel_task->op_code = SPDK_ACCEL_OPC_CRC32C; 533 accel_task->src_domain = NULL; 534 accel_task->dst_domain = NULL; 535 536 return accel_submit_task(accel_ch, accel_task); 537 } 538 539 /* Accel framework public API for chained CRC-32C function */ 540 int 541 spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, 542 struct iovec *iov, uint32_t iov_cnt, uint32_t seed, 543 spdk_accel_completion_cb cb_fn, void *cb_arg) 544 { 545 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 546 struct spdk_accel_task *accel_task; 547 548 if (iov == NULL) { 549 SPDK_ERRLOG("iov should not be NULL"); 550 return -EINVAL; 551 } 552 553 if (!iov_cnt) { 554 SPDK_ERRLOG("iovcnt should not be zero value\n"); 555 return -EINVAL; 556 } 557 558 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 559 if (spdk_unlikely(accel_task == NULL)) { 560 SPDK_ERRLOG("no memory\n"); 561 assert(0); 562 return -ENOMEM; 563 } 564 565 accel_task->s.iovs = iov; 566 accel_task->s.iovcnt = iov_cnt; 567 accel_task->nbytes = accel_get_iovlen(iov, iov_cnt); 568 accel_task->crc_dst = crc_dst; 569 accel_task->seed = seed; 570 accel_task->op_code = SPDK_ACCEL_OPC_CRC32C; 571 accel_task->src_domain = NULL; 572 accel_task->dst_domain = NULL; 573 574 return accel_submit_task(accel_ch, accel_task); 575 } 576 577 /* Accel framework public API for copy with CRC-32C function */ 578 int 579 spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, 580 void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes, 581 spdk_accel_completion_cb cb_fn, void *cb_arg) 582 { 583 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 584 struct spdk_accel_task *accel_task; 585 586 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 587 if (spdk_unlikely(accel_task == NULL)) { 588 return -ENOMEM; 589 } 590 591 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 592 593 accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC]; 594 accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 595 accel_task->d.iovs[0].iov_base = dst; 596 accel_task->d.iovs[0].iov_len = nbytes; 597 accel_task->d.iovcnt = 1; 598 accel_task->s.iovs[0].iov_base = src; 599 accel_task->s.iovs[0].iov_len = nbytes; 600 accel_task->s.iovcnt = 1; 601 accel_task->nbytes = nbytes; 602 accel_task->crc_dst = crc_dst; 603 accel_task->seed = seed; 604 accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C; 605 accel_task->src_domain = NULL; 606 accel_task->dst_domain = NULL; 607 608 return accel_submit_task(accel_ch, accel_task); 609 } 610 611 /* Accel framework public API for chained copy + CRC-32C function */ 612 int 613 spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, 614 struct iovec *src_iovs, uint32_t iov_cnt, uint32_t *crc_dst, 615 uint32_t seed, spdk_accel_completion_cb cb_fn, void *cb_arg) 616 { 617 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 618 struct spdk_accel_task *accel_task; 619 uint64_t nbytes; 620 621 if (src_iovs == NULL) { 622 SPDK_ERRLOG("iov should not be NULL"); 623 return -EINVAL; 624 } 625 626 if (!iov_cnt) { 627 SPDK_ERRLOG("iovcnt should not be zero value\n"); 628 return -EINVAL; 629 } 630 631 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 632 if (spdk_unlikely(accel_task == NULL)) { 633 SPDK_ERRLOG("no memory\n"); 634 assert(0); 635 return -ENOMEM; 636 } 637 638 nbytes = accel_get_iovlen(src_iovs, iov_cnt); 639 640 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 641 642 accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 643 accel_task->d.iovs[0].iov_base = dst; 644 accel_task->d.iovs[0].iov_len = nbytes; 645 accel_task->d.iovcnt = 1; 646 accel_task->s.iovs = src_iovs; 647 accel_task->s.iovcnt = iov_cnt; 648 accel_task->nbytes = nbytes; 649 accel_task->crc_dst = crc_dst; 650 accel_task->seed = seed; 651 accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C; 652 accel_task->src_domain = NULL; 653 accel_task->dst_domain = NULL; 654 655 return accel_submit_task(accel_ch, accel_task); 656 } 657 658 int 659 spdk_accel_submit_compress(struct spdk_io_channel *ch, void *dst, uint64_t nbytes, 660 struct iovec *src_iovs, size_t src_iovcnt, uint32_t *output_size, 661 spdk_accel_completion_cb cb_fn, void *cb_arg) 662 { 663 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 664 struct spdk_accel_task *accel_task; 665 666 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 667 if (spdk_unlikely(accel_task == NULL)) { 668 return -ENOMEM; 669 } 670 671 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 672 673 accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 674 accel_task->d.iovs[0].iov_base = dst; 675 accel_task->d.iovs[0].iov_len = nbytes; 676 accel_task->d.iovcnt = 1; 677 accel_task->output_size = output_size; 678 accel_task->s.iovs = src_iovs; 679 accel_task->s.iovcnt = src_iovcnt; 680 accel_task->nbytes = nbytes; 681 accel_task->op_code = SPDK_ACCEL_OPC_COMPRESS; 682 accel_task->src_domain = NULL; 683 accel_task->dst_domain = NULL; 684 685 return accel_submit_task(accel_ch, accel_task); 686 } 687 688 int 689 spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_iovs, 690 size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt, 691 uint32_t *output_size, spdk_accel_completion_cb cb_fn, 692 void *cb_arg) 693 { 694 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 695 struct spdk_accel_task *accel_task; 696 697 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 698 if (spdk_unlikely(accel_task == NULL)) { 699 return -ENOMEM; 700 } 701 702 accel_task->output_size = output_size; 703 accel_task->s.iovs = src_iovs; 704 accel_task->s.iovcnt = src_iovcnt; 705 accel_task->d.iovs = dst_iovs; 706 accel_task->d.iovcnt = dst_iovcnt; 707 accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 708 accel_task->op_code = SPDK_ACCEL_OPC_DECOMPRESS; 709 accel_task->src_domain = NULL; 710 accel_task->dst_domain = NULL; 711 712 return accel_submit_task(accel_ch, accel_task); 713 } 714 715 int 716 spdk_accel_submit_encrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key, 717 struct iovec *dst_iovs, uint32_t dst_iovcnt, 718 struct iovec *src_iovs, uint32_t src_iovcnt, 719 uint64_t iv, uint32_t block_size, 720 spdk_accel_completion_cb cb_fn, void *cb_arg) 721 { 722 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 723 struct spdk_accel_task *accel_task; 724 725 if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) { 726 return -EINVAL; 727 } 728 729 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 730 if (spdk_unlikely(accel_task == NULL)) { 731 return -ENOMEM; 732 } 733 734 accel_task->crypto_key = key; 735 accel_task->s.iovs = src_iovs; 736 accel_task->s.iovcnt = src_iovcnt; 737 accel_task->d.iovs = dst_iovs; 738 accel_task->d.iovcnt = dst_iovcnt; 739 accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 740 accel_task->iv = iv; 741 accel_task->block_size = block_size; 742 accel_task->op_code = SPDK_ACCEL_OPC_ENCRYPT; 743 accel_task->src_domain = NULL; 744 accel_task->dst_domain = NULL; 745 746 return accel_submit_task(accel_ch, accel_task); 747 } 748 749 int 750 spdk_accel_submit_decrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key, 751 struct iovec *dst_iovs, uint32_t dst_iovcnt, 752 struct iovec *src_iovs, uint32_t src_iovcnt, 753 uint64_t iv, uint32_t block_size, 754 spdk_accel_completion_cb cb_fn, void *cb_arg) 755 { 756 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 757 struct spdk_accel_task *accel_task; 758 759 if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) { 760 return -EINVAL; 761 } 762 763 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 764 if (spdk_unlikely(accel_task == NULL)) { 765 return -ENOMEM; 766 } 767 768 accel_task->crypto_key = key; 769 accel_task->s.iovs = src_iovs; 770 accel_task->s.iovcnt = src_iovcnt; 771 accel_task->d.iovs = dst_iovs; 772 accel_task->d.iovcnt = dst_iovcnt; 773 accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 774 accel_task->iv = iv; 775 accel_task->block_size = block_size; 776 accel_task->op_code = SPDK_ACCEL_OPC_DECRYPT; 777 accel_task->src_domain = NULL; 778 accel_task->dst_domain = NULL; 779 780 return accel_submit_task(accel_ch, accel_task); 781 } 782 783 int 784 spdk_accel_submit_xor(struct spdk_io_channel *ch, void *dst, void **sources, uint32_t nsrcs, 785 uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg) 786 { 787 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 788 struct spdk_accel_task *accel_task; 789 790 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 791 if (spdk_unlikely(accel_task == NULL)) { 792 return -ENOMEM; 793 } 794 795 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 796 797 accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 798 accel_task->nsrcs.srcs = sources; 799 accel_task->nsrcs.cnt = nsrcs; 800 accel_task->d.iovs[0].iov_base = dst; 801 accel_task->d.iovs[0].iov_len = nbytes; 802 accel_task->d.iovcnt = 1; 803 accel_task->nbytes = nbytes; 804 accel_task->op_code = SPDK_ACCEL_OPC_XOR; 805 accel_task->src_domain = NULL; 806 accel_task->dst_domain = NULL; 807 808 return accel_submit_task(accel_ch, accel_task); 809 } 810 811 int 812 spdk_accel_submit_dif_verify(struct spdk_io_channel *ch, 813 struct iovec *iovs, size_t iovcnt, uint32_t num_blocks, 814 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err, 815 spdk_accel_completion_cb cb_fn, void *cb_arg) 816 { 817 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 818 struct spdk_accel_task *accel_task; 819 820 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 821 if (accel_task == NULL) { 822 return -ENOMEM; 823 } 824 825 accel_task->s.iovs = iovs; 826 accel_task->s.iovcnt = iovcnt; 827 accel_task->dif.ctx = ctx; 828 accel_task->dif.err = err; 829 accel_task->dif.num_blocks = num_blocks; 830 accel_task->nbytes = num_blocks * ctx->block_size; 831 accel_task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY; 832 accel_task->src_domain = NULL; 833 accel_task->dst_domain = NULL; 834 835 return accel_submit_task(accel_ch, accel_task); 836 } 837 838 int 839 spdk_accel_submit_dif_generate(struct spdk_io_channel *ch, 840 struct iovec *iovs, size_t iovcnt, uint32_t num_blocks, 841 const struct spdk_dif_ctx *ctx, 842 spdk_accel_completion_cb cb_fn, void *cb_arg) 843 { 844 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 845 struct spdk_accel_task *accel_task; 846 847 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 848 if (accel_task == NULL) { 849 return -ENOMEM; 850 } 851 852 accel_task->s.iovs = iovs; 853 accel_task->s.iovcnt = iovcnt; 854 accel_task->dif.ctx = ctx; 855 accel_task->dif.num_blocks = num_blocks; 856 accel_task->nbytes = num_blocks * ctx->block_size; 857 accel_task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE; 858 accel_task->src_domain = NULL; 859 accel_task->dst_domain = NULL; 860 861 return accel_submit_task(accel_ch, accel_task); 862 } 863 864 int 865 spdk_accel_submit_dif_generate_copy(struct spdk_io_channel *ch, struct iovec *dst_iovs, 866 size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt, 867 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 868 spdk_accel_completion_cb cb_fn, void *cb_arg) 869 { 870 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 871 struct spdk_accel_task *accel_task; 872 873 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 874 if (accel_task == NULL) { 875 return -ENOMEM; 876 } 877 878 accel_task->s.iovs = src_iovs; 879 accel_task->s.iovcnt = src_iovcnt; 880 accel_task->d.iovs = dst_iovs; 881 accel_task->d.iovcnt = dst_iovcnt; 882 accel_task->dif.ctx = ctx; 883 accel_task->dif.num_blocks = num_blocks; 884 accel_task->nbytes = num_blocks * ctx->block_size; 885 accel_task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE_COPY; 886 accel_task->src_domain = NULL; 887 accel_task->dst_domain = NULL; 888 889 return accel_submit_task(accel_ch, accel_task); 890 } 891 892 int 893 spdk_accel_submit_dif_verify_copy(struct spdk_io_channel *ch, 894 struct iovec *dst_iovs, size_t dst_iovcnt, 895 struct iovec *src_iovs, size_t src_iovcnt, uint32_t num_blocks, 896 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err, 897 spdk_accel_completion_cb cb_fn, void *cb_arg) 898 { 899 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 900 struct spdk_accel_task *accel_task; 901 902 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 903 if (accel_task == NULL) { 904 return -ENOMEM; 905 } 906 907 accel_task->s.iovs = src_iovs; 908 accel_task->s.iovcnt = src_iovcnt; 909 accel_task->d.iovs = dst_iovs; 910 accel_task->d.iovcnt = dst_iovcnt; 911 accel_task->dif.ctx = ctx; 912 accel_task->dif.err = err; 913 accel_task->dif.num_blocks = num_blocks; 914 accel_task->nbytes = num_blocks * ctx->block_size; 915 accel_task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY_COPY; 916 accel_task->src_domain = NULL; 917 accel_task->dst_domain = NULL; 918 919 return accel_submit_task(accel_ch, accel_task); 920 } 921 922 static inline struct accel_buffer * 923 accel_get_buf(struct accel_io_channel *ch, uint64_t len) 924 { 925 struct accel_buffer *buf; 926 927 buf = SLIST_FIRST(&ch->buf_pool); 928 if (spdk_unlikely(buf == NULL)) { 929 accel_update_stats(ch, retry.bufdesc, 1); 930 return NULL; 931 } 932 933 SLIST_REMOVE_HEAD(&ch->buf_pool, link); 934 buf->len = len; 935 buf->buf = NULL; 936 buf->seq = NULL; 937 buf->cb_fn = NULL; 938 939 return buf; 940 } 941 942 static inline void 943 accel_put_buf(struct accel_io_channel *ch, struct accel_buffer *buf) 944 { 945 if (buf->buf != NULL) { 946 spdk_iobuf_put(&ch->iobuf, buf->buf, buf->len); 947 } 948 949 SLIST_INSERT_HEAD(&ch->buf_pool, buf, link); 950 } 951 952 static inline struct spdk_accel_sequence * 953 accel_sequence_get(struct accel_io_channel *ch) 954 { 955 struct spdk_accel_sequence *seq; 956 957 seq = SLIST_FIRST(&ch->seq_pool); 958 if (spdk_unlikely(seq == NULL)) { 959 accel_update_stats(ch, retry.sequence, 1); 960 return NULL; 961 } 962 963 SLIST_REMOVE_HEAD(&ch->seq_pool, link); 964 965 TAILQ_INIT(&seq->tasks); 966 SLIST_INIT(&seq->bounce_bufs); 967 968 seq->ch = ch; 969 seq->status = 0; 970 seq->state = ACCEL_SEQUENCE_STATE_INIT; 971 seq->in_process_sequence = false; 972 973 return seq; 974 } 975 976 static inline void 977 accel_sequence_put(struct spdk_accel_sequence *seq) 978 { 979 struct accel_io_channel *ch = seq->ch; 980 struct accel_buffer *buf; 981 982 while (!SLIST_EMPTY(&seq->bounce_bufs)) { 983 buf = SLIST_FIRST(&seq->bounce_bufs); 984 SLIST_REMOVE_HEAD(&seq->bounce_bufs, link); 985 accel_put_buf(seq->ch, buf); 986 } 987 988 assert(TAILQ_EMPTY(&seq->tasks)); 989 seq->ch = NULL; 990 991 SLIST_INSERT_HEAD(&ch->seq_pool, seq, link); 992 } 993 994 static void accel_sequence_task_cb(void *cb_arg, int status); 995 996 static inline struct spdk_accel_task * 997 accel_sequence_get_task(struct accel_io_channel *ch, struct spdk_accel_sequence *seq, 998 spdk_accel_step_cb cb_fn, void *cb_arg) 999 { 1000 struct spdk_accel_task *task; 1001 1002 task = _get_task(ch, NULL, NULL); 1003 if (spdk_unlikely(task == NULL)) { 1004 return task; 1005 } 1006 1007 task->step_cb_fn = cb_fn; 1008 task->cb_arg = cb_arg; 1009 task->seq = seq; 1010 1011 return task; 1012 } 1013 1014 int 1015 spdk_accel_append_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1016 struct iovec *dst_iovs, uint32_t dst_iovcnt, 1017 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 1018 struct iovec *src_iovs, uint32_t src_iovcnt, 1019 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 1020 spdk_accel_step_cb cb_fn, void *cb_arg) 1021 { 1022 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1023 struct spdk_accel_task *task; 1024 struct spdk_accel_sequence *seq = *pseq; 1025 1026 if (seq == NULL) { 1027 seq = accel_sequence_get(accel_ch); 1028 if (spdk_unlikely(seq == NULL)) { 1029 return -ENOMEM; 1030 } 1031 } 1032 1033 assert(seq->ch == accel_ch); 1034 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1035 if (spdk_unlikely(task == NULL)) { 1036 if (*pseq == NULL) { 1037 accel_sequence_put(seq); 1038 } 1039 1040 return -ENOMEM; 1041 } 1042 1043 task->dst_domain = dst_domain; 1044 task->dst_domain_ctx = dst_domain_ctx; 1045 task->d.iovs = dst_iovs; 1046 task->d.iovcnt = dst_iovcnt; 1047 task->src_domain = src_domain; 1048 task->src_domain_ctx = src_domain_ctx; 1049 task->s.iovs = src_iovs; 1050 task->s.iovcnt = src_iovcnt; 1051 task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 1052 task->op_code = SPDK_ACCEL_OPC_COPY; 1053 1054 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1055 *pseq = seq; 1056 1057 return 0; 1058 } 1059 1060 int 1061 spdk_accel_append_fill(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1062 void *buf, uint64_t len, 1063 struct spdk_memory_domain *domain, void *domain_ctx, uint8_t pattern, 1064 spdk_accel_step_cb cb_fn, void *cb_arg) 1065 { 1066 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1067 struct spdk_accel_task *task; 1068 struct spdk_accel_sequence *seq = *pseq; 1069 1070 if (seq == NULL) { 1071 seq = accel_sequence_get(accel_ch); 1072 if (spdk_unlikely(seq == NULL)) { 1073 return -ENOMEM; 1074 } 1075 } 1076 1077 assert(seq->ch == accel_ch); 1078 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1079 if (spdk_unlikely(task == NULL)) { 1080 if (*pseq == NULL) { 1081 accel_sequence_put(seq); 1082 } 1083 1084 return -ENOMEM; 1085 } 1086 1087 memset(&task->fill_pattern, pattern, sizeof(uint64_t)); 1088 1089 task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool); 1090 if (spdk_unlikely(!task->aux)) { 1091 SPDK_ERRLOG("Fatal problem, aux data was not allocated\n"); 1092 if (*pseq == NULL) { 1093 accel_sequence_put((seq)); 1094 } 1095 STAILQ_INSERT_HEAD(&task->accel_ch->task_pool, task, link); 1096 task->seq = NULL; 1097 assert(0); 1098 return -ENOMEM; 1099 } 1100 SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link); 1101 task->has_aux = true; 1102 1103 task->d.iovs = &task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 1104 task->d.iovs[0].iov_base = buf; 1105 task->d.iovs[0].iov_len = len; 1106 task->d.iovcnt = 1; 1107 task->nbytes = len; 1108 task->src_domain = NULL; 1109 task->dst_domain = domain; 1110 task->dst_domain_ctx = domain_ctx; 1111 task->op_code = SPDK_ACCEL_OPC_FILL; 1112 1113 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1114 *pseq = seq; 1115 1116 return 0; 1117 } 1118 1119 int 1120 spdk_accel_append_decompress(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1121 struct iovec *dst_iovs, size_t dst_iovcnt, 1122 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 1123 struct iovec *src_iovs, size_t src_iovcnt, 1124 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 1125 spdk_accel_step_cb cb_fn, void *cb_arg) 1126 { 1127 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1128 struct spdk_accel_task *task; 1129 struct spdk_accel_sequence *seq = *pseq; 1130 1131 if (seq == NULL) { 1132 seq = accel_sequence_get(accel_ch); 1133 if (spdk_unlikely(seq == NULL)) { 1134 return -ENOMEM; 1135 } 1136 } 1137 1138 assert(seq->ch == accel_ch); 1139 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1140 if (spdk_unlikely(task == NULL)) { 1141 if (*pseq == NULL) { 1142 accel_sequence_put(seq); 1143 } 1144 1145 return -ENOMEM; 1146 } 1147 1148 /* TODO: support output_size for chaining */ 1149 task->output_size = NULL; 1150 task->dst_domain = dst_domain; 1151 task->dst_domain_ctx = dst_domain_ctx; 1152 task->d.iovs = dst_iovs; 1153 task->d.iovcnt = dst_iovcnt; 1154 task->src_domain = src_domain; 1155 task->src_domain_ctx = src_domain_ctx; 1156 task->s.iovs = src_iovs; 1157 task->s.iovcnt = src_iovcnt; 1158 task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 1159 task->op_code = SPDK_ACCEL_OPC_DECOMPRESS; 1160 1161 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1162 *pseq = seq; 1163 1164 return 0; 1165 } 1166 1167 int 1168 spdk_accel_append_encrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1169 struct spdk_accel_crypto_key *key, 1170 struct iovec *dst_iovs, uint32_t dst_iovcnt, 1171 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 1172 struct iovec *src_iovs, uint32_t src_iovcnt, 1173 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 1174 uint64_t iv, uint32_t block_size, 1175 spdk_accel_step_cb cb_fn, void *cb_arg) 1176 { 1177 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1178 struct spdk_accel_task *task; 1179 struct spdk_accel_sequence *seq = *pseq; 1180 1181 assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size); 1182 1183 if (seq == NULL) { 1184 seq = accel_sequence_get(accel_ch); 1185 if (spdk_unlikely(seq == NULL)) { 1186 return -ENOMEM; 1187 } 1188 } 1189 1190 assert(seq->ch == accel_ch); 1191 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1192 if (spdk_unlikely(task == NULL)) { 1193 if (*pseq == NULL) { 1194 accel_sequence_put(seq); 1195 } 1196 1197 return -ENOMEM; 1198 } 1199 1200 task->crypto_key = key; 1201 task->src_domain = src_domain; 1202 task->src_domain_ctx = src_domain_ctx; 1203 task->s.iovs = src_iovs; 1204 task->s.iovcnt = src_iovcnt; 1205 task->dst_domain = dst_domain; 1206 task->dst_domain_ctx = dst_domain_ctx; 1207 task->d.iovs = dst_iovs; 1208 task->d.iovcnt = dst_iovcnt; 1209 task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 1210 task->iv = iv; 1211 task->block_size = block_size; 1212 task->op_code = SPDK_ACCEL_OPC_ENCRYPT; 1213 1214 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1215 *pseq = seq; 1216 1217 return 0; 1218 } 1219 1220 int 1221 spdk_accel_append_decrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1222 struct spdk_accel_crypto_key *key, 1223 struct iovec *dst_iovs, uint32_t dst_iovcnt, 1224 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 1225 struct iovec *src_iovs, uint32_t src_iovcnt, 1226 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 1227 uint64_t iv, uint32_t block_size, 1228 spdk_accel_step_cb cb_fn, void *cb_arg) 1229 { 1230 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1231 struct spdk_accel_task *task; 1232 struct spdk_accel_sequence *seq = *pseq; 1233 1234 assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size); 1235 1236 if (seq == NULL) { 1237 seq = accel_sequence_get(accel_ch); 1238 if (spdk_unlikely(seq == NULL)) { 1239 return -ENOMEM; 1240 } 1241 } 1242 1243 assert(seq->ch == accel_ch); 1244 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1245 if (spdk_unlikely(task == NULL)) { 1246 if (*pseq == NULL) { 1247 accel_sequence_put(seq); 1248 } 1249 1250 return -ENOMEM; 1251 } 1252 1253 task->crypto_key = key; 1254 task->src_domain = src_domain; 1255 task->src_domain_ctx = src_domain_ctx; 1256 task->s.iovs = src_iovs; 1257 task->s.iovcnt = src_iovcnt; 1258 task->dst_domain = dst_domain; 1259 task->dst_domain_ctx = dst_domain_ctx; 1260 task->d.iovs = dst_iovs; 1261 task->d.iovcnt = dst_iovcnt; 1262 task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 1263 task->iv = iv; 1264 task->block_size = block_size; 1265 task->op_code = SPDK_ACCEL_OPC_DECRYPT; 1266 1267 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1268 *pseq = seq; 1269 1270 return 0; 1271 } 1272 1273 int 1274 spdk_accel_append_crc32c(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1275 uint32_t *dst, struct iovec *iovs, uint32_t iovcnt, 1276 struct spdk_memory_domain *domain, void *domain_ctx, 1277 uint32_t seed, spdk_accel_step_cb cb_fn, void *cb_arg) 1278 { 1279 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1280 struct spdk_accel_task *task; 1281 struct spdk_accel_sequence *seq = *pseq; 1282 1283 if (seq == NULL) { 1284 seq = accel_sequence_get(accel_ch); 1285 if (spdk_unlikely(seq == NULL)) { 1286 return -ENOMEM; 1287 } 1288 } 1289 1290 assert(seq->ch == accel_ch); 1291 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1292 if (spdk_unlikely(task == NULL)) { 1293 if (*pseq == NULL) { 1294 accel_sequence_put(seq); 1295 } 1296 1297 return -ENOMEM; 1298 } 1299 1300 task->s.iovs = iovs; 1301 task->s.iovcnt = iovcnt; 1302 task->src_domain = domain; 1303 task->src_domain_ctx = domain_ctx; 1304 task->nbytes = accel_get_iovlen(iovs, iovcnt); 1305 task->crc_dst = dst; 1306 task->seed = seed; 1307 task->op_code = SPDK_ACCEL_OPC_CRC32C; 1308 task->dst_domain = NULL; 1309 1310 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1311 *pseq = seq; 1312 1313 return 0; 1314 } 1315 1316 int 1317 spdk_accel_get_buf(struct spdk_io_channel *ch, uint64_t len, void **buf, 1318 struct spdk_memory_domain **domain, void **domain_ctx) 1319 { 1320 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1321 struct accel_buffer *accel_buf; 1322 1323 accel_buf = accel_get_buf(accel_ch, len); 1324 if (spdk_unlikely(accel_buf == NULL)) { 1325 return -ENOMEM; 1326 } 1327 1328 accel_buf->ch = accel_ch; 1329 1330 /* We always return the same pointer and identify the buffers through domain_ctx */ 1331 *buf = ACCEL_BUFFER_BASE; 1332 *domain_ctx = accel_buf; 1333 *domain = g_accel_domain; 1334 1335 return 0; 1336 } 1337 1338 void 1339 spdk_accel_put_buf(struct spdk_io_channel *ch, void *buf, 1340 struct spdk_memory_domain *domain, void *domain_ctx) 1341 { 1342 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1343 struct accel_buffer *accel_buf = domain_ctx; 1344 1345 assert(domain == g_accel_domain); 1346 assert(buf == ACCEL_BUFFER_BASE); 1347 1348 accel_put_buf(accel_ch, accel_buf); 1349 } 1350 1351 static void 1352 accel_sequence_complete_task(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1353 { 1354 struct accel_io_channel *ch = seq->ch; 1355 spdk_accel_step_cb cb_fn; 1356 void *cb_arg; 1357 1358 TAILQ_REMOVE(&seq->tasks, task, seq_link); 1359 cb_fn = task->step_cb_fn; 1360 cb_arg = task->cb_arg; 1361 task->seq = NULL; 1362 if (task->has_aux) { 1363 SLIST_INSERT_HEAD(&ch->task_aux_data_pool, task->aux, link); 1364 task->aux = NULL; 1365 task->has_aux = false; 1366 } 1367 STAILQ_INSERT_HEAD(&ch->task_pool, task, link); 1368 if (cb_fn != NULL) { 1369 cb_fn(cb_arg); 1370 } 1371 } 1372 1373 static void 1374 accel_sequence_complete_tasks(struct spdk_accel_sequence *seq) 1375 { 1376 struct spdk_accel_task *task; 1377 1378 while (!TAILQ_EMPTY(&seq->tasks)) { 1379 task = TAILQ_FIRST(&seq->tasks); 1380 accel_sequence_complete_task(seq, task); 1381 } 1382 } 1383 1384 static void 1385 accel_sequence_complete(struct spdk_accel_sequence *seq) 1386 { 1387 SPDK_DEBUGLOG(accel, "Completed sequence: %p with status: %d\n", seq, seq->status); 1388 1389 accel_update_stats(seq->ch, sequence_executed, 1); 1390 if (spdk_unlikely(seq->status != 0)) { 1391 accel_update_stats(seq->ch, sequence_failed, 1); 1392 } 1393 1394 /* First notify all users that appended operations to this sequence */ 1395 accel_sequence_complete_tasks(seq); 1396 1397 /* Then notify the user that finished the sequence */ 1398 seq->cb_fn(seq->cb_arg, seq->status); 1399 1400 accel_sequence_put(seq); 1401 } 1402 1403 static void 1404 accel_update_virt_iov(struct iovec *diov, struct iovec *siov, struct accel_buffer *accel_buf) 1405 { 1406 uintptr_t offset; 1407 1408 offset = (uintptr_t)siov->iov_base & ACCEL_BUFFER_OFFSET_MASK; 1409 assert(offset < accel_buf->len); 1410 1411 diov->iov_base = (char *)accel_buf->buf + offset; 1412 diov->iov_len = siov->iov_len; 1413 } 1414 1415 static void 1416 accel_sequence_set_virtbuf(struct spdk_accel_sequence *seq, struct accel_buffer *buf) 1417 { 1418 struct spdk_accel_task *task; 1419 struct iovec *iov; 1420 1421 /* Now that we've allocated the actual data buffer for this accel_buffer, update all tasks 1422 * in a sequence that were using it. 1423 */ 1424 TAILQ_FOREACH(task, &seq->tasks, seq_link) { 1425 if (task->src_domain == g_accel_domain && task->src_domain_ctx == buf) { 1426 if (!task->has_aux) { 1427 task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool); 1428 assert(task->aux && "Can't allocate aux data structure"); 1429 task->has_aux = true; 1430 SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link); 1431 } 1432 1433 iov = &task->aux->iovs[SPDK_ACCEL_AXU_IOV_VIRT_SRC]; 1434 assert(task->s.iovcnt == 1); 1435 accel_update_virt_iov(iov, &task->s.iovs[0], buf); 1436 task->src_domain = NULL; 1437 task->s.iovs = iov; 1438 } 1439 if (task->dst_domain == g_accel_domain && task->dst_domain_ctx == buf) { 1440 if (!task->has_aux) { 1441 task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool); 1442 assert(task->aux && "Can't allocate aux data structure"); 1443 task->has_aux = true; 1444 SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link); 1445 } 1446 1447 iov = &task->aux->iovs[SPDK_ACCEL_AXU_IOV_VIRT_DST]; 1448 assert(task->d.iovcnt == 1); 1449 accel_update_virt_iov(iov, &task->d.iovs[0], buf); 1450 task->dst_domain = NULL; 1451 task->d.iovs = iov; 1452 } 1453 } 1454 } 1455 1456 static void accel_process_sequence(struct spdk_accel_sequence *seq); 1457 1458 static void 1459 accel_iobuf_get_virtbuf_cb(struct spdk_iobuf_entry *entry, void *buf) 1460 { 1461 struct accel_buffer *accel_buf; 1462 1463 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1464 1465 assert(accel_buf->seq != NULL); 1466 assert(accel_buf->buf == NULL); 1467 accel_buf->buf = buf; 1468 1469 assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF); 1470 accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF); 1471 accel_sequence_set_virtbuf(accel_buf->seq, accel_buf); 1472 accel_process_sequence(accel_buf->seq); 1473 } 1474 1475 static bool 1476 accel_sequence_alloc_buf(struct spdk_accel_sequence *seq, struct accel_buffer *buf, 1477 spdk_iobuf_get_cb cb_fn) 1478 { 1479 struct accel_io_channel *ch = seq->ch; 1480 1481 assert(buf->seq == NULL); 1482 1483 buf->seq = seq; 1484 1485 /* Buffer might be already allocated by memory domain translation. */ 1486 if (buf->buf) { 1487 return true; 1488 } 1489 1490 buf->buf = spdk_iobuf_get(&ch->iobuf, buf->len, &buf->iobuf, cb_fn); 1491 if (spdk_unlikely(buf->buf == NULL)) { 1492 accel_update_stats(ch, retry.iobuf, 1); 1493 return false; 1494 } 1495 1496 return true; 1497 } 1498 1499 static bool 1500 accel_sequence_check_virtbuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1501 { 1502 /* If a task doesn't have dst/src (e.g. fill, crc32), its dst/src domain should be set to 1503 * NULL */ 1504 if (task->src_domain == g_accel_domain) { 1505 if (!accel_sequence_alloc_buf(seq, task->src_domain_ctx, 1506 accel_iobuf_get_virtbuf_cb)) { 1507 return false; 1508 } 1509 1510 accel_sequence_set_virtbuf(seq, task->src_domain_ctx); 1511 } 1512 1513 if (task->dst_domain == g_accel_domain) { 1514 if (!accel_sequence_alloc_buf(seq, task->dst_domain_ctx, 1515 accel_iobuf_get_virtbuf_cb)) { 1516 return false; 1517 } 1518 1519 accel_sequence_set_virtbuf(seq, task->dst_domain_ctx); 1520 } 1521 1522 return true; 1523 } 1524 1525 static void 1526 accel_sequence_get_buf_cb(struct spdk_iobuf_entry *entry, void *buf) 1527 { 1528 struct accel_buffer *accel_buf; 1529 1530 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1531 1532 assert(accel_buf->seq != NULL); 1533 assert(accel_buf->buf == NULL); 1534 accel_buf->buf = buf; 1535 1536 accel_sequence_set_virtbuf(accel_buf->seq, accel_buf); 1537 accel_buf->cb_fn(accel_buf->seq, accel_buf->cb_ctx); 1538 } 1539 1540 bool 1541 spdk_accel_alloc_sequence_buf(struct spdk_accel_sequence *seq, void *buf, 1542 struct spdk_memory_domain *domain, void *domain_ctx, 1543 spdk_accel_sequence_get_buf_cb cb_fn, void *cb_ctx) 1544 { 1545 struct accel_buffer *accel_buf = domain_ctx; 1546 1547 assert(domain == g_accel_domain); 1548 accel_buf->cb_fn = cb_fn; 1549 accel_buf->cb_ctx = cb_ctx; 1550 1551 if (!accel_sequence_alloc_buf(seq, accel_buf, accel_sequence_get_buf_cb)) { 1552 return false; 1553 } 1554 1555 accel_sequence_set_virtbuf(seq, accel_buf); 1556 1557 return true; 1558 } 1559 1560 struct spdk_accel_task * 1561 spdk_accel_sequence_first_task(struct spdk_accel_sequence *seq) 1562 { 1563 return TAILQ_FIRST(&seq->tasks); 1564 } 1565 1566 struct spdk_accel_task * 1567 spdk_accel_sequence_next_task(struct spdk_accel_task *task) 1568 { 1569 return TAILQ_NEXT(task, seq_link); 1570 } 1571 1572 static inline void 1573 accel_set_bounce_buffer(struct spdk_accel_bounce_buffer *bounce, struct iovec **iovs, 1574 uint32_t *iovcnt, struct spdk_memory_domain **domain, void **domain_ctx, 1575 struct accel_buffer *buf) 1576 { 1577 bounce->orig_iovs = *iovs; 1578 bounce->orig_iovcnt = *iovcnt; 1579 bounce->orig_domain = *domain; 1580 bounce->orig_domain_ctx = *domain_ctx; 1581 bounce->iov.iov_base = buf->buf; 1582 bounce->iov.iov_len = buf->len; 1583 1584 *iovs = &bounce->iov; 1585 *iovcnt = 1; 1586 *domain = NULL; 1587 } 1588 1589 static void 1590 accel_iobuf_get_src_bounce_cb(struct spdk_iobuf_entry *entry, void *buf) 1591 { 1592 struct spdk_accel_task *task; 1593 struct accel_buffer *accel_buf; 1594 1595 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1596 assert(accel_buf->buf == NULL); 1597 accel_buf->buf = buf; 1598 1599 task = TAILQ_FIRST(&accel_buf->seq->tasks); 1600 assert(task != NULL); 1601 1602 assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF); 1603 accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF); 1604 assert(task->aux); 1605 assert(task->has_aux); 1606 accel_set_bounce_buffer(&task->aux->bounce.s, &task->s.iovs, &task->s.iovcnt, &task->src_domain, 1607 &task->src_domain_ctx, accel_buf); 1608 accel_process_sequence(accel_buf->seq); 1609 } 1610 1611 static void 1612 accel_iobuf_get_dst_bounce_cb(struct spdk_iobuf_entry *entry, void *buf) 1613 { 1614 struct spdk_accel_task *task; 1615 struct accel_buffer *accel_buf; 1616 1617 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1618 assert(accel_buf->buf == NULL); 1619 accel_buf->buf = buf; 1620 1621 task = TAILQ_FIRST(&accel_buf->seq->tasks); 1622 assert(task != NULL); 1623 1624 assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF); 1625 accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF); 1626 assert(task->aux); 1627 assert(task->has_aux); 1628 accel_set_bounce_buffer(&task->aux->bounce.d, &task->d.iovs, &task->d.iovcnt, &task->dst_domain, 1629 &task->dst_domain_ctx, accel_buf); 1630 accel_process_sequence(accel_buf->seq); 1631 } 1632 1633 static int 1634 accel_sequence_check_bouncebuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1635 { 1636 struct accel_buffer *buf; 1637 1638 if (task->src_domain != NULL) { 1639 /* By the time we're here, accel buffers should have been allocated */ 1640 assert(task->src_domain != g_accel_domain); 1641 1642 if (!task->has_aux) { 1643 task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool); 1644 if (spdk_unlikely(!task->aux)) { 1645 SPDK_ERRLOG("Can't allocate aux data structure\n"); 1646 assert(0); 1647 return -EAGAIN; 1648 } 1649 task->has_aux = true; 1650 SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link); 1651 } 1652 buf = accel_get_buf(seq->ch, accel_get_iovlen(task->s.iovs, task->s.iovcnt)); 1653 if (buf == NULL) { 1654 SPDK_ERRLOG("Couldn't allocate buffer descriptor\n"); 1655 return -ENOMEM; 1656 } 1657 1658 SLIST_INSERT_HEAD(&seq->bounce_bufs, buf, link); 1659 if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_src_bounce_cb)) { 1660 return -EAGAIN; 1661 } 1662 1663 accel_set_bounce_buffer(&task->aux->bounce.s, &task->s.iovs, &task->s.iovcnt, 1664 &task->src_domain, &task->src_domain_ctx, buf); 1665 } 1666 1667 if (task->dst_domain != NULL) { 1668 /* By the time we're here, accel buffers should have been allocated */ 1669 assert(task->dst_domain != g_accel_domain); 1670 1671 if (!task->has_aux) { 1672 task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool); 1673 if (spdk_unlikely(!task->aux)) { 1674 SPDK_ERRLOG("Can't allocate aux data structure\n"); 1675 assert(0); 1676 return -EAGAIN; 1677 } 1678 task->has_aux = true; 1679 SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link); 1680 } 1681 buf = accel_get_buf(seq->ch, accel_get_iovlen(task->d.iovs, task->d.iovcnt)); 1682 if (buf == NULL) { 1683 /* The src buffer will be released when a sequence is completed */ 1684 SPDK_ERRLOG("Couldn't allocate buffer descriptor\n"); 1685 return -ENOMEM; 1686 } 1687 1688 SLIST_INSERT_HEAD(&seq->bounce_bufs, buf, link); 1689 if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_dst_bounce_cb)) { 1690 return -EAGAIN; 1691 } 1692 1693 accel_set_bounce_buffer(&task->aux->bounce.d, &task->d.iovs, &task->d.iovcnt, 1694 &task->dst_domain, &task->dst_domain_ctx, buf); 1695 } 1696 1697 return 0; 1698 } 1699 1700 static void 1701 accel_task_pull_data_cb(void *ctx, int status) 1702 { 1703 struct spdk_accel_sequence *seq = ctx; 1704 1705 assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA); 1706 if (spdk_likely(status == 0)) { 1707 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK); 1708 } else { 1709 accel_sequence_set_fail(seq, status); 1710 } 1711 1712 accel_process_sequence(seq); 1713 } 1714 1715 static void 1716 accel_task_pull_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1717 { 1718 int rc; 1719 1720 assert(task->has_aux); 1721 assert(task->aux); 1722 assert(task->aux->bounce.s.orig_iovs != NULL); 1723 assert(task->aux->bounce.s.orig_domain != NULL); 1724 assert(task->aux->bounce.s.orig_domain != g_accel_domain); 1725 assert(!g_modules_opc[task->op_code].supports_memory_domains); 1726 1727 rc = spdk_memory_domain_pull_data(task->aux->bounce.s.orig_domain, 1728 task->aux->bounce.s.orig_domain_ctx, 1729 task->aux->bounce.s.orig_iovs, task->aux->bounce.s.orig_iovcnt, 1730 task->s.iovs, task->s.iovcnt, 1731 accel_task_pull_data_cb, seq); 1732 if (spdk_unlikely(rc != 0)) { 1733 SPDK_ERRLOG("Failed to pull data from memory domain: %s, rc: %d\n", 1734 spdk_memory_domain_get_dma_device_id(task->aux->bounce.s.orig_domain), rc); 1735 accel_sequence_set_fail(seq, rc); 1736 } 1737 } 1738 1739 static void 1740 accel_task_push_data_cb(void *ctx, int status) 1741 { 1742 struct spdk_accel_sequence *seq = ctx; 1743 1744 assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA); 1745 if (spdk_likely(status == 0)) { 1746 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK); 1747 } else { 1748 accel_sequence_set_fail(seq, status); 1749 } 1750 1751 accel_process_sequence(seq); 1752 } 1753 1754 static void 1755 accel_task_push_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1756 { 1757 int rc; 1758 1759 assert(task->has_aux); 1760 assert(task->aux); 1761 assert(task->aux->bounce.d.orig_iovs != NULL); 1762 assert(task->aux->bounce.d.orig_domain != NULL); 1763 assert(task->aux->bounce.d.orig_domain != g_accel_domain); 1764 assert(!g_modules_opc[task->op_code].supports_memory_domains); 1765 1766 rc = spdk_memory_domain_push_data(task->aux->bounce.d.orig_domain, 1767 task->aux->bounce.d.orig_domain_ctx, 1768 task->aux->bounce.d.orig_iovs, task->aux->bounce.d.orig_iovcnt, 1769 task->d.iovs, task->d.iovcnt, 1770 accel_task_push_data_cb, seq); 1771 if (spdk_unlikely(rc != 0)) { 1772 SPDK_ERRLOG("Failed to push data to memory domain: %s, rc: %d\n", 1773 spdk_memory_domain_get_dma_device_id(task->aux->bounce.s.orig_domain), rc); 1774 accel_sequence_set_fail(seq, rc); 1775 } 1776 } 1777 1778 static void 1779 accel_process_sequence(struct spdk_accel_sequence *seq) 1780 { 1781 struct accel_io_channel *accel_ch = seq->ch; 1782 struct spdk_accel_task *task; 1783 enum accel_sequence_state state; 1784 int rc; 1785 1786 /* Prevent recursive calls to this function */ 1787 if (spdk_unlikely(seq->in_process_sequence)) { 1788 return; 1789 } 1790 seq->in_process_sequence = true; 1791 1792 task = TAILQ_FIRST(&seq->tasks); 1793 do { 1794 state = seq->state; 1795 switch (state) { 1796 case ACCEL_SEQUENCE_STATE_INIT: 1797 if (g_accel_driver != NULL) { 1798 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS); 1799 break; 1800 } 1801 /* Fall through */ 1802 case ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF: 1803 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF); 1804 if (!accel_sequence_check_virtbuf(seq, task)) { 1805 /* We couldn't allocate a buffer, wait until one is available */ 1806 break; 1807 } 1808 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF); 1809 /* Fall through */ 1810 case ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF: 1811 /* If a module supports memory domains, we don't need to allocate bounce 1812 * buffers */ 1813 if (g_modules_opc[task->op_code].supports_memory_domains) { 1814 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK); 1815 break; 1816 } 1817 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF); 1818 rc = accel_sequence_check_bouncebuf(seq, task); 1819 if (spdk_unlikely(rc != 0)) { 1820 /* We couldn't allocate a buffer, wait until one is available */ 1821 if (rc == -EAGAIN) { 1822 break; 1823 } 1824 accel_sequence_set_fail(seq, rc); 1825 break; 1826 } 1827 if (task->has_aux && task->s.iovs == &task->aux->bounce.s.iov) { 1828 assert(task->aux->bounce.s.orig_iovs); 1829 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PULL_DATA); 1830 break; 1831 } 1832 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK); 1833 /* Fall through */ 1834 case ACCEL_SEQUENCE_STATE_EXEC_TASK: 1835 SPDK_DEBUGLOG(accel, "Executing %s operation, sequence: %p\n", 1836 g_opcode_strings[task->op_code], seq); 1837 1838 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_TASK); 1839 rc = accel_submit_task(accel_ch, task); 1840 if (spdk_unlikely(rc != 0)) { 1841 SPDK_ERRLOG("Failed to submit %s operation, sequence: %p\n", 1842 g_opcode_strings[task->op_code], seq); 1843 accel_sequence_set_fail(seq, rc); 1844 } 1845 break; 1846 case ACCEL_SEQUENCE_STATE_PULL_DATA: 1847 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA); 1848 accel_task_pull_data(seq, task); 1849 break; 1850 case ACCEL_SEQUENCE_STATE_COMPLETE_TASK: 1851 if (task->has_aux && task->d.iovs == &task->aux->bounce.d.iov) { 1852 assert(task->aux->bounce.d.orig_iovs); 1853 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PUSH_DATA); 1854 break; 1855 } 1856 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK); 1857 break; 1858 case ACCEL_SEQUENCE_STATE_PUSH_DATA: 1859 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA); 1860 accel_task_push_data(seq, task); 1861 break; 1862 case ACCEL_SEQUENCE_STATE_NEXT_TASK: 1863 accel_sequence_complete_task(seq, task); 1864 /* Check if there are any remaining tasks */ 1865 task = TAILQ_FIRST(&seq->tasks); 1866 if (task == NULL) { 1867 /* Immediately return here to make sure we don't touch the sequence 1868 * after it's completed */ 1869 accel_sequence_complete(seq); 1870 return; 1871 } 1872 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_INIT); 1873 break; 1874 case ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS: 1875 assert(!TAILQ_EMPTY(&seq->tasks)); 1876 1877 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS); 1878 rc = g_accel_driver->execute_sequence(accel_ch->driver_channel, seq); 1879 if (spdk_unlikely(rc != 0)) { 1880 SPDK_ERRLOG("Failed to execute sequence: %p using driver: %s\n", 1881 seq, g_accel_driver->name); 1882 accel_sequence_set_fail(seq, rc); 1883 } 1884 break; 1885 case ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS: 1886 /* Get the task again, as the driver might have completed some tasks 1887 * synchronously */ 1888 task = TAILQ_FIRST(&seq->tasks); 1889 if (task == NULL) { 1890 /* Immediately return here to make sure we don't touch the sequence 1891 * after it's completed */ 1892 accel_sequence_complete(seq); 1893 return; 1894 } 1895 /* We don't want to execute the next task through the driver, so we 1896 * explicitly omit the INIT state here */ 1897 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF); 1898 break; 1899 case ACCEL_SEQUENCE_STATE_ERROR: 1900 /* Immediately return here to make sure we don't touch the sequence 1901 * after it's completed */ 1902 assert(seq->status != 0); 1903 accel_sequence_complete(seq); 1904 return; 1905 case ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF: 1906 case ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF: 1907 case ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA: 1908 case ACCEL_SEQUENCE_STATE_AWAIT_TASK: 1909 case ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA: 1910 case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS: 1911 break; 1912 default: 1913 assert(0 && "bad state"); 1914 break; 1915 } 1916 } while (seq->state != state); 1917 1918 seq->in_process_sequence = false; 1919 } 1920 1921 static void 1922 accel_sequence_task_cb(void *cb_arg, int status) 1923 { 1924 struct spdk_accel_sequence *seq = cb_arg; 1925 struct spdk_accel_task *task = TAILQ_FIRST(&seq->tasks); 1926 1927 switch (seq->state) { 1928 case ACCEL_SEQUENCE_STATE_AWAIT_TASK: 1929 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_COMPLETE_TASK); 1930 if (spdk_unlikely(status != 0)) { 1931 SPDK_ERRLOG("Failed to execute %s operation, sequence: %p\n", 1932 g_opcode_strings[task->op_code], seq); 1933 accel_sequence_set_fail(seq, status); 1934 } 1935 1936 accel_process_sequence(seq); 1937 break; 1938 case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS: 1939 assert(g_accel_driver != NULL); 1940 /* Immediately remove the task from the outstanding list to make sure the next call 1941 * to spdk_accel_sequence_first_task() doesn't return it */ 1942 accel_sequence_complete_task(seq, task); 1943 if (spdk_unlikely(status != 0)) { 1944 SPDK_ERRLOG("Failed to execute %s operation, sequence: %p through " 1945 "driver: %s\n", g_opcode_strings[task->op_code], seq, 1946 g_accel_driver->name); 1947 /* Update status without using accel_sequence_set_fail() to avoid changing 1948 * seq's state to ERROR until driver calls spdk_accel_sequence_continue() */ 1949 seq->status = status; 1950 } 1951 break; 1952 default: 1953 assert(0 && "bad state"); 1954 break; 1955 } 1956 } 1957 1958 void 1959 spdk_accel_sequence_continue(struct spdk_accel_sequence *seq) 1960 { 1961 assert(g_accel_driver != NULL); 1962 assert(seq->state == ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS); 1963 1964 if (spdk_likely(seq->status == 0)) { 1965 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS); 1966 } else { 1967 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR); 1968 } 1969 1970 accel_process_sequence(seq); 1971 } 1972 1973 static bool 1974 accel_compare_iovs(struct iovec *iova, uint32_t iovacnt, struct iovec *iovb, uint32_t iovbcnt) 1975 { 1976 /* For now, just do a dumb check that the iovecs arrays are exactly the same */ 1977 if (iovacnt != iovbcnt) { 1978 return false; 1979 } 1980 1981 return memcmp(iova, iovb, sizeof(*iova) * iovacnt) == 0; 1982 } 1983 1984 static bool 1985 accel_task_set_dstbuf(struct spdk_accel_task *task, struct spdk_accel_task *next) 1986 { 1987 struct spdk_accel_task *prev; 1988 1989 switch (task->op_code) { 1990 case SPDK_ACCEL_OPC_DECOMPRESS: 1991 case SPDK_ACCEL_OPC_FILL: 1992 case SPDK_ACCEL_OPC_ENCRYPT: 1993 case SPDK_ACCEL_OPC_DECRYPT: 1994 if (task->dst_domain != next->src_domain) { 1995 return false; 1996 } 1997 if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt, 1998 next->s.iovs, next->s.iovcnt)) { 1999 return false; 2000 } 2001 task->d.iovs = next->d.iovs; 2002 task->d.iovcnt = next->d.iovcnt; 2003 task->dst_domain = next->dst_domain; 2004 task->dst_domain_ctx = next->dst_domain_ctx; 2005 break; 2006 case SPDK_ACCEL_OPC_CRC32C: 2007 /* crc32 is special, because it doesn't have a dst buffer */ 2008 if (task->src_domain != next->src_domain) { 2009 return false; 2010 } 2011 if (!accel_compare_iovs(task->s.iovs, task->s.iovcnt, 2012 next->s.iovs, next->s.iovcnt)) { 2013 return false; 2014 } 2015 /* We can only change crc32's buffer if we can change previous task's buffer */ 2016 prev = TAILQ_PREV(task, accel_sequence_tasks, seq_link); 2017 if (prev == NULL) { 2018 return false; 2019 } 2020 if (!accel_task_set_dstbuf(prev, next)) { 2021 return false; 2022 } 2023 task->s.iovs = next->d.iovs; 2024 task->s.iovcnt = next->d.iovcnt; 2025 task->src_domain = next->dst_domain; 2026 task->src_domain_ctx = next->dst_domain_ctx; 2027 break; 2028 default: 2029 return false; 2030 } 2031 2032 return true; 2033 } 2034 2035 static void 2036 accel_sequence_merge_tasks(struct spdk_accel_sequence *seq, struct spdk_accel_task *task, 2037 struct spdk_accel_task **next_task) 2038 { 2039 struct spdk_accel_task *next = *next_task; 2040 2041 switch (task->op_code) { 2042 case SPDK_ACCEL_OPC_COPY: 2043 /* We only allow changing src of operations that actually have a src, e.g. we never 2044 * do it for fill. Theoretically, it is possible, but we'd have to be careful to 2045 * change the src of the operation after fill (which in turn could also be a fill). 2046 * So, for the sake of simplicity, skip this type of operations for now. 2047 */ 2048 if (next->op_code != SPDK_ACCEL_OPC_DECOMPRESS && 2049 next->op_code != SPDK_ACCEL_OPC_COPY && 2050 next->op_code != SPDK_ACCEL_OPC_ENCRYPT && 2051 next->op_code != SPDK_ACCEL_OPC_DECRYPT && 2052 next->op_code != SPDK_ACCEL_OPC_COPY_CRC32C) { 2053 break; 2054 } 2055 if (task->dst_domain != next->src_domain) { 2056 break; 2057 } 2058 if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt, 2059 next->s.iovs, next->s.iovcnt)) { 2060 break; 2061 } 2062 next->s.iovs = task->s.iovs; 2063 next->s.iovcnt = task->s.iovcnt; 2064 next->src_domain = task->src_domain; 2065 next->src_domain_ctx = task->src_domain_ctx; 2066 accel_sequence_complete_task(seq, task); 2067 break; 2068 case SPDK_ACCEL_OPC_DECOMPRESS: 2069 case SPDK_ACCEL_OPC_FILL: 2070 case SPDK_ACCEL_OPC_ENCRYPT: 2071 case SPDK_ACCEL_OPC_DECRYPT: 2072 case SPDK_ACCEL_OPC_CRC32C: 2073 /* We can only merge tasks when one of them is a copy */ 2074 if (next->op_code != SPDK_ACCEL_OPC_COPY) { 2075 break; 2076 } 2077 if (!accel_task_set_dstbuf(task, next)) { 2078 break; 2079 } 2080 /* We're removing next_task from the tasks queue, so we need to update its pointer, 2081 * so that the TAILQ_FOREACH_SAFE() loop below works correctly */ 2082 *next_task = TAILQ_NEXT(next, seq_link); 2083 accel_sequence_complete_task(seq, next); 2084 break; 2085 default: 2086 assert(0 && "bad opcode"); 2087 break; 2088 } 2089 } 2090 2091 void 2092 spdk_accel_sequence_finish(struct spdk_accel_sequence *seq, 2093 spdk_accel_completion_cb cb_fn, void *cb_arg) 2094 { 2095 struct spdk_accel_task *task, *next; 2096 2097 /* Try to remove any copy operations if possible */ 2098 TAILQ_FOREACH_SAFE(task, &seq->tasks, seq_link, next) { 2099 if (next == NULL) { 2100 break; 2101 } 2102 accel_sequence_merge_tasks(seq, task, &next); 2103 } 2104 2105 seq->cb_fn = cb_fn; 2106 seq->cb_arg = cb_arg; 2107 2108 accel_process_sequence(seq); 2109 } 2110 2111 void 2112 spdk_accel_sequence_reverse(struct spdk_accel_sequence *seq) 2113 { 2114 struct accel_sequence_tasks tasks = TAILQ_HEAD_INITIALIZER(tasks); 2115 struct spdk_accel_task *task; 2116 2117 TAILQ_SWAP(&tasks, &seq->tasks, spdk_accel_task, seq_link); 2118 2119 while (!TAILQ_EMPTY(&tasks)) { 2120 task = TAILQ_FIRST(&tasks); 2121 TAILQ_REMOVE(&tasks, task, seq_link); 2122 TAILQ_INSERT_HEAD(&seq->tasks, task, seq_link); 2123 } 2124 } 2125 2126 void 2127 spdk_accel_sequence_abort(struct spdk_accel_sequence *seq) 2128 { 2129 if (seq == NULL) { 2130 return; 2131 } 2132 2133 accel_sequence_complete_tasks(seq); 2134 accel_sequence_put(seq); 2135 } 2136 2137 struct spdk_memory_domain * 2138 spdk_accel_get_memory_domain(void) 2139 { 2140 return g_accel_domain; 2141 } 2142 2143 static struct spdk_accel_module_if * 2144 _module_find_by_name(const char *name) 2145 { 2146 struct spdk_accel_module_if *accel_module = NULL; 2147 2148 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 2149 if (strcmp(name, accel_module->name) == 0) { 2150 break; 2151 } 2152 } 2153 2154 return accel_module; 2155 } 2156 2157 static inline struct spdk_accel_crypto_key * 2158 _accel_crypto_key_get(const char *name) 2159 { 2160 struct spdk_accel_crypto_key *key; 2161 2162 assert(spdk_spin_held(&g_keyring_spin)); 2163 2164 TAILQ_FOREACH(key, &g_keyring, link) { 2165 if (strcmp(name, key->param.key_name) == 0) { 2166 return key; 2167 } 2168 } 2169 2170 return NULL; 2171 } 2172 2173 static void 2174 accel_crypto_key_free_mem(struct spdk_accel_crypto_key *key) 2175 { 2176 if (key->param.hex_key) { 2177 spdk_memset_s(key->param.hex_key, key->key_size * 2, 0, key->key_size * 2); 2178 free(key->param.hex_key); 2179 } 2180 if (key->param.hex_key2) { 2181 spdk_memset_s(key->param.hex_key2, key->key2_size * 2, 0, key->key2_size * 2); 2182 free(key->param.hex_key2); 2183 } 2184 free(key->param.tweak_mode); 2185 free(key->param.key_name); 2186 free(key->param.cipher); 2187 if (key->key) { 2188 spdk_memset_s(key->key, key->key_size, 0, key->key_size); 2189 free(key->key); 2190 } 2191 if (key->key2) { 2192 spdk_memset_s(key->key2, key->key2_size, 0, key->key2_size); 2193 free(key->key2); 2194 } 2195 free(key); 2196 } 2197 2198 static void 2199 accel_crypto_key_destroy_unsafe(struct spdk_accel_crypto_key *key) 2200 { 2201 assert(key->module_if); 2202 assert(key->module_if->crypto_key_deinit); 2203 2204 key->module_if->crypto_key_deinit(key); 2205 accel_crypto_key_free_mem(key); 2206 } 2207 2208 /* 2209 * This function mitigates a timing side channel which could be caused by using strcmp() 2210 * Please refer to chapter "Mitigating Information Leakage Based on Variable Timing" in 2211 * the article [1] for more details 2212 * [1] https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/secure-coding/mitigate-timing-side-channel-crypto-implementation.html 2213 */ 2214 static bool 2215 accel_aes_xts_keys_equal(const char *k1, size_t k1_len, const char *k2, size_t k2_len) 2216 { 2217 size_t i; 2218 volatile size_t x = k1_len ^ k2_len; 2219 2220 for (i = 0; ((i < k1_len) & (i < k2_len)); i++) { 2221 x |= k1[i] ^ k2[i]; 2222 } 2223 2224 return x == 0; 2225 } 2226 2227 static const char *g_tweak_modes[] = { 2228 [SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA] = "SIMPLE_LBA", 2229 [SPDK_ACCEL_CRYPTO_TWEAK_MODE_JOIN_NEG_LBA_WITH_LBA] = "JOIN_NEG_LBA_WITH_LBA", 2230 [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_FULL_LBA] = "INCR_512_FULL_LBA", 2231 [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_UPPER_LBA] = "INCR_512_UPPER_LBA", 2232 }; 2233 2234 static const char *g_ciphers[] = { 2235 [SPDK_ACCEL_CIPHER_AES_CBC] = "AES_CBC", 2236 [SPDK_ACCEL_CIPHER_AES_XTS] = "AES_XTS", 2237 }; 2238 2239 int 2240 spdk_accel_crypto_key_create(const struct spdk_accel_crypto_key_create_param *param) 2241 { 2242 struct spdk_accel_module_if *module; 2243 struct spdk_accel_crypto_key *key; 2244 size_t hex_key_size, hex_key2_size; 2245 bool found = false; 2246 size_t i; 2247 int rc; 2248 2249 if (!param || !param->hex_key || !param->cipher || !param->key_name) { 2250 return -EINVAL; 2251 } 2252 2253 if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) { 2254 /* hardly ever possible, but let's check and warn the user */ 2255 SPDK_ERRLOG("Different accel modules are used for encryption and decryption\n"); 2256 } 2257 module = g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module; 2258 2259 if (!module) { 2260 SPDK_ERRLOG("No accel module found assigned for crypto operation\n"); 2261 return -ENOENT; 2262 } 2263 2264 if (!module->crypto_key_init || !module->crypto_supports_cipher) { 2265 SPDK_ERRLOG("Module %s doesn't support crypto operations\n", module->name); 2266 return -ENOTSUP; 2267 } 2268 2269 key = calloc(1, sizeof(*key)); 2270 if (!key) { 2271 return -ENOMEM; 2272 } 2273 2274 key->param.key_name = strdup(param->key_name); 2275 if (!key->param.key_name) { 2276 rc = -ENOMEM; 2277 goto error; 2278 } 2279 2280 for (i = 0; i < SPDK_COUNTOF(g_ciphers); ++i) { 2281 assert(g_ciphers[i]); 2282 2283 if (strncmp(param->cipher, g_ciphers[i], strlen(g_ciphers[i])) == 0) { 2284 key->cipher = i; 2285 found = true; 2286 break; 2287 } 2288 } 2289 2290 if (!found) { 2291 SPDK_ERRLOG("Failed to parse cipher\n"); 2292 rc = -EINVAL; 2293 goto error; 2294 } 2295 2296 key->param.cipher = strdup(param->cipher); 2297 if (!key->param.cipher) { 2298 rc = -ENOMEM; 2299 goto error; 2300 } 2301 2302 hex_key_size = strnlen(param->hex_key, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 2303 if (hex_key_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) { 2304 SPDK_ERRLOG("key1 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 2305 rc = -EINVAL; 2306 goto error; 2307 } 2308 2309 if (hex_key_size == 0) { 2310 SPDK_ERRLOG("key1 size cannot be 0\n"); 2311 rc = -EINVAL; 2312 goto error; 2313 } 2314 2315 key->param.hex_key = strdup(param->hex_key); 2316 if (!key->param.hex_key) { 2317 rc = -ENOMEM; 2318 goto error; 2319 } 2320 2321 key->key_size = hex_key_size / 2; 2322 key->key = spdk_unhexlify(key->param.hex_key); 2323 if (!key->key) { 2324 SPDK_ERRLOG("Failed to unhexlify key1\n"); 2325 rc = -EINVAL; 2326 goto error; 2327 } 2328 2329 if (param->hex_key2) { 2330 hex_key2_size = strnlen(param->hex_key2, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 2331 if (hex_key2_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) { 2332 SPDK_ERRLOG("key2 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 2333 rc = -EINVAL; 2334 goto error; 2335 } 2336 2337 if (hex_key2_size == 0) { 2338 SPDK_ERRLOG("key2 size cannot be 0\n"); 2339 rc = -EINVAL; 2340 goto error; 2341 } 2342 2343 key->param.hex_key2 = strdup(param->hex_key2); 2344 if (!key->param.hex_key2) { 2345 rc = -ENOMEM; 2346 goto error; 2347 } 2348 2349 key->key2_size = hex_key2_size / 2; 2350 key->key2 = spdk_unhexlify(key->param.hex_key2); 2351 if (!key->key2) { 2352 SPDK_ERRLOG("Failed to unhexlify key2\n"); 2353 rc = -EINVAL; 2354 goto error; 2355 } 2356 } 2357 2358 key->tweak_mode = ACCEL_CRYPTO_TWEAK_MODE_DEFAULT; 2359 if (param->tweak_mode) { 2360 found = false; 2361 2362 key->param.tweak_mode = strdup(param->tweak_mode); 2363 if (!key->param.tweak_mode) { 2364 rc = -ENOMEM; 2365 goto error; 2366 } 2367 2368 for (i = 0; i < SPDK_COUNTOF(g_tweak_modes); ++i) { 2369 assert(g_tweak_modes[i]); 2370 2371 if (strncmp(param->tweak_mode, g_tweak_modes[i], strlen(g_tweak_modes[i])) == 0) { 2372 key->tweak_mode = i; 2373 found = true; 2374 break; 2375 } 2376 } 2377 2378 if (!found) { 2379 SPDK_ERRLOG("Failed to parse tweak mode\n"); 2380 rc = -EINVAL; 2381 goto error; 2382 } 2383 } 2384 2385 if ((!module->crypto_supports_tweak_mode && key->tweak_mode != ACCEL_CRYPTO_TWEAK_MODE_DEFAULT) || 2386 (module->crypto_supports_tweak_mode && !module->crypto_supports_tweak_mode(key->tweak_mode))) { 2387 SPDK_ERRLOG("Module %s doesn't support %s tweak mode\n", module->name, 2388 g_tweak_modes[key->tweak_mode]); 2389 rc = -EINVAL; 2390 goto error; 2391 } 2392 2393 if (!module->crypto_supports_cipher(key->cipher, key->key_size)) { 2394 SPDK_ERRLOG("Module %s doesn't support %s cipher with %zu key size\n", module->name, 2395 g_ciphers[key->cipher], key->key_size); 2396 rc = -EINVAL; 2397 goto error; 2398 } 2399 2400 if (key->cipher == SPDK_ACCEL_CIPHER_AES_XTS) { 2401 if (!key->key2) { 2402 SPDK_ERRLOG("%s key2 is missing\n", g_ciphers[key->cipher]); 2403 rc = -EINVAL; 2404 goto error; 2405 } 2406 2407 if (key->key_size != key->key2_size) { 2408 SPDK_ERRLOG("%s key size %zu is not equal to key2 size %zu\n", g_ciphers[key->cipher], 2409 key->key_size, 2410 key->key2_size); 2411 rc = -EINVAL; 2412 goto error; 2413 } 2414 2415 if (accel_aes_xts_keys_equal(key->key, key->key_size, key->key2, key->key2_size)) { 2416 SPDK_ERRLOG("%s identical keys are not secure\n", g_ciphers[key->cipher]); 2417 rc = -EINVAL; 2418 goto error; 2419 } 2420 } 2421 2422 if (key->cipher == SPDK_ACCEL_CIPHER_AES_CBC) { 2423 if (key->key2_size) { 2424 SPDK_ERRLOG("%s doesn't use key2\n", g_ciphers[key->cipher]); 2425 rc = -EINVAL; 2426 goto error; 2427 } 2428 } 2429 2430 key->module_if = module; 2431 2432 spdk_spin_lock(&g_keyring_spin); 2433 if (_accel_crypto_key_get(param->key_name)) { 2434 rc = -EEXIST; 2435 } else { 2436 rc = module->crypto_key_init(key); 2437 if (rc) { 2438 SPDK_ERRLOG("Module %s failed to initialize crypto key\n", module->name); 2439 } else { 2440 TAILQ_INSERT_TAIL(&g_keyring, key, link); 2441 } 2442 } 2443 spdk_spin_unlock(&g_keyring_spin); 2444 2445 if (rc) { 2446 goto error; 2447 } 2448 2449 return 0; 2450 2451 error: 2452 accel_crypto_key_free_mem(key); 2453 return rc; 2454 } 2455 2456 int 2457 spdk_accel_crypto_key_destroy(struct spdk_accel_crypto_key *key) 2458 { 2459 if (!key || !key->module_if) { 2460 return -EINVAL; 2461 } 2462 2463 spdk_spin_lock(&g_keyring_spin); 2464 if (!_accel_crypto_key_get(key->param.key_name)) { 2465 spdk_spin_unlock(&g_keyring_spin); 2466 return -ENOENT; 2467 } 2468 TAILQ_REMOVE(&g_keyring, key, link); 2469 spdk_spin_unlock(&g_keyring_spin); 2470 2471 accel_crypto_key_destroy_unsafe(key); 2472 2473 return 0; 2474 } 2475 2476 struct spdk_accel_crypto_key * 2477 spdk_accel_crypto_key_get(const char *name) 2478 { 2479 struct spdk_accel_crypto_key *key; 2480 2481 spdk_spin_lock(&g_keyring_spin); 2482 key = _accel_crypto_key_get(name); 2483 spdk_spin_unlock(&g_keyring_spin); 2484 2485 return key; 2486 } 2487 2488 /* Helper function when accel modules register with the framework. */ 2489 void 2490 spdk_accel_module_list_add(struct spdk_accel_module_if *accel_module) 2491 { 2492 struct spdk_accel_module_if *tmp; 2493 2494 if (_module_find_by_name(accel_module->name)) { 2495 SPDK_NOTICELOG("Module %s already registered\n", accel_module->name); 2496 assert(false); 2497 return; 2498 } 2499 2500 TAILQ_FOREACH(tmp, &spdk_accel_module_list, tailq) { 2501 if (accel_module->priority < tmp->priority) { 2502 break; 2503 } 2504 } 2505 2506 if (tmp != NULL) { 2507 TAILQ_INSERT_BEFORE(tmp, accel_module, tailq); 2508 } else { 2509 TAILQ_INSERT_TAIL(&spdk_accel_module_list, accel_module, tailq); 2510 } 2511 } 2512 2513 /* Framework level channel create callback. */ 2514 static int 2515 accel_create_channel(void *io_device, void *ctx_buf) 2516 { 2517 struct accel_io_channel *accel_ch = ctx_buf; 2518 struct spdk_accel_task *accel_task; 2519 struct spdk_accel_task_aux_data *accel_task_aux; 2520 struct spdk_accel_sequence *seq; 2521 struct accel_buffer *buf; 2522 size_t task_size_aligned; 2523 uint8_t *task_mem; 2524 uint32_t i = 0, j; 2525 int rc; 2526 2527 task_size_aligned = SPDK_ALIGN_CEIL(g_max_accel_module_size, SPDK_CACHE_LINE_SIZE); 2528 accel_ch->task_pool_base = aligned_alloc(SPDK_CACHE_LINE_SIZE, 2529 g_opts.task_count * task_size_aligned); 2530 if (!accel_ch->task_pool_base) { 2531 return -ENOMEM; 2532 } 2533 memset(accel_ch->task_pool_base, 0, g_opts.task_count * task_size_aligned); 2534 2535 accel_ch->seq_pool_base = aligned_alloc(SPDK_CACHE_LINE_SIZE, 2536 g_opts.sequence_count * sizeof(struct spdk_accel_sequence)); 2537 if (accel_ch->seq_pool_base == NULL) { 2538 goto err; 2539 } 2540 memset(accel_ch->seq_pool_base, 0, g_opts.sequence_count * sizeof(struct spdk_accel_sequence)); 2541 2542 accel_ch->task_aux_data_base = calloc(g_opts.task_count, sizeof(struct spdk_accel_task_aux_data)); 2543 if (accel_ch->task_aux_data_base == NULL) { 2544 goto err; 2545 } 2546 2547 accel_ch->buf_pool_base = calloc(g_opts.buf_count, sizeof(struct accel_buffer)); 2548 if (accel_ch->buf_pool_base == NULL) { 2549 goto err; 2550 } 2551 2552 STAILQ_INIT(&accel_ch->task_pool); 2553 SLIST_INIT(&accel_ch->task_aux_data_pool); 2554 SLIST_INIT(&accel_ch->seq_pool); 2555 SLIST_INIT(&accel_ch->buf_pool); 2556 2557 task_mem = accel_ch->task_pool_base; 2558 for (i = 0; i < g_opts.task_count; i++) { 2559 accel_task = (struct spdk_accel_task *)task_mem; 2560 accel_task->aux = NULL; 2561 STAILQ_INSERT_TAIL(&accel_ch->task_pool, accel_task, link); 2562 task_mem += task_size_aligned; 2563 accel_task_aux = &accel_ch->task_aux_data_base[i]; 2564 SLIST_INSERT_HEAD(&accel_ch->task_aux_data_pool, accel_task_aux, link); 2565 } 2566 for (i = 0; i < g_opts.sequence_count; i++) { 2567 seq = &accel_ch->seq_pool_base[i]; 2568 SLIST_INSERT_HEAD(&accel_ch->seq_pool, seq, link); 2569 } 2570 for (i = 0; i < g_opts.buf_count; i++) { 2571 buf = &accel_ch->buf_pool_base[i]; 2572 SLIST_INSERT_HEAD(&accel_ch->buf_pool, buf, link); 2573 } 2574 2575 /* Assign modules and get IO channels for each */ 2576 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 2577 accel_ch->module_ch[i] = g_modules_opc[i].module->get_io_channel(); 2578 /* This can happen if idxd runs out of channels. */ 2579 if (accel_ch->module_ch[i] == NULL) { 2580 SPDK_ERRLOG("Module %s failed to get io channel\n", g_modules_opc[i].module->name); 2581 goto err; 2582 } 2583 } 2584 2585 if (g_accel_driver != NULL) { 2586 accel_ch->driver_channel = g_accel_driver->get_io_channel(); 2587 if (accel_ch->driver_channel == NULL) { 2588 SPDK_ERRLOG("Failed to get driver's IO channel\n"); 2589 goto err; 2590 } 2591 } 2592 2593 rc = spdk_iobuf_channel_init(&accel_ch->iobuf, "accel", g_opts.small_cache_size, 2594 g_opts.large_cache_size); 2595 if (rc != 0) { 2596 SPDK_ERRLOG("Failed to initialize iobuf accel channel\n"); 2597 goto err; 2598 } 2599 2600 return 0; 2601 err: 2602 if (accel_ch->driver_channel != NULL) { 2603 spdk_put_io_channel(accel_ch->driver_channel); 2604 } 2605 for (j = 0; j < i; j++) { 2606 spdk_put_io_channel(accel_ch->module_ch[j]); 2607 } 2608 free(accel_ch->task_pool_base); 2609 free(accel_ch->task_aux_data_base); 2610 free(accel_ch->seq_pool_base); 2611 free(accel_ch->buf_pool_base); 2612 2613 return -ENOMEM; 2614 } 2615 2616 static void 2617 accel_add_stats(struct accel_stats *total, struct accel_stats *stats) 2618 { 2619 int i; 2620 2621 total->sequence_executed += stats->sequence_executed; 2622 total->sequence_failed += stats->sequence_failed; 2623 total->retry.task += stats->retry.task; 2624 total->retry.sequence += stats->retry.sequence; 2625 total->retry.iobuf += stats->retry.iobuf; 2626 total->retry.bufdesc += stats->retry.bufdesc; 2627 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2628 total->operations[i].executed += stats->operations[i].executed; 2629 total->operations[i].failed += stats->operations[i].failed; 2630 total->operations[i].num_bytes += stats->operations[i].num_bytes; 2631 } 2632 } 2633 2634 /* Framework level channel destroy callback. */ 2635 static void 2636 accel_destroy_channel(void *io_device, void *ctx_buf) 2637 { 2638 struct accel_io_channel *accel_ch = ctx_buf; 2639 int i; 2640 2641 spdk_iobuf_channel_fini(&accel_ch->iobuf); 2642 2643 if (accel_ch->driver_channel != NULL) { 2644 spdk_put_io_channel(accel_ch->driver_channel); 2645 } 2646 2647 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 2648 assert(accel_ch->module_ch[i] != NULL); 2649 spdk_put_io_channel(accel_ch->module_ch[i]); 2650 accel_ch->module_ch[i] = NULL; 2651 } 2652 2653 /* Update global stats to make sure channel's stats aren't lost after a channel is gone */ 2654 spdk_spin_lock(&g_stats_lock); 2655 accel_add_stats(&g_stats, &accel_ch->stats); 2656 spdk_spin_unlock(&g_stats_lock); 2657 2658 free(accel_ch->task_pool_base); 2659 free(accel_ch->task_aux_data_base); 2660 free(accel_ch->seq_pool_base); 2661 free(accel_ch->buf_pool_base); 2662 } 2663 2664 struct spdk_io_channel * 2665 spdk_accel_get_io_channel(void) 2666 { 2667 return spdk_get_io_channel(&spdk_accel_module_list); 2668 } 2669 2670 static int 2671 accel_module_initialize(void) 2672 { 2673 struct spdk_accel_module_if *accel_module, *tmp_module; 2674 int rc = 0, module_rc; 2675 2676 TAILQ_FOREACH_SAFE(accel_module, &spdk_accel_module_list, tailq, tmp_module) { 2677 module_rc = accel_module->module_init(); 2678 if (module_rc) { 2679 TAILQ_REMOVE(&spdk_accel_module_list, accel_module, tailq); 2680 if (module_rc == -ENODEV) { 2681 SPDK_NOTICELOG("No devices for module %s, skipping\n", accel_module->name); 2682 } else if (!rc) { 2683 SPDK_ERRLOG("Module %s initialization failed with %d\n", accel_module->name, module_rc); 2684 rc = module_rc; 2685 } 2686 continue; 2687 } 2688 2689 SPDK_DEBUGLOG(accel, "Module %s initialized.\n", accel_module->name); 2690 } 2691 2692 return rc; 2693 } 2694 2695 static void 2696 accel_module_init_opcode(enum spdk_accel_opcode opcode) 2697 { 2698 struct accel_module *module = &g_modules_opc[opcode]; 2699 struct spdk_accel_module_if *module_if = module->module; 2700 2701 if (module_if->get_memory_domains != NULL) { 2702 module->supports_memory_domains = module_if->get_memory_domains(NULL, 0) > 0; 2703 } 2704 } 2705 2706 static int 2707 accel_memory_domain_translate(struct spdk_memory_domain *src_domain, void *src_domain_ctx, 2708 struct spdk_memory_domain *dst_domain, struct spdk_memory_domain_translation_ctx *dst_domain_ctx, 2709 void *addr, size_t len, struct spdk_memory_domain_translation_result *result) 2710 { 2711 struct accel_buffer *buf = src_domain_ctx; 2712 2713 SPDK_DEBUGLOG(accel, "translate addr %p, len %zu\n", addr, len); 2714 2715 assert(g_accel_domain == src_domain); 2716 assert(spdk_memory_domain_get_system_domain() == dst_domain); 2717 assert(buf->buf == NULL); 2718 assert(addr == ACCEL_BUFFER_BASE); 2719 assert(len == buf->len); 2720 2721 buf->buf = spdk_iobuf_get(&buf->ch->iobuf, buf->len, NULL, NULL); 2722 if (spdk_unlikely(buf->buf == NULL)) { 2723 return -ENOMEM; 2724 } 2725 2726 result->iov_count = 1; 2727 result->iov.iov_base = buf->buf; 2728 result->iov.iov_len = buf->len; 2729 SPDK_DEBUGLOG(accel, "translated addr %p\n", result->iov.iov_base); 2730 return 0; 2731 } 2732 2733 static void 2734 accel_memory_domain_invalidate(struct spdk_memory_domain *domain, void *domain_ctx, 2735 struct iovec *iov, uint32_t iovcnt) 2736 { 2737 struct accel_buffer *buf = domain_ctx; 2738 2739 SPDK_DEBUGLOG(accel, "invalidate addr %p, len %zu\n", iov[0].iov_base, iov[0].iov_len); 2740 2741 assert(g_accel_domain == domain); 2742 assert(iovcnt == 1); 2743 assert(buf->buf != NULL); 2744 assert(iov[0].iov_base == buf->buf); 2745 assert(iov[0].iov_len == buf->len); 2746 2747 spdk_iobuf_put(&buf->ch->iobuf, buf->buf, buf->len); 2748 buf->buf = NULL; 2749 } 2750 2751 int 2752 spdk_accel_initialize(void) 2753 { 2754 enum spdk_accel_opcode op; 2755 struct spdk_accel_module_if *accel_module = NULL; 2756 int rc; 2757 2758 /* 2759 * We need a unique identifier for the accel framework, so use the 2760 * spdk_accel_module_list address for this purpose. 2761 */ 2762 spdk_io_device_register(&spdk_accel_module_list, accel_create_channel, accel_destroy_channel, 2763 sizeof(struct accel_io_channel), "accel"); 2764 2765 spdk_spin_init(&g_keyring_spin); 2766 spdk_spin_init(&g_stats_lock); 2767 2768 rc = spdk_memory_domain_create(&g_accel_domain, SPDK_DMA_DEVICE_TYPE_ACCEL, NULL, 2769 "SPDK_ACCEL_DMA_DEVICE"); 2770 if (rc != 0) { 2771 SPDK_ERRLOG("Failed to create accel memory domain\n"); 2772 return rc; 2773 } 2774 2775 spdk_memory_domain_set_translation(g_accel_domain, accel_memory_domain_translate); 2776 spdk_memory_domain_set_invalidate(g_accel_domain, accel_memory_domain_invalidate); 2777 2778 g_modules_started = true; 2779 rc = accel_module_initialize(); 2780 if (rc) { 2781 return rc; 2782 } 2783 2784 if (g_accel_driver != NULL && g_accel_driver->init != NULL) { 2785 rc = g_accel_driver->init(); 2786 if (rc != 0) { 2787 SPDK_ERRLOG("Failed to initialize driver %s: %s\n", g_accel_driver->name, 2788 spdk_strerror(-rc)); 2789 return rc; 2790 } 2791 } 2792 2793 /* The module list is order by priority, with the highest priority modules being at the end 2794 * of the list. The software module should be somewhere at the beginning of the list, 2795 * before all HW modules. 2796 * NOTE: all opcodes must be supported by software in the event that no HW modules are 2797 * initialized to support the operation. 2798 */ 2799 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 2800 for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) { 2801 if (accel_module->supports_opcode(op)) { 2802 g_modules_opc[op].module = accel_module; 2803 SPDK_DEBUGLOG(accel, "OPC 0x%x now assigned to %s\n", op, accel_module->name); 2804 } 2805 } 2806 2807 if (accel_module->get_ctx_size != NULL) { 2808 g_max_accel_module_size = spdk_max(g_max_accel_module_size, 2809 accel_module->get_ctx_size()); 2810 } 2811 } 2812 2813 /* Now lets check for overrides and apply all that exist */ 2814 for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) { 2815 if (g_modules_opc_override[op] != NULL) { 2816 accel_module = _module_find_by_name(g_modules_opc_override[op]); 2817 if (accel_module == NULL) { 2818 SPDK_ERRLOG("Invalid module name of %s\n", g_modules_opc_override[op]); 2819 return -EINVAL; 2820 } 2821 if (accel_module->supports_opcode(op) == false) { 2822 SPDK_ERRLOG("Module %s does not support op code %d\n", accel_module->name, op); 2823 return -EINVAL; 2824 } 2825 g_modules_opc[op].module = accel_module; 2826 } 2827 } 2828 2829 if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) { 2830 SPDK_ERRLOG("Different accel modules are assigned to encrypt and decrypt operations"); 2831 return -EINVAL; 2832 } 2833 2834 for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) { 2835 assert(g_modules_opc[op].module != NULL); 2836 accel_module_init_opcode(op); 2837 } 2838 2839 rc = spdk_iobuf_register_module("accel"); 2840 if (rc != 0) { 2841 SPDK_ERRLOG("Failed to register accel iobuf module\n"); 2842 return rc; 2843 } 2844 2845 return 0; 2846 } 2847 2848 static void 2849 accel_module_finish_cb(void) 2850 { 2851 spdk_accel_fini_cb cb_fn = g_fini_cb_fn; 2852 2853 cb_fn(g_fini_cb_arg); 2854 g_fini_cb_fn = NULL; 2855 g_fini_cb_arg = NULL; 2856 } 2857 2858 static void 2859 accel_write_overridden_opc(struct spdk_json_write_ctx *w, const char *opc_str, 2860 const char *module_str) 2861 { 2862 spdk_json_write_object_begin(w); 2863 spdk_json_write_named_string(w, "method", "accel_assign_opc"); 2864 spdk_json_write_named_object_begin(w, "params"); 2865 spdk_json_write_named_string(w, "opname", opc_str); 2866 spdk_json_write_named_string(w, "module", module_str); 2867 spdk_json_write_object_end(w); 2868 spdk_json_write_object_end(w); 2869 } 2870 2871 static void 2872 __accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key) 2873 { 2874 spdk_json_write_named_string(w, "name", key->param.key_name); 2875 spdk_json_write_named_string(w, "cipher", key->param.cipher); 2876 spdk_json_write_named_string(w, "key", key->param.hex_key); 2877 if (key->param.hex_key2) { 2878 spdk_json_write_named_string(w, "key2", key->param.hex_key2); 2879 } 2880 2881 if (key->param.tweak_mode) { 2882 spdk_json_write_named_string(w, "tweak_mode", key->param.tweak_mode); 2883 } 2884 } 2885 2886 void 2887 _accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key) 2888 { 2889 spdk_json_write_object_begin(w); 2890 __accel_crypto_key_dump_param(w, key); 2891 spdk_json_write_object_end(w); 2892 } 2893 2894 static void 2895 _accel_crypto_key_write_config_json(struct spdk_json_write_ctx *w, 2896 struct spdk_accel_crypto_key *key) 2897 { 2898 spdk_json_write_object_begin(w); 2899 spdk_json_write_named_string(w, "method", "accel_crypto_key_create"); 2900 spdk_json_write_named_object_begin(w, "params"); 2901 __accel_crypto_key_dump_param(w, key); 2902 spdk_json_write_object_end(w); 2903 spdk_json_write_object_end(w); 2904 } 2905 2906 static void 2907 accel_write_options(struct spdk_json_write_ctx *w) 2908 { 2909 spdk_json_write_object_begin(w); 2910 spdk_json_write_named_string(w, "method", "accel_set_options"); 2911 spdk_json_write_named_object_begin(w, "params"); 2912 spdk_json_write_named_uint32(w, "small_cache_size", g_opts.small_cache_size); 2913 spdk_json_write_named_uint32(w, "large_cache_size", g_opts.large_cache_size); 2914 spdk_json_write_named_uint32(w, "task_count", g_opts.task_count); 2915 spdk_json_write_named_uint32(w, "sequence_count", g_opts.sequence_count); 2916 spdk_json_write_named_uint32(w, "buf_count", g_opts.buf_count); 2917 spdk_json_write_object_end(w); 2918 spdk_json_write_object_end(w); 2919 } 2920 2921 static void 2922 _accel_crypto_keys_write_config_json(struct spdk_json_write_ctx *w, bool full_dump) 2923 { 2924 struct spdk_accel_crypto_key *key; 2925 2926 spdk_spin_lock(&g_keyring_spin); 2927 TAILQ_FOREACH(key, &g_keyring, link) { 2928 if (full_dump) { 2929 _accel_crypto_key_write_config_json(w, key); 2930 } else { 2931 _accel_crypto_key_dump_param(w, key); 2932 } 2933 } 2934 spdk_spin_unlock(&g_keyring_spin); 2935 } 2936 2937 void 2938 _accel_crypto_keys_dump_param(struct spdk_json_write_ctx *w) 2939 { 2940 _accel_crypto_keys_write_config_json(w, false); 2941 } 2942 2943 void 2944 spdk_accel_write_config_json(struct spdk_json_write_ctx *w) 2945 { 2946 struct spdk_accel_module_if *accel_module; 2947 int i; 2948 2949 spdk_json_write_array_begin(w); 2950 accel_write_options(w); 2951 2952 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 2953 if (accel_module->write_config_json) { 2954 accel_module->write_config_json(w); 2955 } 2956 } 2957 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 2958 if (g_modules_opc_override[i]) { 2959 accel_write_overridden_opc(w, g_opcode_strings[i], g_modules_opc_override[i]); 2960 } 2961 } 2962 2963 _accel_crypto_keys_write_config_json(w, true); 2964 2965 spdk_json_write_array_end(w); 2966 } 2967 2968 void 2969 spdk_accel_module_finish(void) 2970 { 2971 if (!g_accel_module) { 2972 g_accel_module = TAILQ_FIRST(&spdk_accel_module_list); 2973 } else { 2974 g_accel_module = TAILQ_NEXT(g_accel_module, tailq); 2975 } 2976 2977 if (!g_accel_module) { 2978 if (g_accel_driver != NULL && g_accel_driver->fini != NULL) { 2979 g_accel_driver->fini(); 2980 } 2981 2982 spdk_spin_destroy(&g_keyring_spin); 2983 spdk_spin_destroy(&g_stats_lock); 2984 if (g_accel_domain) { 2985 spdk_memory_domain_destroy(g_accel_domain); 2986 g_accel_domain = NULL; 2987 } 2988 accel_module_finish_cb(); 2989 return; 2990 } 2991 2992 if (g_accel_module->module_fini) { 2993 spdk_thread_send_msg(spdk_get_thread(), g_accel_module->module_fini, NULL); 2994 } else { 2995 spdk_accel_module_finish(); 2996 } 2997 } 2998 2999 static void 3000 accel_io_device_unregister_cb(void *io_device) 3001 { 3002 struct spdk_accel_crypto_key *key, *key_tmp; 3003 enum spdk_accel_opcode op; 3004 3005 spdk_spin_lock(&g_keyring_spin); 3006 TAILQ_FOREACH_SAFE(key, &g_keyring, link, key_tmp) { 3007 accel_crypto_key_destroy_unsafe(key); 3008 } 3009 spdk_spin_unlock(&g_keyring_spin); 3010 3011 for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) { 3012 if (g_modules_opc_override[op] != NULL) { 3013 free(g_modules_opc_override[op]); 3014 g_modules_opc_override[op] = NULL; 3015 } 3016 g_modules_opc[op].module = NULL; 3017 } 3018 3019 spdk_accel_module_finish(); 3020 } 3021 3022 void 3023 spdk_accel_finish(spdk_accel_fini_cb cb_fn, void *cb_arg) 3024 { 3025 assert(cb_fn != NULL); 3026 3027 g_fini_cb_fn = cb_fn; 3028 g_fini_cb_arg = cb_arg; 3029 3030 spdk_io_device_unregister(&spdk_accel_module_list, accel_io_device_unregister_cb); 3031 } 3032 3033 static struct spdk_accel_driver * 3034 accel_find_driver(const char *name) 3035 { 3036 struct spdk_accel_driver *driver; 3037 3038 TAILQ_FOREACH(driver, &g_accel_drivers, tailq) { 3039 if (strcmp(driver->name, name) == 0) { 3040 return driver; 3041 } 3042 } 3043 3044 return NULL; 3045 } 3046 3047 int 3048 spdk_accel_set_driver(const char *name) 3049 { 3050 struct spdk_accel_driver *driver; 3051 3052 driver = accel_find_driver(name); 3053 if (driver == NULL) { 3054 SPDK_ERRLOG("Couldn't find driver named '%s'\n", name); 3055 return -ENODEV; 3056 } 3057 3058 g_accel_driver = driver; 3059 3060 return 0; 3061 } 3062 3063 void 3064 spdk_accel_driver_register(struct spdk_accel_driver *driver) 3065 { 3066 if (accel_find_driver(driver->name)) { 3067 SPDK_ERRLOG("Driver named '%s' has already been registered\n", driver->name); 3068 assert(0); 3069 return; 3070 } 3071 3072 TAILQ_INSERT_TAIL(&g_accel_drivers, driver, tailq); 3073 } 3074 3075 int 3076 spdk_accel_set_opts(const struct spdk_accel_opts *opts) 3077 { 3078 if (!opts) { 3079 SPDK_ERRLOG("opts cannot be NULL\n"); 3080 return -1; 3081 } 3082 3083 if (!opts->opts_size) { 3084 SPDK_ERRLOG("opts_size inside opts cannot be zero value\n"); 3085 return -1; 3086 } 3087 3088 #define SET_FIELD(field) \ 3089 if (offsetof(struct spdk_accel_opts, field) + sizeof(opts->field) <= opts->opts_size) { \ 3090 g_opts.field = opts->field; \ 3091 } \ 3092 3093 SET_FIELD(small_cache_size); 3094 SET_FIELD(large_cache_size); 3095 SET_FIELD(task_count); 3096 SET_FIELD(sequence_count); 3097 SET_FIELD(buf_count); 3098 3099 g_opts.opts_size = opts->opts_size; 3100 3101 #undef SET_FIELD 3102 3103 return 0; 3104 } 3105 3106 void 3107 spdk_accel_get_opts(struct spdk_accel_opts *opts, size_t opts_size) 3108 { 3109 if (!opts) { 3110 SPDK_ERRLOG("opts should not be NULL\n"); 3111 return; 3112 } 3113 3114 if (!opts_size) { 3115 SPDK_ERRLOG("opts_size should not be zero value\n"); 3116 return; 3117 } 3118 3119 opts->opts_size = opts_size; 3120 3121 #define SET_FIELD(field) \ 3122 if (offsetof(struct spdk_accel_opts, field) + sizeof(opts->field) <= opts_size) { \ 3123 opts->field = g_opts.field; \ 3124 } \ 3125 3126 SET_FIELD(small_cache_size); 3127 SET_FIELD(large_cache_size); 3128 SET_FIELD(task_count); 3129 SET_FIELD(sequence_count); 3130 SET_FIELD(buf_count); 3131 3132 #undef SET_FIELD 3133 3134 /* Do not remove this statement, you should always update this statement when you adding a new field, 3135 * and do not forget to add the SET_FIELD statement for your added field. */ 3136 SPDK_STATIC_ASSERT(sizeof(struct spdk_accel_opts) == 28, "Incorrect size"); 3137 } 3138 3139 struct accel_get_stats_ctx { 3140 struct accel_stats stats; 3141 accel_get_stats_cb cb_fn; 3142 void *cb_arg; 3143 }; 3144 3145 static void 3146 accel_get_channel_stats_done(struct spdk_io_channel_iter *iter, int status) 3147 { 3148 struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter); 3149 3150 ctx->cb_fn(&ctx->stats, ctx->cb_arg); 3151 free(ctx); 3152 } 3153 3154 static void 3155 accel_get_channel_stats(struct spdk_io_channel_iter *iter) 3156 { 3157 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(iter); 3158 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 3159 struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter); 3160 3161 accel_add_stats(&ctx->stats, &accel_ch->stats); 3162 spdk_for_each_channel_continue(iter, 0); 3163 } 3164 3165 int 3166 accel_get_stats(accel_get_stats_cb cb_fn, void *cb_arg) 3167 { 3168 struct accel_get_stats_ctx *ctx; 3169 3170 ctx = calloc(1, sizeof(*ctx)); 3171 if (ctx == NULL) { 3172 return -ENOMEM; 3173 } 3174 3175 spdk_spin_lock(&g_stats_lock); 3176 accel_add_stats(&ctx->stats, &g_stats); 3177 spdk_spin_unlock(&g_stats_lock); 3178 3179 ctx->cb_fn = cb_fn; 3180 ctx->cb_arg = cb_arg; 3181 3182 spdk_for_each_channel(&spdk_accel_module_list, accel_get_channel_stats, ctx, 3183 accel_get_channel_stats_done); 3184 3185 return 0; 3186 } 3187 3188 void 3189 spdk_accel_get_opcode_stats(struct spdk_io_channel *ch, enum spdk_accel_opcode opcode, 3190 struct spdk_accel_opcode_stats *stats, size_t size) 3191 { 3192 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 3193 3194 #define FIELD_OK(field) \ 3195 offsetof(struct spdk_accel_opcode_stats, field) + sizeof(stats->field) <= size 3196 3197 #define SET_FIELD(field, value) \ 3198 if (FIELD_OK(field)) { \ 3199 stats->field = value; \ 3200 } 3201 3202 SET_FIELD(executed, accel_ch->stats.operations[opcode].executed); 3203 SET_FIELD(failed, accel_ch->stats.operations[opcode].failed); 3204 SET_FIELD(num_bytes, accel_ch->stats.operations[opcode].num_bytes); 3205 3206 #undef FIELD_OK 3207 #undef SET_FIELD 3208 } 3209 3210 uint8_t 3211 spdk_accel_get_buf_align(enum spdk_accel_opcode opcode, 3212 const struct spdk_accel_operation_exec_ctx *ctx) 3213 { 3214 struct spdk_accel_module_if *module = g_modules_opc[opcode].module; 3215 struct spdk_accel_opcode_info modinfo = {}, drvinfo = {}; 3216 3217 if (g_accel_driver != NULL && g_accel_driver->get_operation_info != NULL) { 3218 g_accel_driver->get_operation_info(opcode, ctx, &drvinfo); 3219 } 3220 3221 if (module->get_operation_info != NULL) { 3222 module->get_operation_info(opcode, ctx, &modinfo); 3223 } 3224 3225 /* If a driver is set, it'll execute most of the operations, while the rest will usually 3226 * fall back to accel_sw, which doesn't have any alignment requiremenets. However, to be 3227 * extra safe, return the max(driver, module) if a driver delegates some operations to a 3228 * hardware module. */ 3229 return spdk_max(modinfo.required_alignment, drvinfo.required_alignment); 3230 } 3231 3232 struct spdk_accel_module_if * 3233 spdk_accel_get_module(const char *name) 3234 { 3235 struct spdk_accel_module_if *module; 3236 3237 TAILQ_FOREACH(module, &spdk_accel_module_list, tailq) { 3238 if (strcmp(module->name, name) == 0) { 3239 return module; 3240 } 3241 } 3242 3243 return NULL; 3244 } 3245 3246 int 3247 spdk_accel_get_opc_memory_domains(enum spdk_accel_opcode opcode, 3248 struct spdk_memory_domain **domains, 3249 int array_size) 3250 { 3251 assert(opcode < SPDK_ACCEL_OPC_LAST); 3252 3253 if (g_modules_opc[opcode].module->get_memory_domains) { 3254 return g_modules_opc[opcode].module->get_memory_domains(domains, array_size); 3255 } 3256 3257 return 0; 3258 } 3259 3260 SPDK_LOG_REGISTER_COMPONENT(accel) 3261