1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2020 Intel Corporation. 3 * Copyright (c) 2022, 2023 NVIDIA CORPORATION & AFFILIATES. 4 * All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk/accel_module.h" 10 11 #include "accel_internal.h" 12 13 #include "spdk/dma.h" 14 #include "spdk/env.h" 15 #include "spdk/likely.h" 16 #include "spdk/log.h" 17 #include "spdk/thread.h" 18 #include "spdk/json.h" 19 #include "spdk/crc32.h" 20 #include "spdk/util.h" 21 #include "spdk/hexlify.h" 22 #include "spdk/string.h" 23 24 /* Accelerator Framework: The following provides a top level 25 * generic API for the accelerator functions defined here. Modules, 26 * such as the one in /module/accel/ioat, supply the implementation 27 * with the exception of the pure software implementation contained 28 * later in this file. 29 */ 30 31 #define ALIGN_4K 0x1000 32 #define ACCEL_TASKS_PER_CHANNEL 2048 33 #define ACCEL_SMALL_CACHE_SIZE 128 34 #define ACCEL_LARGE_CACHE_SIZE 16 35 /* Set MSB, so we don't return NULL pointers as buffers */ 36 #define ACCEL_BUFFER_BASE ((void *)(1ull << 63)) 37 #define ACCEL_BUFFER_OFFSET_MASK ((uintptr_t)ACCEL_BUFFER_BASE - 1) 38 39 #define ACCEL_CRYPTO_TWEAK_MODE_DEFAULT SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA 40 #define ACCEL_TASKS_IN_SEQUENCE_LIMIT 8 41 42 struct accel_module { 43 struct spdk_accel_module_if *module; 44 bool supports_memory_domains; 45 }; 46 47 /* Largest context size for all accel modules */ 48 static size_t g_max_accel_module_size = sizeof(struct spdk_accel_task); 49 50 static struct spdk_accel_module_if *g_accel_module = NULL; 51 static spdk_accel_fini_cb g_fini_cb_fn = NULL; 52 static void *g_fini_cb_arg = NULL; 53 static bool g_modules_started = false; 54 static struct spdk_memory_domain *g_accel_domain; 55 56 /* Global list of registered accelerator modules */ 57 static TAILQ_HEAD(, spdk_accel_module_if) spdk_accel_module_list = 58 TAILQ_HEAD_INITIALIZER(spdk_accel_module_list); 59 60 /* Crypto keyring */ 61 static TAILQ_HEAD(, spdk_accel_crypto_key) g_keyring = TAILQ_HEAD_INITIALIZER(g_keyring); 62 static struct spdk_spinlock g_keyring_spin; 63 64 /* Global array mapping capabilities to modules */ 65 static struct accel_module g_modules_opc[SPDK_ACCEL_OPC_LAST] = {}; 66 static char *g_modules_opc_override[SPDK_ACCEL_OPC_LAST] = {}; 67 TAILQ_HEAD(, spdk_accel_driver) g_accel_drivers = TAILQ_HEAD_INITIALIZER(g_accel_drivers); 68 static struct spdk_accel_driver *g_accel_driver; 69 static struct spdk_accel_opts g_opts = { 70 .small_cache_size = ACCEL_SMALL_CACHE_SIZE, 71 .large_cache_size = ACCEL_LARGE_CACHE_SIZE, 72 .task_count = ACCEL_TASKS_PER_CHANNEL, 73 .sequence_count = ACCEL_TASKS_PER_CHANNEL, 74 .buf_count = ACCEL_TASKS_PER_CHANNEL, 75 }; 76 static struct accel_stats g_stats; 77 static struct spdk_spinlock g_stats_lock; 78 79 static const char *g_opcode_strings[SPDK_ACCEL_OPC_LAST] = { 80 "copy", "fill", "dualcast", "compare", "crc32c", "copy_crc32c", 81 "compress", "decompress", "encrypt", "decrypt", "xor", 82 "dif_verify", "dif_verify_copy", "dif_generate", "dif_generate_copy", 83 "dix_generate", "dix_verify" 84 }; 85 86 enum accel_sequence_state { 87 ACCEL_SEQUENCE_STATE_INIT, 88 ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF, 89 ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF, 90 ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF, 91 ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF, 92 ACCEL_SEQUENCE_STATE_PULL_DATA, 93 ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA, 94 ACCEL_SEQUENCE_STATE_EXEC_TASK, 95 ACCEL_SEQUENCE_STATE_AWAIT_TASK, 96 ACCEL_SEQUENCE_STATE_COMPLETE_TASK, 97 ACCEL_SEQUENCE_STATE_NEXT_TASK, 98 ACCEL_SEQUENCE_STATE_PUSH_DATA, 99 ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA, 100 ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS, 101 ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS, 102 ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS, 103 ACCEL_SEQUENCE_STATE_ERROR, 104 ACCEL_SEQUENCE_STATE_MAX, 105 }; 106 107 static const char *g_seq_states[] 108 __attribute__((unused)) = { 109 [ACCEL_SEQUENCE_STATE_INIT] = "init", 110 [ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF] = "check-virtbuf", 111 [ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF] = "await-virtbuf", 112 [ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF] = "check-bouncebuf", 113 [ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF] = "await-bouncebuf", 114 [ACCEL_SEQUENCE_STATE_PULL_DATA] = "pull-data", 115 [ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA] = "await-pull-data", 116 [ACCEL_SEQUENCE_STATE_EXEC_TASK] = "exec-task", 117 [ACCEL_SEQUENCE_STATE_AWAIT_TASK] = "await-task", 118 [ACCEL_SEQUENCE_STATE_COMPLETE_TASK] = "complete-task", 119 [ACCEL_SEQUENCE_STATE_NEXT_TASK] = "next-task", 120 [ACCEL_SEQUENCE_STATE_PUSH_DATA] = "push-data", 121 [ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA] = "await-push-data", 122 [ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS] = "driver-exec-tasks", 123 [ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS] = "driver-await-tasks", 124 [ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS] = "driver-complete-tasks", 125 [ACCEL_SEQUENCE_STATE_ERROR] = "error", 126 [ACCEL_SEQUENCE_STATE_MAX] = "", 127 }; 128 129 #define ACCEL_SEQUENCE_STATE_STRING(s) \ 130 (((s) >= ACCEL_SEQUENCE_STATE_INIT && (s) < ACCEL_SEQUENCE_STATE_MAX) \ 131 ? g_seq_states[s] : "unknown") 132 133 struct accel_buffer { 134 struct spdk_accel_sequence *seq; 135 void *buf; 136 uint64_t len; 137 struct spdk_iobuf_entry iobuf; 138 spdk_accel_sequence_get_buf_cb cb_fn; 139 void *cb_ctx; 140 SLIST_ENTRY(accel_buffer) link; 141 struct accel_io_channel *ch; 142 }; 143 144 struct accel_io_channel { 145 struct spdk_io_channel *module_ch[SPDK_ACCEL_OPC_LAST]; 146 struct spdk_io_channel *driver_channel; 147 void *task_pool_base; 148 struct spdk_accel_sequence *seq_pool_base; 149 struct accel_buffer *buf_pool_base; 150 struct spdk_accel_task_aux_data *task_aux_data_base; 151 STAILQ_HEAD(, spdk_accel_task) task_pool; 152 SLIST_HEAD(, spdk_accel_task_aux_data) task_aux_data_pool; 153 SLIST_HEAD(, spdk_accel_sequence) seq_pool; 154 SLIST_HEAD(, accel_buffer) buf_pool; 155 struct spdk_iobuf_channel iobuf; 156 struct accel_stats stats; 157 }; 158 159 TAILQ_HEAD(accel_sequence_tasks, spdk_accel_task); 160 161 struct spdk_accel_sequence { 162 struct accel_io_channel *ch; 163 struct accel_sequence_tasks tasks; 164 SLIST_HEAD(, accel_buffer) bounce_bufs; 165 int status; 166 /* state uses enum accel_sequence_state */ 167 uint8_t state; 168 bool in_process_sequence; 169 spdk_accel_completion_cb cb_fn; 170 void *cb_arg; 171 SLIST_ENTRY(spdk_accel_sequence) link; 172 }; 173 SPDK_STATIC_ASSERT(sizeof(struct spdk_accel_sequence) == 64, "invalid size"); 174 175 #define accel_update_stats(ch, event, v) \ 176 do { \ 177 (ch)->stats.event += (v); \ 178 } while (0) 179 180 #define accel_update_task_stats(ch, task, event, v) \ 181 accel_update_stats(ch, operations[(task)->op_code].event, v) 182 183 static inline void accel_sequence_task_cb(void *cb_arg, int status); 184 185 static inline void 186 accel_sequence_set_state(struct spdk_accel_sequence *seq, enum accel_sequence_state state) 187 { 188 SPDK_DEBUGLOG(accel, "seq=%p, setting state: %s -> %s\n", seq, 189 ACCEL_SEQUENCE_STATE_STRING(seq->state), ACCEL_SEQUENCE_STATE_STRING(state)); 190 assert(seq->state != ACCEL_SEQUENCE_STATE_ERROR || state == ACCEL_SEQUENCE_STATE_ERROR); 191 seq->state = state; 192 } 193 194 static void 195 accel_sequence_set_fail(struct spdk_accel_sequence *seq, int status) 196 { 197 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR); 198 assert(status != 0); 199 seq->status = status; 200 } 201 202 int 203 spdk_accel_get_opc_module_name(enum spdk_accel_opcode opcode, const char **module_name) 204 { 205 if (opcode >= SPDK_ACCEL_OPC_LAST) { 206 /* invalid opcode */ 207 return -EINVAL; 208 } 209 210 if (g_modules_opc[opcode].module) { 211 *module_name = g_modules_opc[opcode].module->name; 212 } else { 213 return -ENOENT; 214 } 215 216 return 0; 217 } 218 219 void 220 _accel_for_each_module(struct module_info *info, _accel_for_each_module_fn fn) 221 { 222 struct spdk_accel_module_if *accel_module; 223 enum spdk_accel_opcode opcode; 224 int j = 0; 225 226 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 227 for (opcode = 0; opcode < SPDK_ACCEL_OPC_LAST; opcode++) { 228 if (accel_module->supports_opcode(opcode)) { 229 info->ops[j] = opcode; 230 j++; 231 } 232 } 233 info->name = accel_module->name; 234 info->num_ops = j; 235 fn(info); 236 j = 0; 237 } 238 } 239 240 const char * 241 spdk_accel_get_opcode_name(enum spdk_accel_opcode opcode) 242 { 243 if (opcode < SPDK_ACCEL_OPC_LAST) { 244 return g_opcode_strings[opcode]; 245 } 246 247 return NULL; 248 } 249 250 int 251 spdk_accel_assign_opc(enum spdk_accel_opcode opcode, const char *name) 252 { 253 char *copy; 254 255 if (g_modules_started == true) { 256 /* we don't allow re-assignment once things have started */ 257 return -EINVAL; 258 } 259 260 if (opcode >= SPDK_ACCEL_OPC_LAST) { 261 /* invalid opcode */ 262 return -EINVAL; 263 } 264 265 copy = strdup(name); 266 if (copy == NULL) { 267 return -ENOMEM; 268 } 269 270 /* module selection will be validated after the framework starts. */ 271 free(g_modules_opc_override[opcode]); 272 g_modules_opc_override[opcode] = copy; 273 274 return 0; 275 } 276 277 inline static struct spdk_accel_task * 278 _get_task(struct accel_io_channel *accel_ch, spdk_accel_completion_cb cb_fn, void *cb_arg) 279 { 280 struct spdk_accel_task *accel_task; 281 282 accel_task = STAILQ_FIRST(&accel_ch->task_pool); 283 if (spdk_unlikely(accel_task == NULL)) { 284 accel_update_stats(accel_ch, retry.task, 1); 285 return NULL; 286 } 287 288 accel_update_stats(accel_ch, task_outstanding, 1); 289 STAILQ_REMOVE_HEAD(&accel_ch->task_pool, link); 290 accel_task->link.stqe_next = NULL; 291 292 accel_task->cb_fn = cb_fn; 293 accel_task->cb_arg = cb_arg; 294 accel_task->accel_ch = accel_ch; 295 accel_task->s.iovs = NULL; 296 accel_task->d.iovs = NULL; 297 298 return accel_task; 299 } 300 301 static void 302 _put_task(struct accel_io_channel *ch, struct spdk_accel_task *task) 303 { 304 STAILQ_INSERT_HEAD(&ch->task_pool, task, link); 305 accel_update_stats(ch, task_outstanding, -1); 306 } 307 308 void 309 spdk_accel_task_complete(struct spdk_accel_task *accel_task, int status) 310 { 311 struct accel_io_channel *accel_ch = accel_task->accel_ch; 312 spdk_accel_completion_cb cb_fn; 313 void *cb_arg; 314 315 accel_update_task_stats(accel_ch, accel_task, executed, 1); 316 accel_update_task_stats(accel_ch, accel_task, num_bytes, accel_task->nbytes); 317 if (spdk_unlikely(status != 0)) { 318 accel_update_task_stats(accel_ch, accel_task, failed, 1); 319 } 320 321 if (accel_task->seq) { 322 accel_sequence_task_cb(accel_task->seq, status); 323 return; 324 } 325 326 cb_fn = accel_task->cb_fn; 327 cb_arg = accel_task->cb_arg; 328 329 if (accel_task->has_aux) { 330 SLIST_INSERT_HEAD(&accel_ch->task_aux_data_pool, accel_task->aux, link); 331 accel_task->aux = NULL; 332 accel_task->has_aux = false; 333 } 334 335 /* We should put the accel_task into the list firstly in order to avoid 336 * the accel task list is exhausted when there is recursive call to 337 * allocate accel_task in user's call back function (cb_fn) 338 */ 339 _put_task(accel_ch, accel_task); 340 341 cb_fn(cb_arg, status); 342 } 343 344 static inline int 345 accel_submit_task(struct accel_io_channel *accel_ch, struct spdk_accel_task *task) 346 { 347 struct spdk_io_channel *module_ch = accel_ch->module_ch[task->op_code]; 348 struct spdk_accel_module_if *module = g_modules_opc[task->op_code].module; 349 int rc; 350 351 rc = module->submit_tasks(module_ch, task); 352 if (spdk_unlikely(rc != 0)) { 353 accel_update_task_stats(accel_ch, task, failed, 1); 354 } 355 356 return rc; 357 } 358 359 static inline uint64_t 360 accel_get_iovlen(struct iovec *iovs, uint32_t iovcnt) 361 { 362 uint64_t result = 0; 363 uint32_t i; 364 365 for (i = 0; i < iovcnt; ++i) { 366 result += iovs[i].iov_len; 367 } 368 369 return result; 370 } 371 372 #define ACCEL_TASK_ALLOC_AUX_BUF(task) \ 373 do { \ 374 (task)->aux = SLIST_FIRST(&(task)->accel_ch->task_aux_data_pool); \ 375 if (spdk_unlikely(!(task)->aux)) { \ 376 SPDK_ERRLOG("Fatal problem, aux data was not allocated\n"); \ 377 _put_task(task->accel_ch, task); \ 378 assert(0); \ 379 return -ENOMEM; \ 380 } \ 381 SLIST_REMOVE_HEAD(&(task)->accel_ch->task_aux_data_pool, link); \ 382 (task)->has_aux = true; \ 383 } while (0) 384 385 /* Accel framework public API for copy function */ 386 int 387 spdk_accel_submit_copy(struct spdk_io_channel *ch, void *dst, void *src, 388 uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg) 389 { 390 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 391 struct spdk_accel_task *accel_task; 392 393 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 394 if (spdk_unlikely(accel_task == NULL)) { 395 return -ENOMEM; 396 } 397 398 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 399 400 accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC]; 401 accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 402 accel_task->d.iovs[0].iov_base = dst; 403 accel_task->d.iovs[0].iov_len = nbytes; 404 accel_task->d.iovcnt = 1; 405 accel_task->s.iovs[0].iov_base = src; 406 accel_task->s.iovs[0].iov_len = nbytes; 407 accel_task->s.iovcnt = 1; 408 accel_task->nbytes = nbytes; 409 accel_task->op_code = SPDK_ACCEL_OPC_COPY; 410 accel_task->src_domain = NULL; 411 accel_task->dst_domain = NULL; 412 413 return accel_submit_task(accel_ch, accel_task); 414 } 415 416 /* Accel framework public API for dual cast copy function */ 417 int 418 spdk_accel_submit_dualcast(struct spdk_io_channel *ch, void *dst1, 419 void *dst2, void *src, uint64_t nbytes, 420 spdk_accel_completion_cb cb_fn, void *cb_arg) 421 { 422 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 423 struct spdk_accel_task *accel_task; 424 425 if ((uintptr_t)dst1 & (ALIGN_4K - 1) || (uintptr_t)dst2 & (ALIGN_4K - 1)) { 426 SPDK_ERRLOG("Dualcast requires 4K alignment on dst addresses\n"); 427 return -EINVAL; 428 } 429 430 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 431 if (spdk_unlikely(accel_task == NULL)) { 432 return -ENOMEM; 433 } 434 435 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 436 437 accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC]; 438 accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 439 accel_task->d2.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST2]; 440 accel_task->d.iovs[0].iov_base = dst1; 441 accel_task->d.iovs[0].iov_len = nbytes; 442 accel_task->d.iovcnt = 1; 443 accel_task->d2.iovs[0].iov_base = dst2; 444 accel_task->d2.iovs[0].iov_len = nbytes; 445 accel_task->d2.iovcnt = 1; 446 accel_task->s.iovs[0].iov_base = src; 447 accel_task->s.iovs[0].iov_len = nbytes; 448 accel_task->s.iovcnt = 1; 449 accel_task->nbytes = nbytes; 450 accel_task->op_code = SPDK_ACCEL_OPC_DUALCAST; 451 accel_task->src_domain = NULL; 452 accel_task->dst_domain = NULL; 453 454 return accel_submit_task(accel_ch, accel_task); 455 } 456 457 /* Accel framework public API for compare function */ 458 459 int 460 spdk_accel_submit_compare(struct spdk_io_channel *ch, void *src1, 461 void *src2, uint64_t nbytes, spdk_accel_completion_cb cb_fn, 462 void *cb_arg) 463 { 464 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 465 struct spdk_accel_task *accel_task; 466 467 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 468 if (spdk_unlikely(accel_task == NULL)) { 469 return -ENOMEM; 470 } 471 472 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 473 474 accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC]; 475 accel_task->s2.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC2]; 476 accel_task->s.iovs[0].iov_base = src1; 477 accel_task->s.iovs[0].iov_len = nbytes; 478 accel_task->s.iovcnt = 1; 479 accel_task->s2.iovs[0].iov_base = src2; 480 accel_task->s2.iovs[0].iov_len = nbytes; 481 accel_task->s2.iovcnt = 1; 482 accel_task->nbytes = nbytes; 483 accel_task->op_code = SPDK_ACCEL_OPC_COMPARE; 484 accel_task->src_domain = NULL; 485 accel_task->dst_domain = NULL; 486 487 return accel_submit_task(accel_ch, accel_task); 488 } 489 490 /* Accel framework public API for fill function */ 491 int 492 spdk_accel_submit_fill(struct spdk_io_channel *ch, void *dst, 493 uint8_t fill, uint64_t nbytes, 494 spdk_accel_completion_cb cb_fn, void *cb_arg) 495 { 496 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 497 struct spdk_accel_task *accel_task; 498 499 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 500 if (spdk_unlikely(accel_task == NULL)) { 501 return -ENOMEM; 502 } 503 504 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 505 506 accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 507 accel_task->d.iovs[0].iov_base = dst; 508 accel_task->d.iovs[0].iov_len = nbytes; 509 accel_task->d.iovcnt = 1; 510 accel_task->nbytes = nbytes; 511 memset(&accel_task->fill_pattern, fill, sizeof(uint64_t)); 512 accel_task->op_code = SPDK_ACCEL_OPC_FILL; 513 accel_task->src_domain = NULL; 514 accel_task->dst_domain = NULL; 515 516 return accel_submit_task(accel_ch, accel_task); 517 } 518 519 /* Accel framework public API for CRC-32C function */ 520 int 521 spdk_accel_submit_crc32c(struct spdk_io_channel *ch, uint32_t *crc_dst, 522 void *src, uint32_t seed, uint64_t nbytes, spdk_accel_completion_cb cb_fn, 523 void *cb_arg) 524 { 525 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 526 struct spdk_accel_task *accel_task; 527 528 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 529 if (spdk_unlikely(accel_task == NULL)) { 530 return -ENOMEM; 531 } 532 533 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 534 535 accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC]; 536 accel_task->s.iovs[0].iov_base = src; 537 accel_task->s.iovs[0].iov_len = nbytes; 538 accel_task->s.iovcnt = 1; 539 accel_task->nbytes = nbytes; 540 accel_task->crc_dst = crc_dst; 541 accel_task->seed = seed; 542 accel_task->op_code = SPDK_ACCEL_OPC_CRC32C; 543 accel_task->src_domain = NULL; 544 accel_task->dst_domain = NULL; 545 546 return accel_submit_task(accel_ch, accel_task); 547 } 548 549 /* Accel framework public API for chained CRC-32C function */ 550 int 551 spdk_accel_submit_crc32cv(struct spdk_io_channel *ch, uint32_t *crc_dst, 552 struct iovec *iov, uint32_t iov_cnt, uint32_t seed, 553 spdk_accel_completion_cb cb_fn, void *cb_arg) 554 { 555 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 556 struct spdk_accel_task *accel_task; 557 558 if (iov == NULL) { 559 SPDK_ERRLOG("iov should not be NULL"); 560 return -EINVAL; 561 } 562 563 if (!iov_cnt) { 564 SPDK_ERRLOG("iovcnt should not be zero value\n"); 565 return -EINVAL; 566 } 567 568 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 569 if (spdk_unlikely(accel_task == NULL)) { 570 SPDK_ERRLOG("no memory\n"); 571 assert(0); 572 return -ENOMEM; 573 } 574 575 accel_task->s.iovs = iov; 576 accel_task->s.iovcnt = iov_cnt; 577 accel_task->nbytes = accel_get_iovlen(iov, iov_cnt); 578 accel_task->crc_dst = crc_dst; 579 accel_task->seed = seed; 580 accel_task->op_code = SPDK_ACCEL_OPC_CRC32C; 581 accel_task->src_domain = NULL; 582 accel_task->dst_domain = NULL; 583 584 return accel_submit_task(accel_ch, accel_task); 585 } 586 587 /* Accel framework public API for copy with CRC-32C function */ 588 int 589 spdk_accel_submit_copy_crc32c(struct spdk_io_channel *ch, void *dst, 590 void *src, uint32_t *crc_dst, uint32_t seed, uint64_t nbytes, 591 spdk_accel_completion_cb cb_fn, void *cb_arg) 592 { 593 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 594 struct spdk_accel_task *accel_task; 595 596 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 597 if (spdk_unlikely(accel_task == NULL)) { 598 return -ENOMEM; 599 } 600 601 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 602 603 accel_task->s.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_SRC]; 604 accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 605 accel_task->d.iovs[0].iov_base = dst; 606 accel_task->d.iovs[0].iov_len = nbytes; 607 accel_task->d.iovcnt = 1; 608 accel_task->s.iovs[0].iov_base = src; 609 accel_task->s.iovs[0].iov_len = nbytes; 610 accel_task->s.iovcnt = 1; 611 accel_task->nbytes = nbytes; 612 accel_task->crc_dst = crc_dst; 613 accel_task->seed = seed; 614 accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C; 615 accel_task->src_domain = NULL; 616 accel_task->dst_domain = NULL; 617 618 return accel_submit_task(accel_ch, accel_task); 619 } 620 621 /* Accel framework public API for chained copy + CRC-32C function */ 622 int 623 spdk_accel_submit_copy_crc32cv(struct spdk_io_channel *ch, void *dst, 624 struct iovec *src_iovs, uint32_t iov_cnt, uint32_t *crc_dst, 625 uint32_t seed, spdk_accel_completion_cb cb_fn, void *cb_arg) 626 { 627 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 628 struct spdk_accel_task *accel_task; 629 uint64_t nbytes; 630 631 if (src_iovs == NULL) { 632 SPDK_ERRLOG("iov should not be NULL"); 633 return -EINVAL; 634 } 635 636 if (!iov_cnt) { 637 SPDK_ERRLOG("iovcnt should not be zero value\n"); 638 return -EINVAL; 639 } 640 641 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 642 if (spdk_unlikely(accel_task == NULL)) { 643 SPDK_ERRLOG("no memory\n"); 644 assert(0); 645 return -ENOMEM; 646 } 647 648 nbytes = accel_get_iovlen(src_iovs, iov_cnt); 649 650 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 651 652 accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 653 accel_task->d.iovs[0].iov_base = dst; 654 accel_task->d.iovs[0].iov_len = nbytes; 655 accel_task->d.iovcnt = 1; 656 accel_task->s.iovs = src_iovs; 657 accel_task->s.iovcnt = iov_cnt; 658 accel_task->nbytes = nbytes; 659 accel_task->crc_dst = crc_dst; 660 accel_task->seed = seed; 661 accel_task->op_code = SPDK_ACCEL_OPC_COPY_CRC32C; 662 accel_task->src_domain = NULL; 663 accel_task->dst_domain = NULL; 664 665 return accel_submit_task(accel_ch, accel_task); 666 } 667 668 int 669 spdk_accel_get_compress_level_range(enum spdk_accel_comp_algo comp_algo, 670 uint32_t *min_level, uint32_t *max_level) 671 { 672 struct spdk_accel_module_if *module = g_modules_opc[SPDK_ACCEL_OPC_COMPRESS].module; 673 674 if (module->get_compress_level_range == NULL) { 675 SPDK_ERRLOG("Module %s doesn't implement callback fn get_compress_level_range.\n", module->name); 676 return -ENOTSUP; 677 } 678 679 return module->get_compress_level_range(comp_algo, min_level, max_level); 680 } 681 682 static int 683 _accel_check_comp_algo(enum spdk_accel_comp_algo comp_algo) 684 { 685 struct spdk_accel_module_if *module = g_modules_opc[SPDK_ACCEL_OPC_COMPRESS].module; 686 687 if (!module->compress_supports_algo || !module->compress_supports_algo(comp_algo)) { 688 SPDK_ERRLOG("Module %s doesn't support compression algo %d\n", module->name, comp_algo); 689 return -ENOTSUP; 690 } 691 692 return 0; 693 } 694 695 int 696 spdk_accel_submit_compress_ext(struct spdk_io_channel *ch, void *dst, uint64_t nbytes, 697 struct iovec *src_iovs, size_t src_iovcnt, 698 enum spdk_accel_comp_algo comp_algo, uint32_t comp_level, 699 uint32_t *output_size, spdk_accel_completion_cb cb_fn, void *cb_arg) 700 { 701 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 702 struct spdk_accel_task *accel_task; 703 int rc; 704 705 rc = _accel_check_comp_algo(comp_algo); 706 if (spdk_unlikely(rc != 0)) { 707 return rc; 708 } 709 710 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 711 if (spdk_unlikely(accel_task == NULL)) { 712 return -ENOMEM; 713 } 714 715 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 716 717 accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 718 accel_task->d.iovs[0].iov_base = dst; 719 accel_task->d.iovs[0].iov_len = nbytes; 720 accel_task->d.iovcnt = 1; 721 accel_task->output_size = output_size; 722 accel_task->s.iovs = src_iovs; 723 accel_task->s.iovcnt = src_iovcnt; 724 accel_task->nbytes = nbytes; 725 accel_task->op_code = SPDK_ACCEL_OPC_COMPRESS; 726 accel_task->src_domain = NULL; 727 accel_task->dst_domain = NULL; 728 accel_task->comp.algo = comp_algo; 729 accel_task->comp.level = comp_level; 730 731 return accel_submit_task(accel_ch, accel_task); 732 } 733 734 int 735 spdk_accel_submit_decompress_ext(struct spdk_io_channel *ch, struct iovec *dst_iovs, 736 size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt, 737 enum spdk_accel_comp_algo decomp_algo, uint32_t *output_size, 738 spdk_accel_completion_cb cb_fn, void *cb_arg) 739 { 740 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 741 struct spdk_accel_task *accel_task; 742 int rc; 743 744 rc = _accel_check_comp_algo(decomp_algo); 745 if (spdk_unlikely(rc != 0)) { 746 return rc; 747 } 748 749 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 750 if (spdk_unlikely(accel_task == NULL)) { 751 return -ENOMEM; 752 } 753 754 accel_task->output_size = output_size; 755 accel_task->s.iovs = src_iovs; 756 accel_task->s.iovcnt = src_iovcnt; 757 accel_task->d.iovs = dst_iovs; 758 accel_task->d.iovcnt = dst_iovcnt; 759 accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 760 accel_task->op_code = SPDK_ACCEL_OPC_DECOMPRESS; 761 accel_task->src_domain = NULL; 762 accel_task->dst_domain = NULL; 763 accel_task->comp.algo = decomp_algo; 764 765 return accel_submit_task(accel_ch, accel_task); 766 } 767 768 int 769 spdk_accel_submit_compress(struct spdk_io_channel *ch, void *dst, uint64_t nbytes, 770 struct iovec *src_iovs, size_t src_iovcnt, uint32_t *output_size, 771 spdk_accel_completion_cb cb_fn, void *cb_arg) 772 { 773 return spdk_accel_submit_compress_ext(ch, dst, nbytes, src_iovs, src_iovcnt, 774 SPDK_ACCEL_COMP_ALGO_DEFLATE, 1, output_size, cb_fn, cb_arg); 775 } 776 777 int 778 spdk_accel_submit_decompress(struct spdk_io_channel *ch, struct iovec *dst_iovs, 779 size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt, 780 uint32_t *output_size, spdk_accel_completion_cb cb_fn, 781 void *cb_arg) 782 { 783 return spdk_accel_submit_decompress_ext(ch, dst_iovs, dst_iovcnt, src_iovs, src_iovcnt, 784 SPDK_ACCEL_COMP_ALGO_DEFLATE, output_size, cb_fn, cb_arg); 785 } 786 787 int 788 spdk_accel_submit_encrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key, 789 struct iovec *dst_iovs, uint32_t dst_iovcnt, 790 struct iovec *src_iovs, uint32_t src_iovcnt, 791 uint64_t iv, uint32_t block_size, 792 spdk_accel_completion_cb cb_fn, void *cb_arg) 793 { 794 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 795 struct spdk_accel_task *accel_task; 796 797 if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) { 798 return -EINVAL; 799 } 800 801 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 802 if (spdk_unlikely(accel_task == NULL)) { 803 return -ENOMEM; 804 } 805 806 accel_task->crypto_key = key; 807 accel_task->s.iovs = src_iovs; 808 accel_task->s.iovcnt = src_iovcnt; 809 accel_task->d.iovs = dst_iovs; 810 accel_task->d.iovcnt = dst_iovcnt; 811 accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 812 accel_task->iv = iv; 813 accel_task->block_size = block_size; 814 accel_task->op_code = SPDK_ACCEL_OPC_ENCRYPT; 815 accel_task->src_domain = NULL; 816 accel_task->dst_domain = NULL; 817 818 return accel_submit_task(accel_ch, accel_task); 819 } 820 821 int 822 spdk_accel_submit_decrypt(struct spdk_io_channel *ch, struct spdk_accel_crypto_key *key, 823 struct iovec *dst_iovs, uint32_t dst_iovcnt, 824 struct iovec *src_iovs, uint32_t src_iovcnt, 825 uint64_t iv, uint32_t block_size, 826 spdk_accel_completion_cb cb_fn, void *cb_arg) 827 { 828 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 829 struct spdk_accel_task *accel_task; 830 831 if (spdk_unlikely(!dst_iovs || !dst_iovcnt || !src_iovs || !src_iovcnt || !key || !block_size)) { 832 return -EINVAL; 833 } 834 835 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 836 if (spdk_unlikely(accel_task == NULL)) { 837 return -ENOMEM; 838 } 839 840 accel_task->crypto_key = key; 841 accel_task->s.iovs = src_iovs; 842 accel_task->s.iovcnt = src_iovcnt; 843 accel_task->d.iovs = dst_iovs; 844 accel_task->d.iovcnt = dst_iovcnt; 845 accel_task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 846 accel_task->iv = iv; 847 accel_task->block_size = block_size; 848 accel_task->op_code = SPDK_ACCEL_OPC_DECRYPT; 849 accel_task->src_domain = NULL; 850 accel_task->dst_domain = NULL; 851 852 return accel_submit_task(accel_ch, accel_task); 853 } 854 855 int 856 spdk_accel_submit_xor(struct spdk_io_channel *ch, void *dst, void **sources, uint32_t nsrcs, 857 uint64_t nbytes, spdk_accel_completion_cb cb_fn, void *cb_arg) 858 { 859 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 860 struct spdk_accel_task *accel_task; 861 862 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 863 if (spdk_unlikely(accel_task == NULL)) { 864 return -ENOMEM; 865 } 866 867 ACCEL_TASK_ALLOC_AUX_BUF(accel_task); 868 869 accel_task->d.iovs = &accel_task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 870 accel_task->nsrcs.srcs = sources; 871 accel_task->nsrcs.cnt = nsrcs; 872 accel_task->d.iovs[0].iov_base = dst; 873 accel_task->d.iovs[0].iov_len = nbytes; 874 accel_task->d.iovcnt = 1; 875 accel_task->nbytes = nbytes; 876 accel_task->op_code = SPDK_ACCEL_OPC_XOR; 877 accel_task->src_domain = NULL; 878 accel_task->dst_domain = NULL; 879 880 return accel_submit_task(accel_ch, accel_task); 881 } 882 883 int 884 spdk_accel_submit_dif_verify(struct spdk_io_channel *ch, 885 struct iovec *iovs, size_t iovcnt, uint32_t num_blocks, 886 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err, 887 spdk_accel_completion_cb cb_fn, void *cb_arg) 888 { 889 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 890 struct spdk_accel_task *accel_task; 891 892 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 893 if (accel_task == NULL) { 894 return -ENOMEM; 895 } 896 897 accel_task->s.iovs = iovs; 898 accel_task->s.iovcnt = iovcnt; 899 accel_task->dif.ctx = ctx; 900 accel_task->dif.err = err; 901 accel_task->dif.num_blocks = num_blocks; 902 accel_task->nbytes = num_blocks * ctx->block_size; 903 accel_task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY; 904 accel_task->src_domain = NULL; 905 accel_task->dst_domain = NULL; 906 907 return accel_submit_task(accel_ch, accel_task); 908 } 909 910 int 911 spdk_accel_submit_dif_generate(struct spdk_io_channel *ch, 912 struct iovec *iovs, size_t iovcnt, uint32_t num_blocks, 913 const struct spdk_dif_ctx *ctx, 914 spdk_accel_completion_cb cb_fn, void *cb_arg) 915 { 916 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 917 struct spdk_accel_task *accel_task; 918 919 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 920 if (accel_task == NULL) { 921 return -ENOMEM; 922 } 923 924 accel_task->s.iovs = iovs; 925 accel_task->s.iovcnt = iovcnt; 926 accel_task->dif.ctx = ctx; 927 accel_task->dif.num_blocks = num_blocks; 928 accel_task->nbytes = num_blocks * ctx->block_size; 929 accel_task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE; 930 accel_task->src_domain = NULL; 931 accel_task->dst_domain = NULL; 932 933 return accel_submit_task(accel_ch, accel_task); 934 } 935 936 int 937 spdk_accel_submit_dif_generate_copy(struct spdk_io_channel *ch, struct iovec *dst_iovs, 938 size_t dst_iovcnt, struct iovec *src_iovs, size_t src_iovcnt, 939 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 940 spdk_accel_completion_cb cb_fn, void *cb_arg) 941 { 942 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 943 struct spdk_accel_task *accel_task; 944 945 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 946 if (accel_task == NULL) { 947 return -ENOMEM; 948 } 949 950 accel_task->s.iovs = src_iovs; 951 accel_task->s.iovcnt = src_iovcnt; 952 accel_task->d.iovs = dst_iovs; 953 accel_task->d.iovcnt = dst_iovcnt; 954 accel_task->dif.ctx = ctx; 955 accel_task->dif.num_blocks = num_blocks; 956 accel_task->nbytes = num_blocks * ctx->block_size; 957 accel_task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE_COPY; 958 accel_task->src_domain = NULL; 959 accel_task->dst_domain = NULL; 960 961 return accel_submit_task(accel_ch, accel_task); 962 } 963 964 int 965 spdk_accel_submit_dif_verify_copy(struct spdk_io_channel *ch, 966 struct iovec *dst_iovs, size_t dst_iovcnt, 967 struct iovec *src_iovs, size_t src_iovcnt, uint32_t num_blocks, 968 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err, 969 spdk_accel_completion_cb cb_fn, void *cb_arg) 970 { 971 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 972 struct spdk_accel_task *accel_task; 973 974 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 975 if (accel_task == NULL) { 976 return -ENOMEM; 977 } 978 979 accel_task->s.iovs = src_iovs; 980 accel_task->s.iovcnt = src_iovcnt; 981 accel_task->d.iovs = dst_iovs; 982 accel_task->d.iovcnt = dst_iovcnt; 983 accel_task->dif.ctx = ctx; 984 accel_task->dif.err = err; 985 accel_task->dif.num_blocks = num_blocks; 986 accel_task->nbytes = num_blocks * ctx->block_size; 987 accel_task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY_COPY; 988 accel_task->src_domain = NULL; 989 accel_task->dst_domain = NULL; 990 991 return accel_submit_task(accel_ch, accel_task); 992 } 993 994 int 995 spdk_accel_submit_dix_generate(struct spdk_io_channel *ch, struct iovec *iovs, 996 size_t iovcnt, struct iovec *md_iov, uint32_t num_blocks, 997 const struct spdk_dif_ctx *ctx, spdk_accel_completion_cb cb_fn, 998 void *cb_arg) 999 { 1000 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1001 struct spdk_accel_task *accel_task; 1002 1003 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 1004 if (accel_task == NULL) { 1005 return -ENOMEM; 1006 } 1007 1008 accel_task->s.iovs = iovs; 1009 accel_task->s.iovcnt = iovcnt; 1010 accel_task->d.iovs = md_iov; 1011 accel_task->d.iovcnt = 1; 1012 accel_task->dif.ctx = ctx; 1013 accel_task->dif.num_blocks = num_blocks; 1014 accel_task->nbytes = num_blocks * ctx->block_size; 1015 accel_task->op_code = SPDK_ACCEL_OPC_DIX_GENERATE; 1016 accel_task->src_domain = NULL; 1017 accel_task->dst_domain = NULL; 1018 1019 return accel_submit_task(accel_ch, accel_task); 1020 } 1021 1022 int 1023 spdk_accel_submit_dix_verify(struct spdk_io_channel *ch, struct iovec *iovs, 1024 size_t iovcnt, struct iovec *md_iov, uint32_t num_blocks, 1025 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err, 1026 spdk_accel_completion_cb cb_fn, void *cb_arg) 1027 { 1028 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1029 struct spdk_accel_task *accel_task; 1030 1031 accel_task = _get_task(accel_ch, cb_fn, cb_arg); 1032 if (accel_task == NULL) { 1033 return -ENOMEM; 1034 } 1035 1036 accel_task->s.iovs = iovs; 1037 accel_task->s.iovcnt = iovcnt; 1038 accel_task->d.iovs = md_iov; 1039 accel_task->d.iovcnt = 1; 1040 accel_task->dif.ctx = ctx; 1041 accel_task->dif.err = err; 1042 accel_task->dif.num_blocks = num_blocks; 1043 accel_task->nbytes = num_blocks * ctx->block_size; 1044 accel_task->op_code = SPDK_ACCEL_OPC_DIX_VERIFY; 1045 accel_task->src_domain = NULL; 1046 accel_task->dst_domain = NULL; 1047 1048 return accel_submit_task(accel_ch, accel_task); 1049 } 1050 1051 static inline struct accel_buffer * 1052 accel_get_buf(struct accel_io_channel *ch, uint64_t len) 1053 { 1054 struct accel_buffer *buf; 1055 1056 buf = SLIST_FIRST(&ch->buf_pool); 1057 if (spdk_unlikely(buf == NULL)) { 1058 accel_update_stats(ch, retry.bufdesc, 1); 1059 return NULL; 1060 } 1061 1062 SLIST_REMOVE_HEAD(&ch->buf_pool, link); 1063 buf->len = len; 1064 buf->buf = NULL; 1065 buf->seq = NULL; 1066 buf->cb_fn = NULL; 1067 1068 return buf; 1069 } 1070 1071 static inline void 1072 accel_put_buf(struct accel_io_channel *ch, struct accel_buffer *buf) 1073 { 1074 if (buf->buf != NULL) { 1075 spdk_iobuf_put(&ch->iobuf, buf->buf, buf->len); 1076 } 1077 1078 SLIST_INSERT_HEAD(&ch->buf_pool, buf, link); 1079 } 1080 1081 static inline struct spdk_accel_sequence * 1082 accel_sequence_get(struct accel_io_channel *ch) 1083 { 1084 struct spdk_accel_sequence *seq; 1085 1086 assert(g_opts.task_count >= ch->stats.task_outstanding); 1087 1088 /* Sequence cannot be allocated if number of available task objects cannot satisfy required limit. 1089 * This is to prevent potential dead lock when few requests are pending task resource and none can 1090 * advance the processing. This solution should work only if there is single async operation after 1091 * sequence obj obtained, so assume that is possible to happen with io buffer allocation now, if 1092 * there are more async operations then solution should be improved. */ 1093 if (spdk_unlikely(g_opts.task_count - ch->stats.task_outstanding < ACCEL_TASKS_IN_SEQUENCE_LIMIT)) { 1094 return NULL; 1095 } 1096 1097 seq = SLIST_FIRST(&ch->seq_pool); 1098 if (spdk_unlikely(seq == NULL)) { 1099 accel_update_stats(ch, retry.sequence, 1); 1100 return NULL; 1101 } 1102 1103 accel_update_stats(ch, sequence_outstanding, 1); 1104 SLIST_REMOVE_HEAD(&ch->seq_pool, link); 1105 1106 TAILQ_INIT(&seq->tasks); 1107 SLIST_INIT(&seq->bounce_bufs); 1108 1109 seq->ch = ch; 1110 seq->status = 0; 1111 seq->state = ACCEL_SEQUENCE_STATE_INIT; 1112 seq->in_process_sequence = false; 1113 1114 return seq; 1115 } 1116 1117 static inline void 1118 accel_sequence_put(struct spdk_accel_sequence *seq) 1119 { 1120 struct accel_io_channel *ch = seq->ch; 1121 struct accel_buffer *buf; 1122 1123 while (!SLIST_EMPTY(&seq->bounce_bufs)) { 1124 buf = SLIST_FIRST(&seq->bounce_bufs); 1125 SLIST_REMOVE_HEAD(&seq->bounce_bufs, link); 1126 accel_put_buf(seq->ch, buf); 1127 } 1128 1129 assert(TAILQ_EMPTY(&seq->tasks)); 1130 seq->ch = NULL; 1131 1132 SLIST_INSERT_HEAD(&ch->seq_pool, seq, link); 1133 accel_update_stats(ch, sequence_outstanding, -1); 1134 } 1135 1136 static void accel_sequence_task_cb(void *cb_arg, int status); 1137 1138 static inline struct spdk_accel_task * 1139 accel_sequence_get_task(struct accel_io_channel *ch, struct spdk_accel_sequence *seq, 1140 spdk_accel_step_cb cb_fn, void *cb_arg) 1141 { 1142 struct spdk_accel_task *task; 1143 1144 task = _get_task(ch, NULL, NULL); 1145 if (spdk_unlikely(task == NULL)) { 1146 return task; 1147 } 1148 1149 task->step_cb_fn = cb_fn; 1150 task->cb_arg = cb_arg; 1151 task->seq = seq; 1152 1153 return task; 1154 } 1155 1156 int 1157 spdk_accel_append_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1158 struct iovec *dst_iovs, uint32_t dst_iovcnt, 1159 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 1160 struct iovec *src_iovs, uint32_t src_iovcnt, 1161 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 1162 spdk_accel_step_cb cb_fn, void *cb_arg) 1163 { 1164 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1165 struct spdk_accel_task *task; 1166 struct spdk_accel_sequence *seq = *pseq; 1167 1168 if (seq == NULL) { 1169 seq = accel_sequence_get(accel_ch); 1170 if (spdk_unlikely(seq == NULL)) { 1171 return -ENOMEM; 1172 } 1173 } 1174 1175 assert(seq->ch == accel_ch); 1176 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1177 if (spdk_unlikely(task == NULL)) { 1178 if (*pseq == NULL) { 1179 accel_sequence_put(seq); 1180 } 1181 1182 return -ENOMEM; 1183 } 1184 1185 task->dst_domain = dst_domain; 1186 task->dst_domain_ctx = dst_domain_ctx; 1187 task->d.iovs = dst_iovs; 1188 task->d.iovcnt = dst_iovcnt; 1189 task->src_domain = src_domain; 1190 task->src_domain_ctx = src_domain_ctx; 1191 task->s.iovs = src_iovs; 1192 task->s.iovcnt = src_iovcnt; 1193 task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 1194 task->op_code = SPDK_ACCEL_OPC_COPY; 1195 1196 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1197 *pseq = seq; 1198 1199 return 0; 1200 } 1201 1202 int 1203 spdk_accel_append_fill(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1204 void *buf, uint64_t len, 1205 struct spdk_memory_domain *domain, void *domain_ctx, uint8_t pattern, 1206 spdk_accel_step_cb cb_fn, void *cb_arg) 1207 { 1208 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1209 struct spdk_accel_task *task; 1210 struct spdk_accel_sequence *seq = *pseq; 1211 1212 if (seq == NULL) { 1213 seq = accel_sequence_get(accel_ch); 1214 if (spdk_unlikely(seq == NULL)) { 1215 return -ENOMEM; 1216 } 1217 } 1218 1219 assert(seq->ch == accel_ch); 1220 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1221 if (spdk_unlikely(task == NULL)) { 1222 if (*pseq == NULL) { 1223 accel_sequence_put(seq); 1224 } 1225 1226 return -ENOMEM; 1227 } 1228 1229 memset(&task->fill_pattern, pattern, sizeof(uint64_t)); 1230 1231 task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool); 1232 if (spdk_unlikely(!task->aux)) { 1233 SPDK_ERRLOG("Fatal problem, aux data was not allocated\n"); 1234 if (*pseq == NULL) { 1235 accel_sequence_put((seq)); 1236 } 1237 1238 task->seq = NULL; 1239 _put_task(task->accel_ch, task); 1240 assert(0); 1241 return -ENOMEM; 1242 } 1243 SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link); 1244 task->has_aux = true; 1245 1246 task->d.iovs = &task->aux->iovs[SPDK_ACCEL_AUX_IOV_DST]; 1247 task->d.iovs[0].iov_base = buf; 1248 task->d.iovs[0].iov_len = len; 1249 task->d.iovcnt = 1; 1250 task->nbytes = len; 1251 task->src_domain = NULL; 1252 task->dst_domain = domain; 1253 task->dst_domain_ctx = domain_ctx; 1254 task->op_code = SPDK_ACCEL_OPC_FILL; 1255 1256 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1257 *pseq = seq; 1258 1259 return 0; 1260 } 1261 1262 int 1263 spdk_accel_append_decompress(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1264 struct iovec *dst_iovs, size_t dst_iovcnt, 1265 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 1266 struct iovec *src_iovs, size_t src_iovcnt, 1267 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 1268 spdk_accel_step_cb cb_fn, void *cb_arg) 1269 { 1270 return spdk_accel_append_decompress_ext(pseq, ch, dst_iovs, dst_iovcnt, dst_domain, 1271 dst_domain_ctx, src_iovs, src_iovcnt, src_domain, 1272 src_domain_ctx, SPDK_ACCEL_COMP_ALGO_DEFLATE, 1273 cb_fn, cb_arg); 1274 } 1275 1276 int 1277 spdk_accel_append_decompress_ext(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1278 struct iovec *dst_iovs, size_t dst_iovcnt, 1279 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 1280 struct iovec *src_iovs, size_t src_iovcnt, 1281 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 1282 enum spdk_accel_comp_algo decomp_algo, 1283 spdk_accel_step_cb cb_fn, void *cb_arg) 1284 { 1285 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1286 struct spdk_accel_task *task; 1287 struct spdk_accel_sequence *seq = *pseq; 1288 int rc; 1289 1290 rc = _accel_check_comp_algo(decomp_algo); 1291 if (spdk_unlikely(rc != 0)) { 1292 return rc; 1293 } 1294 1295 if (seq == NULL) { 1296 seq = accel_sequence_get(accel_ch); 1297 if (spdk_unlikely(seq == NULL)) { 1298 return -ENOMEM; 1299 } 1300 } 1301 1302 assert(seq->ch == accel_ch); 1303 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1304 if (spdk_unlikely(task == NULL)) { 1305 if (*pseq == NULL) { 1306 accel_sequence_put(seq); 1307 } 1308 1309 return -ENOMEM; 1310 } 1311 1312 /* TODO: support output_size for chaining */ 1313 task->output_size = NULL; 1314 task->dst_domain = dst_domain; 1315 task->dst_domain_ctx = dst_domain_ctx; 1316 task->d.iovs = dst_iovs; 1317 task->d.iovcnt = dst_iovcnt; 1318 task->src_domain = src_domain; 1319 task->src_domain_ctx = src_domain_ctx; 1320 task->s.iovs = src_iovs; 1321 task->s.iovcnt = src_iovcnt; 1322 task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 1323 task->op_code = SPDK_ACCEL_OPC_DECOMPRESS; 1324 task->comp.algo = decomp_algo; 1325 1326 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1327 *pseq = seq; 1328 1329 return 0; 1330 } 1331 1332 int 1333 spdk_accel_append_encrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1334 struct spdk_accel_crypto_key *key, 1335 struct iovec *dst_iovs, uint32_t dst_iovcnt, 1336 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 1337 struct iovec *src_iovs, uint32_t src_iovcnt, 1338 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 1339 uint64_t iv, uint32_t block_size, 1340 spdk_accel_step_cb cb_fn, void *cb_arg) 1341 { 1342 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1343 struct spdk_accel_task *task; 1344 struct spdk_accel_sequence *seq = *pseq; 1345 1346 assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size); 1347 1348 if (seq == NULL) { 1349 seq = accel_sequence_get(accel_ch); 1350 if (spdk_unlikely(seq == NULL)) { 1351 return -ENOMEM; 1352 } 1353 } 1354 1355 assert(seq->ch == accel_ch); 1356 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1357 if (spdk_unlikely(task == NULL)) { 1358 if (*pseq == NULL) { 1359 accel_sequence_put(seq); 1360 } 1361 1362 return -ENOMEM; 1363 } 1364 1365 task->crypto_key = key; 1366 task->src_domain = src_domain; 1367 task->src_domain_ctx = src_domain_ctx; 1368 task->s.iovs = src_iovs; 1369 task->s.iovcnt = src_iovcnt; 1370 task->dst_domain = dst_domain; 1371 task->dst_domain_ctx = dst_domain_ctx; 1372 task->d.iovs = dst_iovs; 1373 task->d.iovcnt = dst_iovcnt; 1374 task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 1375 task->iv = iv; 1376 task->block_size = block_size; 1377 task->op_code = SPDK_ACCEL_OPC_ENCRYPT; 1378 1379 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1380 *pseq = seq; 1381 1382 return 0; 1383 } 1384 1385 int 1386 spdk_accel_append_decrypt(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1387 struct spdk_accel_crypto_key *key, 1388 struct iovec *dst_iovs, uint32_t dst_iovcnt, 1389 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 1390 struct iovec *src_iovs, uint32_t src_iovcnt, 1391 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 1392 uint64_t iv, uint32_t block_size, 1393 spdk_accel_step_cb cb_fn, void *cb_arg) 1394 { 1395 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1396 struct spdk_accel_task *task; 1397 struct spdk_accel_sequence *seq = *pseq; 1398 1399 assert(dst_iovs && dst_iovcnt && src_iovs && src_iovcnt && key && block_size); 1400 1401 if (seq == NULL) { 1402 seq = accel_sequence_get(accel_ch); 1403 if (spdk_unlikely(seq == NULL)) { 1404 return -ENOMEM; 1405 } 1406 } 1407 1408 assert(seq->ch == accel_ch); 1409 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1410 if (spdk_unlikely(task == NULL)) { 1411 if (*pseq == NULL) { 1412 accel_sequence_put(seq); 1413 } 1414 1415 return -ENOMEM; 1416 } 1417 1418 task->crypto_key = key; 1419 task->src_domain = src_domain; 1420 task->src_domain_ctx = src_domain_ctx; 1421 task->s.iovs = src_iovs; 1422 task->s.iovcnt = src_iovcnt; 1423 task->dst_domain = dst_domain; 1424 task->dst_domain_ctx = dst_domain_ctx; 1425 task->d.iovs = dst_iovs; 1426 task->d.iovcnt = dst_iovcnt; 1427 task->nbytes = accel_get_iovlen(src_iovs, src_iovcnt); 1428 task->iv = iv; 1429 task->block_size = block_size; 1430 task->op_code = SPDK_ACCEL_OPC_DECRYPT; 1431 1432 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1433 *pseq = seq; 1434 1435 return 0; 1436 } 1437 1438 int 1439 spdk_accel_append_crc32c(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1440 uint32_t *dst, struct iovec *iovs, uint32_t iovcnt, 1441 struct spdk_memory_domain *domain, void *domain_ctx, 1442 uint32_t seed, spdk_accel_step_cb cb_fn, void *cb_arg) 1443 { 1444 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1445 struct spdk_accel_task *task; 1446 struct spdk_accel_sequence *seq = *pseq; 1447 1448 if (seq == NULL) { 1449 seq = accel_sequence_get(accel_ch); 1450 if (spdk_unlikely(seq == NULL)) { 1451 return -ENOMEM; 1452 } 1453 } 1454 1455 assert(seq->ch == accel_ch); 1456 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1457 if (spdk_unlikely(task == NULL)) { 1458 if (*pseq == NULL) { 1459 accel_sequence_put(seq); 1460 } 1461 1462 return -ENOMEM; 1463 } 1464 1465 task->s.iovs = iovs; 1466 task->s.iovcnt = iovcnt; 1467 task->src_domain = domain; 1468 task->src_domain_ctx = domain_ctx; 1469 task->nbytes = accel_get_iovlen(iovs, iovcnt); 1470 task->crc_dst = dst; 1471 task->seed = seed; 1472 task->op_code = SPDK_ACCEL_OPC_CRC32C; 1473 task->dst_domain = NULL; 1474 1475 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1476 *pseq = seq; 1477 1478 return 0; 1479 } 1480 1481 int 1482 spdk_accel_append_dif_verify(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1483 struct iovec *iovs, size_t iovcnt, 1484 struct spdk_memory_domain *domain, void *domain_ctx, 1485 uint32_t num_blocks, 1486 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err, 1487 spdk_accel_step_cb cb_fn, void *cb_arg) 1488 { 1489 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1490 struct spdk_accel_task *task; 1491 struct spdk_accel_sequence *seq = *pseq; 1492 1493 if (seq == NULL) { 1494 seq = accel_sequence_get(accel_ch); 1495 if (spdk_unlikely(seq == NULL)) { 1496 return -ENOMEM; 1497 } 1498 } 1499 1500 assert(seq->ch == accel_ch); 1501 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1502 if (spdk_unlikely(task == NULL)) { 1503 if (*pseq == NULL) { 1504 accel_sequence_put(seq); 1505 } 1506 1507 return -ENOMEM; 1508 } 1509 1510 task->s.iovs = iovs; 1511 task->s.iovcnt = iovcnt; 1512 task->src_domain = domain; 1513 task->src_domain_ctx = domain_ctx; 1514 task->dst_domain = NULL; 1515 task->dif.ctx = ctx; 1516 task->dif.err = err; 1517 task->dif.num_blocks = num_blocks; 1518 task->nbytes = num_blocks * ctx->block_size; 1519 task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY; 1520 1521 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1522 *pseq = seq; 1523 1524 return 0; 1525 } 1526 1527 int 1528 spdk_accel_append_dif_verify_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1529 struct iovec *dst_iovs, size_t dst_iovcnt, 1530 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 1531 struct iovec *src_iovs, size_t src_iovcnt, 1532 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 1533 uint32_t num_blocks, 1534 const struct spdk_dif_ctx *ctx, struct spdk_dif_error *err, 1535 spdk_accel_step_cb cb_fn, void *cb_arg) 1536 { 1537 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1538 struct spdk_accel_task *task; 1539 struct spdk_accel_sequence *seq = *pseq; 1540 1541 if (seq == NULL) { 1542 seq = accel_sequence_get(accel_ch); 1543 if (spdk_unlikely(seq == NULL)) { 1544 return -ENOMEM; 1545 } 1546 } 1547 1548 assert(seq->ch == accel_ch); 1549 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1550 if (spdk_unlikely(task == NULL)) { 1551 if (*pseq == NULL) { 1552 accel_sequence_put(seq); 1553 } 1554 1555 return -ENOMEM; 1556 } 1557 1558 task->dst_domain = dst_domain; 1559 task->dst_domain_ctx = dst_domain_ctx; 1560 task->d.iovs = dst_iovs; 1561 task->d.iovcnt = dst_iovcnt; 1562 task->src_domain = src_domain; 1563 task->src_domain_ctx = src_domain_ctx; 1564 task->s.iovs = src_iovs; 1565 task->s.iovcnt = src_iovcnt; 1566 task->dif.ctx = ctx; 1567 task->dif.err = err; 1568 task->dif.num_blocks = num_blocks; 1569 task->nbytes = num_blocks * ctx->block_size; 1570 task->op_code = SPDK_ACCEL_OPC_DIF_VERIFY_COPY; 1571 1572 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1573 *pseq = seq; 1574 1575 return 0; 1576 } 1577 1578 int 1579 spdk_accel_append_dif_generate(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1580 struct iovec *iovs, size_t iovcnt, 1581 struct spdk_memory_domain *domain, void *domain_ctx, 1582 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1583 spdk_accel_step_cb cb_fn, void *cb_arg) 1584 { 1585 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1586 struct spdk_accel_task *task; 1587 struct spdk_accel_sequence *seq = *pseq; 1588 1589 if (seq == NULL) { 1590 seq = accel_sequence_get(accel_ch); 1591 if (spdk_unlikely(seq == NULL)) { 1592 return -ENOMEM; 1593 } 1594 } 1595 1596 assert(seq->ch == accel_ch); 1597 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1598 if (spdk_unlikely(task == NULL)) { 1599 if (*pseq == NULL) { 1600 accel_sequence_put(seq); 1601 } 1602 1603 return -ENOMEM; 1604 } 1605 1606 task->s.iovs = iovs; 1607 task->s.iovcnt = iovcnt; 1608 task->src_domain = domain; 1609 task->src_domain_ctx = domain_ctx; 1610 task->dst_domain = NULL; 1611 task->dif.ctx = ctx; 1612 task->dif.num_blocks = num_blocks; 1613 task->nbytes = num_blocks * ctx->block_size; 1614 task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE; 1615 1616 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1617 *pseq = seq; 1618 1619 return 0; 1620 } 1621 1622 int 1623 spdk_accel_append_dif_generate_copy(struct spdk_accel_sequence **pseq, struct spdk_io_channel *ch, 1624 struct iovec *dst_iovs, size_t dst_iovcnt, 1625 struct spdk_memory_domain *dst_domain, void *dst_domain_ctx, 1626 struct iovec *src_iovs, size_t src_iovcnt, 1627 struct spdk_memory_domain *src_domain, void *src_domain_ctx, 1628 uint32_t num_blocks, const struct spdk_dif_ctx *ctx, 1629 spdk_accel_step_cb cb_fn, void *cb_arg) 1630 { 1631 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1632 struct spdk_accel_task *task; 1633 struct spdk_accel_sequence *seq = *pseq; 1634 1635 if (seq == NULL) { 1636 seq = accel_sequence_get(accel_ch); 1637 if (spdk_unlikely(seq == NULL)) { 1638 return -ENOMEM; 1639 } 1640 } 1641 1642 assert(seq->ch == accel_ch); 1643 task = accel_sequence_get_task(accel_ch, seq, cb_fn, cb_arg); 1644 if (spdk_unlikely(task == NULL)) { 1645 if (*pseq == NULL) { 1646 accel_sequence_put(seq); 1647 } 1648 1649 return -ENOMEM; 1650 } 1651 1652 task->dst_domain = dst_domain; 1653 task->dst_domain_ctx = dst_domain_ctx; 1654 task->d.iovs = dst_iovs; 1655 task->d.iovcnt = dst_iovcnt; 1656 task->src_domain = src_domain; 1657 task->src_domain_ctx = src_domain_ctx; 1658 task->s.iovs = src_iovs; 1659 task->s.iovcnt = src_iovcnt; 1660 task->nbytes = num_blocks * ctx->block_size; 1661 task->op_code = SPDK_ACCEL_OPC_DIF_GENERATE_COPY; 1662 1663 TAILQ_INSERT_TAIL(&seq->tasks, task, seq_link); 1664 *pseq = seq; 1665 1666 return 0; 1667 } 1668 1669 int 1670 spdk_accel_get_buf(struct spdk_io_channel *ch, uint64_t len, void **buf, 1671 struct spdk_memory_domain **domain, void **domain_ctx) 1672 { 1673 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1674 struct accel_buffer *accel_buf; 1675 1676 accel_buf = accel_get_buf(accel_ch, len); 1677 if (spdk_unlikely(accel_buf == NULL)) { 1678 return -ENOMEM; 1679 } 1680 1681 accel_buf->ch = accel_ch; 1682 1683 /* We always return the same pointer and identify the buffers through domain_ctx */ 1684 *buf = ACCEL_BUFFER_BASE; 1685 *domain_ctx = accel_buf; 1686 *domain = g_accel_domain; 1687 1688 return 0; 1689 } 1690 1691 void 1692 spdk_accel_put_buf(struct spdk_io_channel *ch, void *buf, 1693 struct spdk_memory_domain *domain, void *domain_ctx) 1694 { 1695 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 1696 struct accel_buffer *accel_buf = domain_ctx; 1697 1698 assert(domain == g_accel_domain); 1699 assert(buf == ACCEL_BUFFER_BASE); 1700 1701 accel_put_buf(accel_ch, accel_buf); 1702 } 1703 1704 static void 1705 accel_sequence_complete_task(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1706 { 1707 struct accel_io_channel *ch = seq->ch; 1708 spdk_accel_step_cb cb_fn; 1709 void *cb_arg; 1710 1711 TAILQ_REMOVE(&seq->tasks, task, seq_link); 1712 cb_fn = task->step_cb_fn; 1713 cb_arg = task->cb_arg; 1714 task->seq = NULL; 1715 if (task->has_aux) { 1716 SLIST_INSERT_HEAD(&ch->task_aux_data_pool, task->aux, link); 1717 task->aux = NULL; 1718 task->has_aux = false; 1719 } 1720 1721 _put_task(ch, task); 1722 1723 if (cb_fn != NULL) { 1724 cb_fn(cb_arg); 1725 } 1726 } 1727 1728 static void 1729 accel_sequence_complete_tasks(struct spdk_accel_sequence *seq) 1730 { 1731 struct spdk_accel_task *task; 1732 1733 while (!TAILQ_EMPTY(&seq->tasks)) { 1734 task = TAILQ_FIRST(&seq->tasks); 1735 accel_sequence_complete_task(seq, task); 1736 } 1737 } 1738 1739 static void 1740 accel_sequence_complete(struct spdk_accel_sequence *seq) 1741 { 1742 spdk_accel_completion_cb cb_fn = seq->cb_fn; 1743 void *cb_arg = seq->cb_arg; 1744 int status = seq->status; 1745 1746 SPDK_DEBUGLOG(accel, "Completed sequence: %p with status: %d\n", seq, status); 1747 1748 accel_update_stats(seq->ch, sequence_executed, 1); 1749 if (spdk_unlikely(status != 0)) { 1750 accel_update_stats(seq->ch, sequence_failed, 1); 1751 } 1752 1753 /* First notify all users that appended operations to this sequence */ 1754 accel_sequence_complete_tasks(seq); 1755 accel_sequence_put(seq); 1756 1757 /* Then notify the user that finished the sequence */ 1758 cb_fn(cb_arg, status); 1759 } 1760 1761 static void 1762 accel_update_virt_iov(struct iovec *diov, struct iovec *siov, struct accel_buffer *accel_buf) 1763 { 1764 uintptr_t offset; 1765 1766 offset = (uintptr_t)siov->iov_base & ACCEL_BUFFER_OFFSET_MASK; 1767 assert(offset < accel_buf->len); 1768 1769 diov->iov_base = (char *)accel_buf->buf + offset; 1770 diov->iov_len = siov->iov_len; 1771 } 1772 1773 static void 1774 accel_sequence_set_virtbuf(struct spdk_accel_sequence *seq, struct accel_buffer *buf) 1775 { 1776 struct spdk_accel_task *task; 1777 struct iovec *iov; 1778 1779 /* Now that we've allocated the actual data buffer for this accel_buffer, update all tasks 1780 * in a sequence that were using it. 1781 */ 1782 TAILQ_FOREACH(task, &seq->tasks, seq_link) { 1783 if (task->src_domain == g_accel_domain && task->src_domain_ctx == buf) { 1784 if (!task->has_aux) { 1785 task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool); 1786 assert(task->aux && "Can't allocate aux data structure"); 1787 task->has_aux = true; 1788 SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link); 1789 } 1790 1791 iov = &task->aux->iovs[SPDK_ACCEL_AXU_IOV_VIRT_SRC]; 1792 assert(task->s.iovcnt == 1); 1793 accel_update_virt_iov(iov, &task->s.iovs[0], buf); 1794 task->src_domain = NULL; 1795 task->s.iovs = iov; 1796 } 1797 if (task->dst_domain == g_accel_domain && task->dst_domain_ctx == buf) { 1798 if (!task->has_aux) { 1799 task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool); 1800 assert(task->aux && "Can't allocate aux data structure"); 1801 task->has_aux = true; 1802 SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link); 1803 } 1804 1805 iov = &task->aux->iovs[SPDK_ACCEL_AXU_IOV_VIRT_DST]; 1806 assert(task->d.iovcnt == 1); 1807 accel_update_virt_iov(iov, &task->d.iovs[0], buf); 1808 task->dst_domain = NULL; 1809 task->d.iovs = iov; 1810 } 1811 } 1812 } 1813 1814 static void accel_process_sequence(struct spdk_accel_sequence *seq); 1815 1816 static void 1817 accel_iobuf_get_virtbuf_cb(struct spdk_iobuf_entry *entry, void *buf) 1818 { 1819 struct accel_buffer *accel_buf; 1820 1821 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1822 1823 assert(accel_buf->seq != NULL); 1824 assert(accel_buf->buf == NULL); 1825 accel_buf->buf = buf; 1826 1827 assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF); 1828 accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF); 1829 accel_sequence_set_virtbuf(accel_buf->seq, accel_buf); 1830 accel_process_sequence(accel_buf->seq); 1831 } 1832 1833 static bool 1834 accel_sequence_alloc_buf(struct spdk_accel_sequence *seq, struct accel_buffer *buf, 1835 spdk_iobuf_get_cb cb_fn) 1836 { 1837 struct accel_io_channel *ch = seq->ch; 1838 1839 assert(buf->seq == NULL); 1840 1841 buf->seq = seq; 1842 1843 /* Buffer might be already allocated by memory domain translation. */ 1844 if (buf->buf) { 1845 return true; 1846 } 1847 1848 buf->buf = spdk_iobuf_get(&ch->iobuf, buf->len, &buf->iobuf, cb_fn); 1849 if (spdk_unlikely(buf->buf == NULL)) { 1850 accel_update_stats(ch, retry.iobuf, 1); 1851 return false; 1852 } 1853 1854 return true; 1855 } 1856 1857 static bool 1858 accel_sequence_check_virtbuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1859 { 1860 /* If a task doesn't have dst/src (e.g. fill, crc32), its dst/src domain should be set to 1861 * NULL */ 1862 if (task->src_domain == g_accel_domain) { 1863 if (!accel_sequence_alloc_buf(seq, task->src_domain_ctx, 1864 accel_iobuf_get_virtbuf_cb)) { 1865 return false; 1866 } 1867 1868 accel_sequence_set_virtbuf(seq, task->src_domain_ctx); 1869 } 1870 1871 if (task->dst_domain == g_accel_domain) { 1872 if (!accel_sequence_alloc_buf(seq, task->dst_domain_ctx, 1873 accel_iobuf_get_virtbuf_cb)) { 1874 return false; 1875 } 1876 1877 accel_sequence_set_virtbuf(seq, task->dst_domain_ctx); 1878 } 1879 1880 return true; 1881 } 1882 1883 static void 1884 accel_sequence_get_buf_cb(struct spdk_iobuf_entry *entry, void *buf) 1885 { 1886 struct accel_buffer *accel_buf; 1887 1888 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1889 1890 assert(accel_buf->seq != NULL); 1891 assert(accel_buf->buf == NULL); 1892 accel_buf->buf = buf; 1893 1894 accel_sequence_set_virtbuf(accel_buf->seq, accel_buf); 1895 accel_buf->cb_fn(accel_buf->seq, accel_buf->cb_ctx); 1896 } 1897 1898 bool 1899 spdk_accel_alloc_sequence_buf(struct spdk_accel_sequence *seq, void *buf, 1900 struct spdk_memory_domain *domain, void *domain_ctx, 1901 spdk_accel_sequence_get_buf_cb cb_fn, void *cb_ctx) 1902 { 1903 struct accel_buffer *accel_buf = domain_ctx; 1904 1905 assert(domain == g_accel_domain); 1906 accel_buf->cb_fn = cb_fn; 1907 accel_buf->cb_ctx = cb_ctx; 1908 1909 if (!accel_sequence_alloc_buf(seq, accel_buf, accel_sequence_get_buf_cb)) { 1910 return false; 1911 } 1912 1913 accel_sequence_set_virtbuf(seq, accel_buf); 1914 1915 return true; 1916 } 1917 1918 struct spdk_accel_task * 1919 spdk_accel_sequence_first_task(struct spdk_accel_sequence *seq) 1920 { 1921 return TAILQ_FIRST(&seq->tasks); 1922 } 1923 1924 struct spdk_accel_task * 1925 spdk_accel_sequence_next_task(struct spdk_accel_task *task) 1926 { 1927 return TAILQ_NEXT(task, seq_link); 1928 } 1929 1930 static inline void 1931 accel_set_bounce_buffer(struct spdk_accel_bounce_buffer *bounce, struct iovec **iovs, 1932 uint32_t *iovcnt, struct spdk_memory_domain **domain, void **domain_ctx, 1933 struct accel_buffer *buf) 1934 { 1935 bounce->orig_iovs = *iovs; 1936 bounce->orig_iovcnt = *iovcnt; 1937 bounce->orig_domain = *domain; 1938 bounce->orig_domain_ctx = *domain_ctx; 1939 bounce->iov.iov_base = buf->buf; 1940 bounce->iov.iov_len = buf->len; 1941 1942 *iovs = &bounce->iov; 1943 *iovcnt = 1; 1944 *domain = NULL; 1945 } 1946 1947 static void 1948 accel_iobuf_get_src_bounce_cb(struct spdk_iobuf_entry *entry, void *buf) 1949 { 1950 struct spdk_accel_task *task; 1951 struct accel_buffer *accel_buf; 1952 1953 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1954 assert(accel_buf->buf == NULL); 1955 accel_buf->buf = buf; 1956 1957 task = TAILQ_FIRST(&accel_buf->seq->tasks); 1958 assert(task != NULL); 1959 1960 assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF); 1961 accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF); 1962 assert(task->aux); 1963 assert(task->has_aux); 1964 accel_set_bounce_buffer(&task->aux->bounce.s, &task->s.iovs, &task->s.iovcnt, &task->src_domain, 1965 &task->src_domain_ctx, accel_buf); 1966 accel_process_sequence(accel_buf->seq); 1967 } 1968 1969 static void 1970 accel_iobuf_get_dst_bounce_cb(struct spdk_iobuf_entry *entry, void *buf) 1971 { 1972 struct spdk_accel_task *task; 1973 struct accel_buffer *accel_buf; 1974 1975 accel_buf = SPDK_CONTAINEROF(entry, struct accel_buffer, iobuf); 1976 assert(accel_buf->buf == NULL); 1977 accel_buf->buf = buf; 1978 1979 task = TAILQ_FIRST(&accel_buf->seq->tasks); 1980 assert(task != NULL); 1981 1982 assert(accel_buf->seq->state == ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF); 1983 accel_sequence_set_state(accel_buf->seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF); 1984 assert(task->aux); 1985 assert(task->has_aux); 1986 accel_set_bounce_buffer(&task->aux->bounce.d, &task->d.iovs, &task->d.iovcnt, &task->dst_domain, 1987 &task->dst_domain_ctx, accel_buf); 1988 accel_process_sequence(accel_buf->seq); 1989 } 1990 1991 static int 1992 accel_sequence_check_bouncebuf(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 1993 { 1994 struct accel_buffer *buf; 1995 1996 if (task->src_domain != NULL) { 1997 /* By the time we're here, accel buffers should have been allocated */ 1998 assert(task->src_domain != g_accel_domain); 1999 2000 if (!task->has_aux) { 2001 task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool); 2002 if (spdk_unlikely(!task->aux)) { 2003 SPDK_ERRLOG("Can't allocate aux data structure\n"); 2004 assert(0); 2005 return -EAGAIN; 2006 } 2007 task->has_aux = true; 2008 SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link); 2009 } 2010 buf = accel_get_buf(seq->ch, accel_get_iovlen(task->s.iovs, task->s.iovcnt)); 2011 if (buf == NULL) { 2012 SPDK_ERRLOG("Couldn't allocate buffer descriptor\n"); 2013 return -ENOMEM; 2014 } 2015 2016 SLIST_INSERT_HEAD(&seq->bounce_bufs, buf, link); 2017 if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_src_bounce_cb)) { 2018 return -EAGAIN; 2019 } 2020 2021 accel_set_bounce_buffer(&task->aux->bounce.s, &task->s.iovs, &task->s.iovcnt, 2022 &task->src_domain, &task->src_domain_ctx, buf); 2023 } 2024 2025 if (task->dst_domain != NULL) { 2026 /* By the time we're here, accel buffers should have been allocated */ 2027 assert(task->dst_domain != g_accel_domain); 2028 2029 if (!task->has_aux) { 2030 task->aux = SLIST_FIRST(&task->accel_ch->task_aux_data_pool); 2031 if (spdk_unlikely(!task->aux)) { 2032 SPDK_ERRLOG("Can't allocate aux data structure\n"); 2033 assert(0); 2034 return -EAGAIN; 2035 } 2036 task->has_aux = true; 2037 SLIST_REMOVE_HEAD(&task->accel_ch->task_aux_data_pool, link); 2038 } 2039 buf = accel_get_buf(seq->ch, accel_get_iovlen(task->d.iovs, task->d.iovcnt)); 2040 if (buf == NULL) { 2041 /* The src buffer will be released when a sequence is completed */ 2042 SPDK_ERRLOG("Couldn't allocate buffer descriptor\n"); 2043 return -ENOMEM; 2044 } 2045 2046 SLIST_INSERT_HEAD(&seq->bounce_bufs, buf, link); 2047 if (!accel_sequence_alloc_buf(seq, buf, accel_iobuf_get_dst_bounce_cb)) { 2048 return -EAGAIN; 2049 } 2050 2051 accel_set_bounce_buffer(&task->aux->bounce.d, &task->d.iovs, &task->d.iovcnt, 2052 &task->dst_domain, &task->dst_domain_ctx, buf); 2053 } 2054 2055 return 0; 2056 } 2057 2058 static void 2059 accel_task_pull_data_cb(void *ctx, int status) 2060 { 2061 struct spdk_accel_sequence *seq = ctx; 2062 2063 assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA); 2064 if (spdk_likely(status == 0)) { 2065 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK); 2066 } else { 2067 accel_sequence_set_fail(seq, status); 2068 } 2069 2070 accel_process_sequence(seq); 2071 } 2072 2073 static void 2074 accel_task_pull_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 2075 { 2076 int rc; 2077 2078 assert(task->has_aux); 2079 assert(task->aux); 2080 assert(task->aux->bounce.s.orig_iovs != NULL); 2081 assert(task->aux->bounce.s.orig_domain != NULL); 2082 assert(task->aux->bounce.s.orig_domain != g_accel_domain); 2083 assert(!g_modules_opc[task->op_code].supports_memory_domains); 2084 2085 rc = spdk_memory_domain_pull_data(task->aux->bounce.s.orig_domain, 2086 task->aux->bounce.s.orig_domain_ctx, 2087 task->aux->bounce.s.orig_iovs, task->aux->bounce.s.orig_iovcnt, 2088 task->s.iovs, task->s.iovcnt, 2089 accel_task_pull_data_cb, seq); 2090 if (spdk_unlikely(rc != 0)) { 2091 SPDK_ERRLOG("Failed to pull data from memory domain: %s, rc: %d\n", 2092 spdk_memory_domain_get_dma_device_id(task->aux->bounce.s.orig_domain), rc); 2093 accel_sequence_set_fail(seq, rc); 2094 } 2095 } 2096 2097 static void 2098 accel_task_push_data_cb(void *ctx, int status) 2099 { 2100 struct spdk_accel_sequence *seq = ctx; 2101 2102 assert(seq->state == ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA); 2103 if (spdk_likely(status == 0)) { 2104 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK); 2105 } else { 2106 accel_sequence_set_fail(seq, status); 2107 } 2108 2109 accel_process_sequence(seq); 2110 } 2111 2112 static void 2113 accel_task_push_data(struct spdk_accel_sequence *seq, struct spdk_accel_task *task) 2114 { 2115 int rc; 2116 2117 assert(task->has_aux); 2118 assert(task->aux); 2119 assert(task->aux->bounce.d.orig_iovs != NULL); 2120 assert(task->aux->bounce.d.orig_domain != NULL); 2121 assert(task->aux->bounce.d.orig_domain != g_accel_domain); 2122 assert(!g_modules_opc[task->op_code].supports_memory_domains); 2123 2124 rc = spdk_memory_domain_push_data(task->aux->bounce.d.orig_domain, 2125 task->aux->bounce.d.orig_domain_ctx, 2126 task->aux->bounce.d.orig_iovs, task->aux->bounce.d.orig_iovcnt, 2127 task->d.iovs, task->d.iovcnt, 2128 accel_task_push_data_cb, seq); 2129 if (spdk_unlikely(rc != 0)) { 2130 SPDK_ERRLOG("Failed to push data to memory domain: %s, rc: %d\n", 2131 spdk_memory_domain_get_dma_device_id(task->aux->bounce.s.orig_domain), rc); 2132 accel_sequence_set_fail(seq, rc); 2133 } 2134 } 2135 2136 static void 2137 accel_process_sequence(struct spdk_accel_sequence *seq) 2138 { 2139 struct accel_io_channel *accel_ch = seq->ch; 2140 struct spdk_accel_task *task; 2141 enum accel_sequence_state state; 2142 int rc; 2143 2144 /* Prevent recursive calls to this function */ 2145 if (spdk_unlikely(seq->in_process_sequence)) { 2146 return; 2147 } 2148 seq->in_process_sequence = true; 2149 2150 task = TAILQ_FIRST(&seq->tasks); 2151 do { 2152 state = seq->state; 2153 switch (state) { 2154 case ACCEL_SEQUENCE_STATE_INIT: 2155 if (g_accel_driver != NULL) { 2156 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS); 2157 break; 2158 } 2159 /* Fall through */ 2160 case ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF: 2161 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF); 2162 if (!accel_sequence_check_virtbuf(seq, task)) { 2163 /* We couldn't allocate a buffer, wait until one is available */ 2164 break; 2165 } 2166 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF); 2167 /* Fall through */ 2168 case ACCEL_SEQUENCE_STATE_CHECK_BOUNCEBUF: 2169 /* If a module supports memory domains, we don't need to allocate bounce 2170 * buffers */ 2171 if (g_modules_opc[task->op_code].supports_memory_domains) { 2172 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK); 2173 break; 2174 } 2175 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF); 2176 rc = accel_sequence_check_bouncebuf(seq, task); 2177 if (spdk_unlikely(rc != 0)) { 2178 /* We couldn't allocate a buffer, wait until one is available */ 2179 if (rc == -EAGAIN) { 2180 break; 2181 } 2182 accel_sequence_set_fail(seq, rc); 2183 break; 2184 } 2185 if (task->has_aux && task->s.iovs == &task->aux->bounce.s.iov) { 2186 assert(task->aux->bounce.s.orig_iovs); 2187 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PULL_DATA); 2188 break; 2189 } 2190 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_EXEC_TASK); 2191 /* Fall through */ 2192 case ACCEL_SEQUENCE_STATE_EXEC_TASK: 2193 SPDK_DEBUGLOG(accel, "Executing %s operation, sequence: %p\n", 2194 g_opcode_strings[task->op_code], seq); 2195 2196 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_TASK); 2197 rc = accel_submit_task(accel_ch, task); 2198 if (spdk_unlikely(rc != 0)) { 2199 SPDK_ERRLOG("Failed to submit %s operation, sequence: %p\n", 2200 g_opcode_strings[task->op_code], seq); 2201 accel_sequence_set_fail(seq, rc); 2202 } 2203 break; 2204 case ACCEL_SEQUENCE_STATE_PULL_DATA: 2205 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA); 2206 accel_task_pull_data(seq, task); 2207 break; 2208 case ACCEL_SEQUENCE_STATE_COMPLETE_TASK: 2209 if (task->has_aux && task->d.iovs == &task->aux->bounce.d.iov) { 2210 assert(task->aux->bounce.d.orig_iovs); 2211 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_PUSH_DATA); 2212 break; 2213 } 2214 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_NEXT_TASK); 2215 break; 2216 case ACCEL_SEQUENCE_STATE_PUSH_DATA: 2217 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA); 2218 accel_task_push_data(seq, task); 2219 break; 2220 case ACCEL_SEQUENCE_STATE_NEXT_TASK: 2221 accel_sequence_complete_task(seq, task); 2222 /* Check if there are any remaining tasks */ 2223 task = TAILQ_FIRST(&seq->tasks); 2224 if (task == NULL) { 2225 /* Immediately return here to make sure we don't touch the sequence 2226 * after it's completed */ 2227 accel_sequence_complete(seq); 2228 return; 2229 } 2230 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_INIT); 2231 break; 2232 case ACCEL_SEQUENCE_STATE_DRIVER_EXEC_TASKS: 2233 assert(!TAILQ_EMPTY(&seq->tasks)); 2234 2235 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS); 2236 rc = g_accel_driver->execute_sequence(accel_ch->driver_channel, seq); 2237 if (spdk_unlikely(rc != 0)) { 2238 SPDK_ERRLOG("Failed to execute sequence: %p using driver: %s\n", 2239 seq, g_accel_driver->name); 2240 accel_sequence_set_fail(seq, rc); 2241 } 2242 break; 2243 case ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS: 2244 /* Get the task again, as the driver might have completed some tasks 2245 * synchronously */ 2246 task = TAILQ_FIRST(&seq->tasks); 2247 if (task == NULL) { 2248 /* Immediately return here to make sure we don't touch the sequence 2249 * after it's completed */ 2250 accel_sequence_complete(seq); 2251 return; 2252 } 2253 /* We don't want to execute the next task through the driver, so we 2254 * explicitly omit the INIT state here */ 2255 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_CHECK_VIRTBUF); 2256 break; 2257 case ACCEL_SEQUENCE_STATE_ERROR: 2258 /* Immediately return here to make sure we don't touch the sequence 2259 * after it's completed */ 2260 assert(seq->status != 0); 2261 accel_sequence_complete(seq); 2262 return; 2263 case ACCEL_SEQUENCE_STATE_AWAIT_VIRTBUF: 2264 case ACCEL_SEQUENCE_STATE_AWAIT_BOUNCEBUF: 2265 case ACCEL_SEQUENCE_STATE_AWAIT_PULL_DATA: 2266 case ACCEL_SEQUENCE_STATE_AWAIT_TASK: 2267 case ACCEL_SEQUENCE_STATE_AWAIT_PUSH_DATA: 2268 case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS: 2269 break; 2270 default: 2271 assert(0 && "bad state"); 2272 break; 2273 } 2274 } while (seq->state != state); 2275 2276 seq->in_process_sequence = false; 2277 } 2278 2279 static void 2280 accel_sequence_task_cb(void *cb_arg, int status) 2281 { 2282 struct spdk_accel_sequence *seq = cb_arg; 2283 struct spdk_accel_task *task = TAILQ_FIRST(&seq->tasks); 2284 2285 switch (seq->state) { 2286 case ACCEL_SEQUENCE_STATE_AWAIT_TASK: 2287 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_COMPLETE_TASK); 2288 if (spdk_unlikely(status != 0)) { 2289 SPDK_ERRLOG("Failed to execute %s operation, sequence: %p\n", 2290 g_opcode_strings[task->op_code], seq); 2291 accel_sequence_set_fail(seq, status); 2292 } 2293 2294 accel_process_sequence(seq); 2295 break; 2296 case ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS: 2297 assert(g_accel_driver != NULL); 2298 /* Immediately remove the task from the outstanding list to make sure the next call 2299 * to spdk_accel_sequence_first_task() doesn't return it */ 2300 accel_sequence_complete_task(seq, task); 2301 if (spdk_unlikely(status != 0)) { 2302 SPDK_ERRLOG("Failed to execute %s operation, sequence: %p through " 2303 "driver: %s\n", g_opcode_strings[task->op_code], seq, 2304 g_accel_driver->name); 2305 /* Update status without using accel_sequence_set_fail() to avoid changing 2306 * seq's state to ERROR until driver calls spdk_accel_sequence_continue() */ 2307 seq->status = status; 2308 } 2309 break; 2310 default: 2311 assert(0 && "bad state"); 2312 break; 2313 } 2314 } 2315 2316 void 2317 spdk_accel_sequence_continue(struct spdk_accel_sequence *seq) 2318 { 2319 assert(g_accel_driver != NULL); 2320 assert(seq->state == ACCEL_SEQUENCE_STATE_DRIVER_AWAIT_TASKS); 2321 2322 if (spdk_likely(seq->status == 0)) { 2323 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_DRIVER_COMPLETE_TASKS); 2324 } else { 2325 accel_sequence_set_state(seq, ACCEL_SEQUENCE_STATE_ERROR); 2326 } 2327 2328 accel_process_sequence(seq); 2329 } 2330 2331 static bool 2332 accel_compare_iovs(struct iovec *iova, uint32_t iovacnt, struct iovec *iovb, uint32_t iovbcnt) 2333 { 2334 /* For now, just do a dumb check that the iovecs arrays are exactly the same */ 2335 if (iovacnt != iovbcnt) { 2336 return false; 2337 } 2338 2339 return memcmp(iova, iovb, sizeof(*iova) * iovacnt) == 0; 2340 } 2341 2342 static bool 2343 accel_task_set_dstbuf(struct spdk_accel_task *task, struct spdk_accel_task *next) 2344 { 2345 struct spdk_accel_task *prev; 2346 2347 switch (task->op_code) { 2348 case SPDK_ACCEL_OPC_DECOMPRESS: 2349 case SPDK_ACCEL_OPC_FILL: 2350 case SPDK_ACCEL_OPC_ENCRYPT: 2351 case SPDK_ACCEL_OPC_DECRYPT: 2352 case SPDK_ACCEL_OPC_DIF_GENERATE_COPY: 2353 case SPDK_ACCEL_OPC_DIF_VERIFY_COPY: 2354 if (task->dst_domain != next->src_domain) { 2355 return false; 2356 } 2357 if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt, 2358 next->s.iovs, next->s.iovcnt)) { 2359 return false; 2360 } 2361 task->d.iovs = next->d.iovs; 2362 task->d.iovcnt = next->d.iovcnt; 2363 task->dst_domain = next->dst_domain; 2364 task->dst_domain_ctx = next->dst_domain_ctx; 2365 break; 2366 case SPDK_ACCEL_OPC_CRC32C: 2367 /* crc32 is special, because it doesn't have a dst buffer */ 2368 if (task->src_domain != next->src_domain) { 2369 return false; 2370 } 2371 if (!accel_compare_iovs(task->s.iovs, task->s.iovcnt, 2372 next->s.iovs, next->s.iovcnt)) { 2373 return false; 2374 } 2375 /* We can only change crc32's buffer if we can change previous task's buffer */ 2376 prev = TAILQ_PREV(task, accel_sequence_tasks, seq_link); 2377 if (prev == NULL) { 2378 return false; 2379 } 2380 if (!accel_task_set_dstbuf(prev, next)) { 2381 return false; 2382 } 2383 task->s.iovs = next->d.iovs; 2384 task->s.iovcnt = next->d.iovcnt; 2385 task->src_domain = next->dst_domain; 2386 task->src_domain_ctx = next->dst_domain_ctx; 2387 break; 2388 default: 2389 return false; 2390 } 2391 2392 return true; 2393 } 2394 2395 static void 2396 accel_sequence_merge_tasks(struct spdk_accel_sequence *seq, struct spdk_accel_task *task, 2397 struct spdk_accel_task **next_task) 2398 { 2399 struct spdk_accel_task *next = *next_task; 2400 2401 switch (task->op_code) { 2402 case SPDK_ACCEL_OPC_COPY: 2403 /* We only allow changing src of operations that actually have a src, e.g. we never 2404 * do it for fill. Theoretically, it is possible, but we'd have to be careful to 2405 * change the src of the operation after fill (which in turn could also be a fill). 2406 * So, for the sake of simplicity, skip this type of operations for now. 2407 */ 2408 if (next->op_code != SPDK_ACCEL_OPC_DECOMPRESS && 2409 next->op_code != SPDK_ACCEL_OPC_COPY && 2410 next->op_code != SPDK_ACCEL_OPC_ENCRYPT && 2411 next->op_code != SPDK_ACCEL_OPC_DECRYPT && 2412 next->op_code != SPDK_ACCEL_OPC_COPY_CRC32C && 2413 next->op_code != SPDK_ACCEL_OPC_DIF_GENERATE_COPY && 2414 next->op_code != SPDK_ACCEL_OPC_DIF_VERIFY_COPY) { 2415 break; 2416 } 2417 if (task->dst_domain != next->src_domain) { 2418 break; 2419 } 2420 if (!accel_compare_iovs(task->d.iovs, task->d.iovcnt, 2421 next->s.iovs, next->s.iovcnt)) { 2422 break; 2423 } 2424 next->s.iovs = task->s.iovs; 2425 next->s.iovcnt = task->s.iovcnt; 2426 next->src_domain = task->src_domain; 2427 next->src_domain_ctx = task->src_domain_ctx; 2428 accel_sequence_complete_task(seq, task); 2429 break; 2430 case SPDK_ACCEL_OPC_DECOMPRESS: 2431 case SPDK_ACCEL_OPC_FILL: 2432 case SPDK_ACCEL_OPC_ENCRYPT: 2433 case SPDK_ACCEL_OPC_DECRYPT: 2434 case SPDK_ACCEL_OPC_CRC32C: 2435 case SPDK_ACCEL_OPC_DIF_GENERATE_COPY: 2436 case SPDK_ACCEL_OPC_DIF_VERIFY_COPY: 2437 /* We can only merge tasks when one of them is a copy */ 2438 if (next->op_code != SPDK_ACCEL_OPC_COPY) { 2439 break; 2440 } 2441 if (!accel_task_set_dstbuf(task, next)) { 2442 break; 2443 } 2444 /* We're removing next_task from the tasks queue, so we need to update its pointer, 2445 * so that the TAILQ_FOREACH_SAFE() loop below works correctly */ 2446 *next_task = TAILQ_NEXT(next, seq_link); 2447 accel_sequence_complete_task(seq, next); 2448 break; 2449 default: 2450 assert(0 && "bad opcode"); 2451 break; 2452 } 2453 } 2454 2455 void 2456 spdk_accel_sequence_finish(struct spdk_accel_sequence *seq, 2457 spdk_accel_completion_cb cb_fn, void *cb_arg) 2458 { 2459 struct spdk_accel_task *task, *next; 2460 2461 /* Try to remove any copy operations if possible */ 2462 TAILQ_FOREACH_SAFE(task, &seq->tasks, seq_link, next) { 2463 if (next == NULL) { 2464 break; 2465 } 2466 accel_sequence_merge_tasks(seq, task, &next); 2467 } 2468 2469 seq->cb_fn = cb_fn; 2470 seq->cb_arg = cb_arg; 2471 2472 accel_process_sequence(seq); 2473 } 2474 2475 void 2476 spdk_accel_sequence_reverse(struct spdk_accel_sequence *seq) 2477 { 2478 struct accel_sequence_tasks tasks = TAILQ_HEAD_INITIALIZER(tasks); 2479 struct spdk_accel_task *task; 2480 2481 TAILQ_SWAP(&tasks, &seq->tasks, spdk_accel_task, seq_link); 2482 2483 while (!TAILQ_EMPTY(&tasks)) { 2484 task = TAILQ_FIRST(&tasks); 2485 TAILQ_REMOVE(&tasks, task, seq_link); 2486 TAILQ_INSERT_HEAD(&seq->tasks, task, seq_link); 2487 } 2488 } 2489 2490 void 2491 spdk_accel_sequence_abort(struct spdk_accel_sequence *seq) 2492 { 2493 if (seq == NULL) { 2494 return; 2495 } 2496 2497 accel_sequence_complete_tasks(seq); 2498 accel_sequence_put(seq); 2499 } 2500 2501 struct spdk_memory_domain * 2502 spdk_accel_get_memory_domain(void) 2503 { 2504 return g_accel_domain; 2505 } 2506 2507 static struct spdk_accel_module_if * 2508 _module_find_by_name(const char *name) 2509 { 2510 struct spdk_accel_module_if *accel_module = NULL; 2511 2512 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 2513 if (strcmp(name, accel_module->name) == 0) { 2514 break; 2515 } 2516 } 2517 2518 return accel_module; 2519 } 2520 2521 static inline struct spdk_accel_crypto_key * 2522 _accel_crypto_key_get(const char *name) 2523 { 2524 struct spdk_accel_crypto_key *key; 2525 2526 assert(spdk_spin_held(&g_keyring_spin)); 2527 2528 TAILQ_FOREACH(key, &g_keyring, link) { 2529 if (strcmp(name, key->param.key_name) == 0) { 2530 return key; 2531 } 2532 } 2533 2534 return NULL; 2535 } 2536 2537 static void 2538 accel_crypto_key_free_mem(struct spdk_accel_crypto_key *key) 2539 { 2540 if (key->param.hex_key) { 2541 spdk_memset_s(key->param.hex_key, key->key_size * 2, 0, key->key_size * 2); 2542 free(key->param.hex_key); 2543 } 2544 if (key->param.hex_key2) { 2545 spdk_memset_s(key->param.hex_key2, key->key2_size * 2, 0, key->key2_size * 2); 2546 free(key->param.hex_key2); 2547 } 2548 free(key->param.tweak_mode); 2549 free(key->param.key_name); 2550 free(key->param.cipher); 2551 if (key->key) { 2552 spdk_memset_s(key->key, key->key_size, 0, key->key_size); 2553 free(key->key); 2554 } 2555 if (key->key2) { 2556 spdk_memset_s(key->key2, key->key2_size, 0, key->key2_size); 2557 free(key->key2); 2558 } 2559 free(key); 2560 } 2561 2562 static void 2563 accel_crypto_key_destroy_unsafe(struct spdk_accel_crypto_key *key) 2564 { 2565 assert(key->module_if); 2566 assert(key->module_if->crypto_key_deinit); 2567 2568 key->module_if->crypto_key_deinit(key); 2569 accel_crypto_key_free_mem(key); 2570 } 2571 2572 /* 2573 * This function mitigates a timing side channel which could be caused by using strcmp() 2574 * Please refer to chapter "Mitigating Information Leakage Based on Variable Timing" in 2575 * the article [1] for more details 2576 * [1] https://www.intel.com/content/www/us/en/developer/articles/technical/software-security-guidance/secure-coding/mitigate-timing-side-channel-crypto-implementation.html 2577 */ 2578 static bool 2579 accel_aes_xts_keys_equal(const char *k1, size_t k1_len, const char *k2, size_t k2_len) 2580 { 2581 size_t i; 2582 volatile size_t x = k1_len ^ k2_len; 2583 2584 for (i = 0; ((i < k1_len) & (i < k2_len)); i++) { 2585 x |= k1[i] ^ k2[i]; 2586 } 2587 2588 return x == 0; 2589 } 2590 2591 static const char *g_tweak_modes[] = { 2592 [SPDK_ACCEL_CRYPTO_TWEAK_MODE_SIMPLE_LBA] = "SIMPLE_LBA", 2593 [SPDK_ACCEL_CRYPTO_TWEAK_MODE_JOIN_NEG_LBA_WITH_LBA] = "JOIN_NEG_LBA_WITH_LBA", 2594 [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_FULL_LBA] = "INCR_512_FULL_LBA", 2595 [SPDK_ACCEL_CRYPTO_TWEAK_MODE_INCR_512_UPPER_LBA] = "INCR_512_UPPER_LBA", 2596 }; 2597 2598 static const char *g_ciphers[] = { 2599 [SPDK_ACCEL_CIPHER_AES_CBC] = "AES_CBC", 2600 [SPDK_ACCEL_CIPHER_AES_XTS] = "AES_XTS", 2601 }; 2602 2603 int 2604 spdk_accel_crypto_key_create(const struct spdk_accel_crypto_key_create_param *param) 2605 { 2606 struct spdk_accel_module_if *module; 2607 struct spdk_accel_crypto_key *key; 2608 size_t hex_key_size, hex_key2_size; 2609 bool found = false; 2610 size_t i; 2611 int rc; 2612 2613 if (!param || !param->hex_key || !param->cipher || !param->key_name) { 2614 return -EINVAL; 2615 } 2616 2617 if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) { 2618 /* hardly ever possible, but let's check and warn the user */ 2619 SPDK_ERRLOG("Different accel modules are used for encryption and decryption\n"); 2620 } 2621 module = g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module; 2622 2623 if (!module) { 2624 SPDK_ERRLOG("No accel module found assigned for crypto operation\n"); 2625 return -ENOENT; 2626 } 2627 2628 if (!module->crypto_key_init || !module->crypto_supports_cipher) { 2629 SPDK_ERRLOG("Module %s doesn't support crypto operations\n", module->name); 2630 return -ENOTSUP; 2631 } 2632 2633 key = calloc(1, sizeof(*key)); 2634 if (!key) { 2635 return -ENOMEM; 2636 } 2637 2638 key->param.key_name = strdup(param->key_name); 2639 if (!key->param.key_name) { 2640 rc = -ENOMEM; 2641 goto error; 2642 } 2643 2644 for (i = 0; i < SPDK_COUNTOF(g_ciphers); ++i) { 2645 assert(g_ciphers[i]); 2646 2647 if (strncmp(param->cipher, g_ciphers[i], strlen(g_ciphers[i])) == 0) { 2648 key->cipher = i; 2649 found = true; 2650 break; 2651 } 2652 } 2653 2654 if (!found) { 2655 SPDK_ERRLOG("Failed to parse cipher\n"); 2656 rc = -EINVAL; 2657 goto error; 2658 } 2659 2660 key->param.cipher = strdup(param->cipher); 2661 if (!key->param.cipher) { 2662 rc = -ENOMEM; 2663 goto error; 2664 } 2665 2666 hex_key_size = strnlen(param->hex_key, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 2667 if (hex_key_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) { 2668 SPDK_ERRLOG("key1 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 2669 rc = -EINVAL; 2670 goto error; 2671 } 2672 2673 if (hex_key_size == 0) { 2674 SPDK_ERRLOG("key1 size cannot be 0\n"); 2675 rc = -EINVAL; 2676 goto error; 2677 } 2678 2679 key->param.hex_key = strdup(param->hex_key); 2680 if (!key->param.hex_key) { 2681 rc = -ENOMEM; 2682 goto error; 2683 } 2684 2685 key->key_size = hex_key_size / 2; 2686 key->key = spdk_unhexlify(key->param.hex_key); 2687 if (!key->key) { 2688 SPDK_ERRLOG("Failed to unhexlify key1\n"); 2689 rc = -EINVAL; 2690 goto error; 2691 } 2692 2693 if (param->hex_key2) { 2694 hex_key2_size = strnlen(param->hex_key2, SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 2695 if (hex_key2_size == SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH) { 2696 SPDK_ERRLOG("key2 size exceeds max %d\n", SPDK_ACCEL_CRYPTO_KEY_MAX_HEX_LENGTH); 2697 rc = -EINVAL; 2698 goto error; 2699 } 2700 2701 if (hex_key2_size == 0) { 2702 SPDK_ERRLOG("key2 size cannot be 0\n"); 2703 rc = -EINVAL; 2704 goto error; 2705 } 2706 2707 key->param.hex_key2 = strdup(param->hex_key2); 2708 if (!key->param.hex_key2) { 2709 rc = -ENOMEM; 2710 goto error; 2711 } 2712 2713 key->key2_size = hex_key2_size / 2; 2714 key->key2 = spdk_unhexlify(key->param.hex_key2); 2715 if (!key->key2) { 2716 SPDK_ERRLOG("Failed to unhexlify key2\n"); 2717 rc = -EINVAL; 2718 goto error; 2719 } 2720 } 2721 2722 key->tweak_mode = ACCEL_CRYPTO_TWEAK_MODE_DEFAULT; 2723 if (param->tweak_mode) { 2724 found = false; 2725 2726 key->param.tweak_mode = strdup(param->tweak_mode); 2727 if (!key->param.tweak_mode) { 2728 rc = -ENOMEM; 2729 goto error; 2730 } 2731 2732 for (i = 0; i < SPDK_COUNTOF(g_tweak_modes); ++i) { 2733 assert(g_tweak_modes[i]); 2734 2735 if (strncmp(param->tweak_mode, g_tweak_modes[i], strlen(g_tweak_modes[i])) == 0) { 2736 key->tweak_mode = i; 2737 found = true; 2738 break; 2739 } 2740 } 2741 2742 if (!found) { 2743 SPDK_ERRLOG("Failed to parse tweak mode\n"); 2744 rc = -EINVAL; 2745 goto error; 2746 } 2747 } 2748 2749 if ((!module->crypto_supports_tweak_mode && key->tweak_mode != ACCEL_CRYPTO_TWEAK_MODE_DEFAULT) || 2750 (module->crypto_supports_tweak_mode && !module->crypto_supports_tweak_mode(key->tweak_mode))) { 2751 SPDK_ERRLOG("Module %s doesn't support %s tweak mode\n", module->name, 2752 g_tweak_modes[key->tweak_mode]); 2753 rc = -EINVAL; 2754 goto error; 2755 } 2756 2757 if (!module->crypto_supports_cipher(key->cipher, key->key_size)) { 2758 SPDK_ERRLOG("Module %s doesn't support %s cipher with %zu key size\n", module->name, 2759 g_ciphers[key->cipher], key->key_size); 2760 rc = -EINVAL; 2761 goto error; 2762 } 2763 2764 if (key->cipher == SPDK_ACCEL_CIPHER_AES_XTS) { 2765 if (!key->key2) { 2766 SPDK_ERRLOG("%s key2 is missing\n", g_ciphers[key->cipher]); 2767 rc = -EINVAL; 2768 goto error; 2769 } 2770 2771 if (key->key_size != key->key2_size) { 2772 SPDK_ERRLOG("%s key size %zu is not equal to key2 size %zu\n", g_ciphers[key->cipher], 2773 key->key_size, 2774 key->key2_size); 2775 rc = -EINVAL; 2776 goto error; 2777 } 2778 2779 if (accel_aes_xts_keys_equal(key->key, key->key_size, key->key2, key->key2_size)) { 2780 SPDK_ERRLOG("%s identical keys are not secure\n", g_ciphers[key->cipher]); 2781 rc = -EINVAL; 2782 goto error; 2783 } 2784 } 2785 2786 if (key->cipher == SPDK_ACCEL_CIPHER_AES_CBC) { 2787 if (key->key2_size) { 2788 SPDK_ERRLOG("%s doesn't use key2\n", g_ciphers[key->cipher]); 2789 rc = -EINVAL; 2790 goto error; 2791 } 2792 } 2793 2794 key->module_if = module; 2795 2796 spdk_spin_lock(&g_keyring_spin); 2797 if (_accel_crypto_key_get(param->key_name)) { 2798 rc = -EEXIST; 2799 } else { 2800 rc = module->crypto_key_init(key); 2801 if (rc) { 2802 SPDK_ERRLOG("Module %s failed to initialize crypto key\n", module->name); 2803 } else { 2804 TAILQ_INSERT_TAIL(&g_keyring, key, link); 2805 } 2806 } 2807 spdk_spin_unlock(&g_keyring_spin); 2808 2809 if (rc) { 2810 goto error; 2811 } 2812 2813 return 0; 2814 2815 error: 2816 accel_crypto_key_free_mem(key); 2817 return rc; 2818 } 2819 2820 int 2821 spdk_accel_crypto_key_destroy(struct spdk_accel_crypto_key *key) 2822 { 2823 if (!key || !key->module_if) { 2824 return -EINVAL; 2825 } 2826 2827 spdk_spin_lock(&g_keyring_spin); 2828 if (!_accel_crypto_key_get(key->param.key_name)) { 2829 spdk_spin_unlock(&g_keyring_spin); 2830 return -ENOENT; 2831 } 2832 TAILQ_REMOVE(&g_keyring, key, link); 2833 spdk_spin_unlock(&g_keyring_spin); 2834 2835 accel_crypto_key_destroy_unsafe(key); 2836 2837 return 0; 2838 } 2839 2840 struct spdk_accel_crypto_key * 2841 spdk_accel_crypto_key_get(const char *name) 2842 { 2843 struct spdk_accel_crypto_key *key; 2844 2845 spdk_spin_lock(&g_keyring_spin); 2846 key = _accel_crypto_key_get(name); 2847 spdk_spin_unlock(&g_keyring_spin); 2848 2849 return key; 2850 } 2851 2852 /* Helper function when accel modules register with the framework. */ 2853 void 2854 spdk_accel_module_list_add(struct spdk_accel_module_if *accel_module) 2855 { 2856 struct spdk_accel_module_if *tmp; 2857 2858 if (_module_find_by_name(accel_module->name)) { 2859 SPDK_NOTICELOG("Module %s already registered\n", accel_module->name); 2860 assert(false); 2861 return; 2862 } 2863 2864 TAILQ_FOREACH(tmp, &spdk_accel_module_list, tailq) { 2865 if (accel_module->priority < tmp->priority) { 2866 break; 2867 } 2868 } 2869 2870 if (tmp != NULL) { 2871 TAILQ_INSERT_BEFORE(tmp, accel_module, tailq); 2872 } else { 2873 TAILQ_INSERT_TAIL(&spdk_accel_module_list, accel_module, tailq); 2874 } 2875 } 2876 2877 /* Framework level channel create callback. */ 2878 static int 2879 accel_create_channel(void *io_device, void *ctx_buf) 2880 { 2881 struct accel_io_channel *accel_ch = ctx_buf; 2882 struct spdk_accel_task *accel_task; 2883 struct spdk_accel_task_aux_data *accel_task_aux; 2884 struct spdk_accel_sequence *seq; 2885 struct accel_buffer *buf; 2886 size_t task_size_aligned; 2887 uint8_t *task_mem; 2888 uint32_t i = 0, j; 2889 int rc; 2890 2891 task_size_aligned = SPDK_ALIGN_CEIL(g_max_accel_module_size, SPDK_CACHE_LINE_SIZE); 2892 accel_ch->task_pool_base = aligned_alloc(SPDK_CACHE_LINE_SIZE, 2893 g_opts.task_count * task_size_aligned); 2894 if (!accel_ch->task_pool_base) { 2895 return -ENOMEM; 2896 } 2897 memset(accel_ch->task_pool_base, 0, g_opts.task_count * task_size_aligned); 2898 2899 accel_ch->seq_pool_base = aligned_alloc(SPDK_CACHE_LINE_SIZE, 2900 g_opts.sequence_count * sizeof(struct spdk_accel_sequence)); 2901 if (accel_ch->seq_pool_base == NULL) { 2902 goto err; 2903 } 2904 memset(accel_ch->seq_pool_base, 0, g_opts.sequence_count * sizeof(struct spdk_accel_sequence)); 2905 2906 accel_ch->task_aux_data_base = calloc(g_opts.task_count, sizeof(struct spdk_accel_task_aux_data)); 2907 if (accel_ch->task_aux_data_base == NULL) { 2908 goto err; 2909 } 2910 2911 accel_ch->buf_pool_base = calloc(g_opts.buf_count, sizeof(struct accel_buffer)); 2912 if (accel_ch->buf_pool_base == NULL) { 2913 goto err; 2914 } 2915 2916 STAILQ_INIT(&accel_ch->task_pool); 2917 SLIST_INIT(&accel_ch->task_aux_data_pool); 2918 SLIST_INIT(&accel_ch->seq_pool); 2919 SLIST_INIT(&accel_ch->buf_pool); 2920 2921 task_mem = accel_ch->task_pool_base; 2922 for (i = 0; i < g_opts.task_count; i++) { 2923 accel_task = (struct spdk_accel_task *)task_mem; 2924 accel_task->aux = NULL; 2925 STAILQ_INSERT_TAIL(&accel_ch->task_pool, accel_task, link); 2926 task_mem += task_size_aligned; 2927 accel_task_aux = &accel_ch->task_aux_data_base[i]; 2928 SLIST_INSERT_HEAD(&accel_ch->task_aux_data_pool, accel_task_aux, link); 2929 } 2930 for (i = 0; i < g_opts.sequence_count; i++) { 2931 seq = &accel_ch->seq_pool_base[i]; 2932 SLIST_INSERT_HEAD(&accel_ch->seq_pool, seq, link); 2933 } 2934 for (i = 0; i < g_opts.buf_count; i++) { 2935 buf = &accel_ch->buf_pool_base[i]; 2936 SLIST_INSERT_HEAD(&accel_ch->buf_pool, buf, link); 2937 } 2938 2939 /* Assign modules and get IO channels for each */ 2940 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 2941 accel_ch->module_ch[i] = g_modules_opc[i].module->get_io_channel(); 2942 /* This can happen if idxd runs out of channels. */ 2943 if (accel_ch->module_ch[i] == NULL) { 2944 SPDK_ERRLOG("Module %s failed to get io channel\n", g_modules_opc[i].module->name); 2945 goto err; 2946 } 2947 } 2948 2949 if (g_accel_driver != NULL) { 2950 accel_ch->driver_channel = g_accel_driver->get_io_channel(); 2951 if (accel_ch->driver_channel == NULL) { 2952 SPDK_ERRLOG("Failed to get driver's IO channel\n"); 2953 goto err; 2954 } 2955 } 2956 2957 rc = spdk_iobuf_channel_init(&accel_ch->iobuf, "accel", g_opts.small_cache_size, 2958 g_opts.large_cache_size); 2959 if (rc != 0) { 2960 SPDK_ERRLOG("Failed to initialize iobuf accel channel\n"); 2961 goto err; 2962 } 2963 2964 return 0; 2965 err: 2966 if (accel_ch->driver_channel != NULL) { 2967 spdk_put_io_channel(accel_ch->driver_channel); 2968 } 2969 for (j = 0; j < i; j++) { 2970 spdk_put_io_channel(accel_ch->module_ch[j]); 2971 } 2972 free(accel_ch->task_pool_base); 2973 free(accel_ch->task_aux_data_base); 2974 free(accel_ch->seq_pool_base); 2975 free(accel_ch->buf_pool_base); 2976 2977 return -ENOMEM; 2978 } 2979 2980 static void 2981 accel_add_stats(struct accel_stats *total, struct accel_stats *stats) 2982 { 2983 int i; 2984 2985 total->sequence_executed += stats->sequence_executed; 2986 total->sequence_failed += stats->sequence_failed; 2987 total->sequence_outstanding += stats->sequence_outstanding; 2988 total->task_outstanding += stats->task_outstanding; 2989 total->retry.task += stats->retry.task; 2990 total->retry.sequence += stats->retry.sequence; 2991 total->retry.iobuf += stats->retry.iobuf; 2992 total->retry.bufdesc += stats->retry.bufdesc; 2993 for (i = 0; i < SPDK_ACCEL_OPC_LAST; ++i) { 2994 total->operations[i].executed += stats->operations[i].executed; 2995 total->operations[i].failed += stats->operations[i].failed; 2996 total->operations[i].num_bytes += stats->operations[i].num_bytes; 2997 } 2998 } 2999 3000 /* Framework level channel destroy callback. */ 3001 static void 3002 accel_destroy_channel(void *io_device, void *ctx_buf) 3003 { 3004 struct accel_io_channel *accel_ch = ctx_buf; 3005 int i; 3006 3007 spdk_iobuf_channel_fini(&accel_ch->iobuf); 3008 3009 if (accel_ch->driver_channel != NULL) { 3010 spdk_put_io_channel(accel_ch->driver_channel); 3011 } 3012 3013 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 3014 assert(accel_ch->module_ch[i] != NULL); 3015 spdk_put_io_channel(accel_ch->module_ch[i]); 3016 accel_ch->module_ch[i] = NULL; 3017 } 3018 3019 /* Update global stats to make sure channel's stats aren't lost after a channel is gone */ 3020 spdk_spin_lock(&g_stats_lock); 3021 accel_add_stats(&g_stats, &accel_ch->stats); 3022 spdk_spin_unlock(&g_stats_lock); 3023 3024 free(accel_ch->task_pool_base); 3025 free(accel_ch->task_aux_data_base); 3026 free(accel_ch->seq_pool_base); 3027 free(accel_ch->buf_pool_base); 3028 } 3029 3030 struct spdk_io_channel * 3031 spdk_accel_get_io_channel(void) 3032 { 3033 return spdk_get_io_channel(&spdk_accel_module_list); 3034 } 3035 3036 static int 3037 accel_module_initialize(void) 3038 { 3039 struct spdk_accel_module_if *accel_module, *tmp_module; 3040 int rc = 0, module_rc; 3041 3042 TAILQ_FOREACH_SAFE(accel_module, &spdk_accel_module_list, tailq, tmp_module) { 3043 module_rc = accel_module->module_init(); 3044 if (module_rc) { 3045 TAILQ_REMOVE(&spdk_accel_module_list, accel_module, tailq); 3046 if (module_rc == -ENODEV) { 3047 SPDK_NOTICELOG("No devices for module %s, skipping\n", accel_module->name); 3048 } else if (!rc) { 3049 SPDK_ERRLOG("Module %s initialization failed with %d\n", accel_module->name, module_rc); 3050 rc = module_rc; 3051 } 3052 continue; 3053 } 3054 3055 SPDK_DEBUGLOG(accel, "Module %s initialized.\n", accel_module->name); 3056 } 3057 3058 return rc; 3059 } 3060 3061 static void 3062 accel_module_init_opcode(enum spdk_accel_opcode opcode) 3063 { 3064 struct accel_module *module = &g_modules_opc[opcode]; 3065 struct spdk_accel_module_if *module_if = module->module; 3066 3067 if (module_if->get_memory_domains != NULL) { 3068 module->supports_memory_domains = module_if->get_memory_domains(NULL, 0) > 0; 3069 } 3070 } 3071 3072 static int 3073 accel_memory_domain_translate(struct spdk_memory_domain *src_domain, void *src_domain_ctx, 3074 struct spdk_memory_domain *dst_domain, struct spdk_memory_domain_translation_ctx *dst_domain_ctx, 3075 void *addr, size_t len, struct spdk_memory_domain_translation_result *result) 3076 { 3077 struct accel_buffer *buf = src_domain_ctx; 3078 3079 SPDK_DEBUGLOG(accel, "translate addr %p, len %zu\n", addr, len); 3080 3081 assert(g_accel_domain == src_domain); 3082 assert(spdk_memory_domain_get_system_domain() == dst_domain); 3083 assert(buf->buf == NULL); 3084 assert(addr == ACCEL_BUFFER_BASE); 3085 assert(len == buf->len); 3086 3087 buf->buf = spdk_iobuf_get(&buf->ch->iobuf, buf->len, NULL, NULL); 3088 if (spdk_unlikely(buf->buf == NULL)) { 3089 return -ENOMEM; 3090 } 3091 3092 result->iov_count = 1; 3093 result->iov.iov_base = buf->buf; 3094 result->iov.iov_len = buf->len; 3095 SPDK_DEBUGLOG(accel, "translated addr %p\n", result->iov.iov_base); 3096 return 0; 3097 } 3098 3099 static void 3100 accel_memory_domain_invalidate(struct spdk_memory_domain *domain, void *domain_ctx, 3101 struct iovec *iov, uint32_t iovcnt) 3102 { 3103 struct accel_buffer *buf = domain_ctx; 3104 3105 SPDK_DEBUGLOG(accel, "invalidate addr %p, len %zu\n", iov[0].iov_base, iov[0].iov_len); 3106 3107 assert(g_accel_domain == domain); 3108 assert(iovcnt == 1); 3109 assert(buf->buf != NULL); 3110 assert(iov[0].iov_base == buf->buf); 3111 assert(iov[0].iov_len == buf->len); 3112 3113 spdk_iobuf_put(&buf->ch->iobuf, buf->buf, buf->len); 3114 buf->buf = NULL; 3115 } 3116 3117 int 3118 spdk_accel_initialize(void) 3119 { 3120 enum spdk_accel_opcode op; 3121 struct spdk_accel_module_if *accel_module = NULL; 3122 int rc; 3123 3124 /* 3125 * We need a unique identifier for the accel framework, so use the 3126 * spdk_accel_module_list address for this purpose. 3127 */ 3128 spdk_io_device_register(&spdk_accel_module_list, accel_create_channel, accel_destroy_channel, 3129 sizeof(struct accel_io_channel), "accel"); 3130 3131 spdk_spin_init(&g_keyring_spin); 3132 spdk_spin_init(&g_stats_lock); 3133 3134 rc = spdk_memory_domain_create(&g_accel_domain, SPDK_DMA_DEVICE_TYPE_ACCEL, NULL, 3135 "SPDK_ACCEL_DMA_DEVICE"); 3136 if (rc != 0) { 3137 SPDK_ERRLOG("Failed to create accel memory domain\n"); 3138 return rc; 3139 } 3140 3141 spdk_memory_domain_set_translation(g_accel_domain, accel_memory_domain_translate); 3142 spdk_memory_domain_set_invalidate(g_accel_domain, accel_memory_domain_invalidate); 3143 3144 g_modules_started = true; 3145 rc = accel_module_initialize(); 3146 if (rc) { 3147 return rc; 3148 } 3149 3150 if (g_accel_driver != NULL && g_accel_driver->init != NULL) { 3151 rc = g_accel_driver->init(); 3152 if (rc != 0) { 3153 SPDK_ERRLOG("Failed to initialize driver %s: %s\n", g_accel_driver->name, 3154 spdk_strerror(-rc)); 3155 return rc; 3156 } 3157 } 3158 3159 /* The module list is order by priority, with the highest priority modules being at the end 3160 * of the list. The software module should be somewhere at the beginning of the list, 3161 * before all HW modules. 3162 * NOTE: all opcodes must be supported by software in the event that no HW modules are 3163 * initialized to support the operation. 3164 */ 3165 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 3166 for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) { 3167 if (accel_module->supports_opcode(op)) { 3168 g_modules_opc[op].module = accel_module; 3169 SPDK_DEBUGLOG(accel, "OPC 0x%x now assigned to %s\n", op, accel_module->name); 3170 } 3171 } 3172 3173 if (accel_module->get_ctx_size != NULL) { 3174 g_max_accel_module_size = spdk_max(g_max_accel_module_size, 3175 accel_module->get_ctx_size()); 3176 } 3177 } 3178 3179 /* Now lets check for overrides and apply all that exist */ 3180 for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) { 3181 if (g_modules_opc_override[op] != NULL) { 3182 accel_module = _module_find_by_name(g_modules_opc_override[op]); 3183 if (accel_module == NULL) { 3184 SPDK_ERRLOG("Invalid module name of %s\n", g_modules_opc_override[op]); 3185 return -EINVAL; 3186 } 3187 if (accel_module->supports_opcode(op) == false) { 3188 SPDK_ERRLOG("Module %s does not support op code %d\n", accel_module->name, op); 3189 return -EINVAL; 3190 } 3191 g_modules_opc[op].module = accel_module; 3192 } 3193 } 3194 3195 if (g_modules_opc[SPDK_ACCEL_OPC_ENCRYPT].module != g_modules_opc[SPDK_ACCEL_OPC_DECRYPT].module) { 3196 SPDK_ERRLOG("Different accel modules are assigned to encrypt and decrypt operations"); 3197 return -EINVAL; 3198 } 3199 if (g_modules_opc[SPDK_ACCEL_OPC_COMPRESS].module != 3200 g_modules_opc[SPDK_ACCEL_OPC_DECOMPRESS].module) { 3201 SPDK_ERRLOG("Different accel modules are assigned to compress and decompress operations"); 3202 return -EINVAL; 3203 } 3204 3205 for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) { 3206 assert(g_modules_opc[op].module != NULL); 3207 accel_module_init_opcode(op); 3208 } 3209 3210 rc = spdk_iobuf_register_module("accel"); 3211 if (rc != 0) { 3212 SPDK_ERRLOG("Failed to register accel iobuf module\n"); 3213 return rc; 3214 } 3215 3216 return 0; 3217 } 3218 3219 static void 3220 accel_module_finish_cb(void) 3221 { 3222 spdk_accel_fini_cb cb_fn = g_fini_cb_fn; 3223 3224 cb_fn(g_fini_cb_arg); 3225 g_fini_cb_fn = NULL; 3226 g_fini_cb_arg = NULL; 3227 } 3228 3229 static void 3230 accel_write_overridden_opc(struct spdk_json_write_ctx *w, const char *opc_str, 3231 const char *module_str) 3232 { 3233 spdk_json_write_object_begin(w); 3234 spdk_json_write_named_string(w, "method", "accel_assign_opc"); 3235 spdk_json_write_named_object_begin(w, "params"); 3236 spdk_json_write_named_string(w, "opname", opc_str); 3237 spdk_json_write_named_string(w, "module", module_str); 3238 spdk_json_write_object_end(w); 3239 spdk_json_write_object_end(w); 3240 } 3241 3242 static void 3243 __accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key) 3244 { 3245 spdk_json_write_named_string(w, "name", key->param.key_name); 3246 spdk_json_write_named_string(w, "cipher", key->param.cipher); 3247 spdk_json_write_named_string(w, "key", key->param.hex_key); 3248 if (key->param.hex_key2) { 3249 spdk_json_write_named_string(w, "key2", key->param.hex_key2); 3250 } 3251 3252 if (key->param.tweak_mode) { 3253 spdk_json_write_named_string(w, "tweak_mode", key->param.tweak_mode); 3254 } 3255 } 3256 3257 void 3258 _accel_crypto_key_dump_param(struct spdk_json_write_ctx *w, struct spdk_accel_crypto_key *key) 3259 { 3260 spdk_json_write_object_begin(w); 3261 __accel_crypto_key_dump_param(w, key); 3262 spdk_json_write_object_end(w); 3263 } 3264 3265 static void 3266 _accel_crypto_key_write_config_json(struct spdk_json_write_ctx *w, 3267 struct spdk_accel_crypto_key *key) 3268 { 3269 spdk_json_write_object_begin(w); 3270 spdk_json_write_named_string(w, "method", "accel_crypto_key_create"); 3271 spdk_json_write_named_object_begin(w, "params"); 3272 __accel_crypto_key_dump_param(w, key); 3273 spdk_json_write_object_end(w); 3274 spdk_json_write_object_end(w); 3275 } 3276 3277 static void 3278 accel_write_options(struct spdk_json_write_ctx *w) 3279 { 3280 spdk_json_write_object_begin(w); 3281 spdk_json_write_named_string(w, "method", "accel_set_options"); 3282 spdk_json_write_named_object_begin(w, "params"); 3283 spdk_json_write_named_uint32(w, "small_cache_size", g_opts.small_cache_size); 3284 spdk_json_write_named_uint32(w, "large_cache_size", g_opts.large_cache_size); 3285 spdk_json_write_named_uint32(w, "task_count", g_opts.task_count); 3286 spdk_json_write_named_uint32(w, "sequence_count", g_opts.sequence_count); 3287 spdk_json_write_named_uint32(w, "buf_count", g_opts.buf_count); 3288 spdk_json_write_object_end(w); 3289 spdk_json_write_object_end(w); 3290 } 3291 3292 static void 3293 _accel_crypto_keys_write_config_json(struct spdk_json_write_ctx *w, bool full_dump) 3294 { 3295 struct spdk_accel_crypto_key *key; 3296 3297 spdk_spin_lock(&g_keyring_spin); 3298 TAILQ_FOREACH(key, &g_keyring, link) { 3299 if (full_dump) { 3300 _accel_crypto_key_write_config_json(w, key); 3301 } else { 3302 _accel_crypto_key_dump_param(w, key); 3303 } 3304 } 3305 spdk_spin_unlock(&g_keyring_spin); 3306 } 3307 3308 void 3309 _accel_crypto_keys_dump_param(struct spdk_json_write_ctx *w) 3310 { 3311 _accel_crypto_keys_write_config_json(w, false); 3312 } 3313 3314 void 3315 spdk_accel_write_config_json(struct spdk_json_write_ctx *w) 3316 { 3317 struct spdk_accel_module_if *accel_module; 3318 int i; 3319 3320 spdk_json_write_array_begin(w); 3321 accel_write_options(w); 3322 3323 if (g_accel_driver != NULL) { 3324 spdk_json_write_object_begin(w); 3325 spdk_json_write_named_string(w, "method", "accel_set_driver"); 3326 spdk_json_write_named_object_begin(w, "params"); 3327 spdk_json_write_named_string(w, "name", g_accel_driver->name); 3328 spdk_json_write_object_end(w); 3329 spdk_json_write_object_end(w); 3330 } 3331 3332 TAILQ_FOREACH(accel_module, &spdk_accel_module_list, tailq) { 3333 if (accel_module->write_config_json) { 3334 accel_module->write_config_json(w); 3335 } 3336 } 3337 for (i = 0; i < SPDK_ACCEL_OPC_LAST; i++) { 3338 if (g_modules_opc_override[i]) { 3339 accel_write_overridden_opc(w, g_opcode_strings[i], g_modules_opc_override[i]); 3340 } 3341 } 3342 3343 _accel_crypto_keys_write_config_json(w, true); 3344 3345 spdk_json_write_array_end(w); 3346 } 3347 3348 void 3349 spdk_accel_module_finish(void) 3350 { 3351 if (!g_accel_module) { 3352 g_accel_module = TAILQ_FIRST(&spdk_accel_module_list); 3353 } else { 3354 g_accel_module = TAILQ_NEXT(g_accel_module, tailq); 3355 } 3356 3357 if (!g_accel_module) { 3358 if (g_accel_driver != NULL && g_accel_driver->fini != NULL) { 3359 g_accel_driver->fini(); 3360 } 3361 3362 spdk_spin_destroy(&g_keyring_spin); 3363 spdk_spin_destroy(&g_stats_lock); 3364 if (g_accel_domain) { 3365 spdk_memory_domain_destroy(g_accel_domain); 3366 g_accel_domain = NULL; 3367 } 3368 accel_module_finish_cb(); 3369 return; 3370 } 3371 3372 if (g_accel_module->module_fini) { 3373 spdk_thread_send_msg(spdk_get_thread(), g_accel_module->module_fini, NULL); 3374 } else { 3375 spdk_accel_module_finish(); 3376 } 3377 } 3378 3379 static void 3380 accel_io_device_unregister_cb(void *io_device) 3381 { 3382 struct spdk_accel_crypto_key *key, *key_tmp; 3383 enum spdk_accel_opcode op; 3384 3385 spdk_spin_lock(&g_keyring_spin); 3386 TAILQ_FOREACH_SAFE(key, &g_keyring, link, key_tmp) { 3387 accel_crypto_key_destroy_unsafe(key); 3388 } 3389 spdk_spin_unlock(&g_keyring_spin); 3390 3391 for (op = 0; op < SPDK_ACCEL_OPC_LAST; op++) { 3392 if (g_modules_opc_override[op] != NULL) { 3393 free(g_modules_opc_override[op]); 3394 g_modules_opc_override[op] = NULL; 3395 } 3396 g_modules_opc[op].module = NULL; 3397 } 3398 3399 spdk_accel_module_finish(); 3400 } 3401 3402 void 3403 spdk_accel_finish(spdk_accel_fini_cb cb_fn, void *cb_arg) 3404 { 3405 assert(cb_fn != NULL); 3406 3407 g_fini_cb_fn = cb_fn; 3408 g_fini_cb_arg = cb_arg; 3409 3410 spdk_io_device_unregister(&spdk_accel_module_list, accel_io_device_unregister_cb); 3411 } 3412 3413 static struct spdk_accel_driver * 3414 accel_find_driver(const char *name) 3415 { 3416 struct spdk_accel_driver *driver; 3417 3418 TAILQ_FOREACH(driver, &g_accel_drivers, tailq) { 3419 if (strcmp(driver->name, name) == 0) { 3420 return driver; 3421 } 3422 } 3423 3424 return NULL; 3425 } 3426 3427 int 3428 spdk_accel_set_driver(const char *name) 3429 { 3430 struct spdk_accel_driver *driver = NULL; 3431 3432 if (name != NULL && name[0] != '\0') { 3433 driver = accel_find_driver(name); 3434 if (driver == NULL) { 3435 SPDK_ERRLOG("Couldn't find driver named '%s'\n", name); 3436 return -ENODEV; 3437 } 3438 } 3439 3440 g_accel_driver = driver; 3441 3442 return 0; 3443 } 3444 3445 const char * 3446 spdk_accel_get_driver_name(void) 3447 { 3448 if (!g_accel_driver) { 3449 return NULL; 3450 } 3451 3452 return g_accel_driver->name; 3453 } 3454 3455 void 3456 spdk_accel_driver_register(struct spdk_accel_driver *driver) 3457 { 3458 if (accel_find_driver(driver->name)) { 3459 SPDK_ERRLOG("Driver named '%s' has already been registered\n", driver->name); 3460 assert(0); 3461 return; 3462 } 3463 3464 TAILQ_INSERT_TAIL(&g_accel_drivers, driver, tailq); 3465 } 3466 3467 int 3468 spdk_accel_set_opts(const struct spdk_accel_opts *opts) 3469 { 3470 if (!opts) { 3471 SPDK_ERRLOG("opts cannot be NULL\n"); 3472 return -1; 3473 } 3474 3475 if (!opts->opts_size) { 3476 SPDK_ERRLOG("opts_size inside opts cannot be zero value\n"); 3477 return -1; 3478 } 3479 3480 if (SPDK_GET_FIELD(opts, task_count, g_opts.task_count, 3481 opts->opts_size) < ACCEL_TASKS_IN_SEQUENCE_LIMIT) { 3482 return -EINVAL; 3483 } 3484 3485 #define SET_FIELD(field) \ 3486 if (offsetof(struct spdk_accel_opts, field) + sizeof(opts->field) <= opts->opts_size) { \ 3487 g_opts.field = opts->field; \ 3488 } \ 3489 3490 SET_FIELD(small_cache_size); 3491 SET_FIELD(large_cache_size); 3492 SET_FIELD(task_count); 3493 SET_FIELD(sequence_count); 3494 SET_FIELD(buf_count); 3495 3496 g_opts.opts_size = opts->opts_size; 3497 3498 #undef SET_FIELD 3499 3500 return 0; 3501 } 3502 3503 void 3504 spdk_accel_get_opts(struct spdk_accel_opts *opts, size_t opts_size) 3505 { 3506 if (!opts) { 3507 SPDK_ERRLOG("opts should not be NULL\n"); 3508 return; 3509 } 3510 3511 if (!opts_size) { 3512 SPDK_ERRLOG("opts_size should not be zero value\n"); 3513 return; 3514 } 3515 3516 opts->opts_size = opts_size; 3517 3518 #define SET_FIELD(field) \ 3519 if (offsetof(struct spdk_accel_opts, field) + sizeof(opts->field) <= opts_size) { \ 3520 opts->field = g_opts.field; \ 3521 } \ 3522 3523 SET_FIELD(small_cache_size); 3524 SET_FIELD(large_cache_size); 3525 SET_FIELD(task_count); 3526 SET_FIELD(sequence_count); 3527 SET_FIELD(buf_count); 3528 3529 #undef SET_FIELD 3530 3531 /* Do not remove this statement, you should always update this statement when you adding a new field, 3532 * and do not forget to add the SET_FIELD statement for your added field. */ 3533 SPDK_STATIC_ASSERT(sizeof(struct spdk_accel_opts) == 28, "Incorrect size"); 3534 } 3535 3536 struct accel_get_stats_ctx { 3537 struct accel_stats stats; 3538 accel_get_stats_cb cb_fn; 3539 void *cb_arg; 3540 }; 3541 3542 static void 3543 accel_get_channel_stats_done(struct spdk_io_channel_iter *iter, int status) 3544 { 3545 struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter); 3546 3547 ctx->cb_fn(&ctx->stats, ctx->cb_arg); 3548 free(ctx); 3549 } 3550 3551 static void 3552 accel_get_channel_stats(struct spdk_io_channel_iter *iter) 3553 { 3554 struct spdk_io_channel *ch = spdk_io_channel_iter_get_channel(iter); 3555 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 3556 struct accel_get_stats_ctx *ctx = spdk_io_channel_iter_get_ctx(iter); 3557 3558 accel_add_stats(&ctx->stats, &accel_ch->stats); 3559 spdk_for_each_channel_continue(iter, 0); 3560 } 3561 3562 int 3563 accel_get_stats(accel_get_stats_cb cb_fn, void *cb_arg) 3564 { 3565 struct accel_get_stats_ctx *ctx; 3566 3567 ctx = calloc(1, sizeof(*ctx)); 3568 if (ctx == NULL) { 3569 return -ENOMEM; 3570 } 3571 3572 spdk_spin_lock(&g_stats_lock); 3573 accel_add_stats(&ctx->stats, &g_stats); 3574 spdk_spin_unlock(&g_stats_lock); 3575 3576 ctx->cb_fn = cb_fn; 3577 ctx->cb_arg = cb_arg; 3578 3579 spdk_for_each_channel(&spdk_accel_module_list, accel_get_channel_stats, ctx, 3580 accel_get_channel_stats_done); 3581 3582 return 0; 3583 } 3584 3585 void 3586 spdk_accel_get_opcode_stats(struct spdk_io_channel *ch, enum spdk_accel_opcode opcode, 3587 struct spdk_accel_opcode_stats *stats, size_t size) 3588 { 3589 struct accel_io_channel *accel_ch = spdk_io_channel_get_ctx(ch); 3590 3591 #define FIELD_OK(field) \ 3592 offsetof(struct spdk_accel_opcode_stats, field) + sizeof(stats->field) <= size 3593 3594 #define SET_FIELD(field, value) \ 3595 if (FIELD_OK(field)) { \ 3596 stats->field = value; \ 3597 } 3598 3599 SET_FIELD(executed, accel_ch->stats.operations[opcode].executed); 3600 SET_FIELD(failed, accel_ch->stats.operations[opcode].failed); 3601 SET_FIELD(num_bytes, accel_ch->stats.operations[opcode].num_bytes); 3602 3603 #undef FIELD_OK 3604 #undef SET_FIELD 3605 } 3606 3607 uint8_t 3608 spdk_accel_get_buf_align(enum spdk_accel_opcode opcode, 3609 const struct spdk_accel_operation_exec_ctx *ctx) 3610 { 3611 struct spdk_accel_module_if *module = g_modules_opc[opcode].module; 3612 struct spdk_accel_opcode_info modinfo = {}, drvinfo = {}; 3613 3614 if (g_accel_driver != NULL && g_accel_driver->get_operation_info != NULL) { 3615 g_accel_driver->get_operation_info(opcode, ctx, &drvinfo); 3616 } 3617 3618 if (module->get_operation_info != NULL) { 3619 module->get_operation_info(opcode, ctx, &modinfo); 3620 } 3621 3622 /* If a driver is set, it'll execute most of the operations, while the rest will usually 3623 * fall back to accel_sw, which doesn't have any alignment requirements. However, to be 3624 * extra safe, return the max(driver, module) if a driver delegates some operations to a 3625 * hardware module. */ 3626 return spdk_max(modinfo.required_alignment, drvinfo.required_alignment); 3627 } 3628 3629 struct spdk_accel_module_if * 3630 spdk_accel_get_module(const char *name) 3631 { 3632 struct spdk_accel_module_if *module; 3633 3634 TAILQ_FOREACH(module, &spdk_accel_module_list, tailq) { 3635 if (strcmp(module->name, name) == 0) { 3636 return module; 3637 } 3638 } 3639 3640 return NULL; 3641 } 3642 3643 int 3644 spdk_accel_get_opc_memory_domains(enum spdk_accel_opcode opcode, 3645 struct spdk_memory_domain **domains, 3646 int array_size) 3647 { 3648 assert(opcode < SPDK_ACCEL_OPC_LAST); 3649 3650 if (g_modules_opc[opcode].module->get_memory_domains) { 3651 return g_modules_opc[opcode].module->get_memory_domains(domains, array_size); 3652 } 3653 3654 return 0; 3655 } 3656 3657 SPDK_LOG_REGISTER_COMPONENT(accel) 3658