1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2022 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include "spdk_internal/accel_module.h" 9 10 #include "spdk/env.h" 11 #include "spdk/likely.h" 12 #include "spdk/log.h" 13 #include "spdk/thread.h" 14 #include "spdk/json.h" 15 #include "spdk/crc32.h" 16 #include "spdk/util.h" 17 18 #ifdef SPDK_CONFIG_PMDK 19 #include "libpmem.h" 20 #endif 21 22 #ifdef SPDK_CONFIG_ISAL 23 #include "../isa-l/include/igzip_lib.h" 24 #endif 25 26 struct sw_accel_io_channel { 27 /* for ISAL */ 28 #ifdef SPDK_CONFIG_ISAL 29 struct isal_zstream stream; 30 struct inflate_state state; 31 #endif 32 struct spdk_poller *completion_poller; 33 TAILQ_HEAD(, spdk_accel_task) tasks_to_complete; 34 }; 35 36 /* Post SW completions to a list and complete in a poller as we don't want to 37 * complete them on the caller's stack as they'll likely submit another. */ 38 inline static void 39 _add_to_comp_list(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task, int status) 40 { 41 accel_task->status = status; 42 TAILQ_INSERT_TAIL(&sw_ch->tasks_to_complete, accel_task, link); 43 } 44 45 /* Used when the SW engine is selected and the durable flag is set. */ 46 inline static int 47 _check_flags(int flags) 48 { 49 if (flags & ACCEL_FLAG_PERSISTENT) { 50 #ifndef SPDK_CONFIG_PMDK 51 /* PMDK is required to use this flag. */ 52 SPDK_ERRLOG("ACCEL_FLAG_PERSISTENT set but PMDK not configured. Configure PMDK or do not use this flag.\n"); 53 return -EINVAL; 54 #endif 55 } 56 return 0; 57 } 58 59 static bool 60 sw_accel_supports_opcode(enum accel_opcode opc) 61 { 62 switch (opc) { 63 case ACCEL_OPC_COPY: 64 case ACCEL_OPC_FILL: 65 case ACCEL_OPC_DUALCAST: 66 case ACCEL_OPC_COMPARE: 67 case ACCEL_OPC_CRC32C: 68 case ACCEL_OPC_COPY_CRC32C: 69 case ACCEL_OPC_COMPRESS: 70 case ACCEL_OPC_DECOMPRESS: 71 return true; 72 default: 73 return false; 74 } 75 } 76 77 static inline void 78 _pmem_memcpy(void *dst, const void *src, size_t len) 79 { 80 #ifdef SPDK_CONFIG_PMDK 81 int is_pmem = pmem_is_pmem(dst, len); 82 83 if (is_pmem) { 84 pmem_memcpy_persist(dst, src, len); 85 } else { 86 memcpy(dst, src, len); 87 pmem_msync(dst, len); 88 } 89 #else 90 SPDK_ERRLOG("Function not defined without SPDK_CONFIG_PMDK enabled.\n"); 91 assert(0); 92 #endif 93 } 94 95 static void 96 _sw_accel_dualcast(void *dst1, void *dst2, void *src, size_t nbytes, int flags) 97 { 98 if (flags & ACCEL_FLAG_PERSISTENT) { 99 _pmem_memcpy(dst1, src, nbytes); 100 _pmem_memcpy(dst2, src, nbytes); 101 } else { 102 memcpy(dst1, src, nbytes); 103 memcpy(dst2, src, nbytes); 104 } 105 } 106 107 static void 108 _sw_accel_copy(void *dst, void *src, size_t nbytes, int flags) 109 { 110 111 if (flags & ACCEL_FLAG_PERSISTENT) { 112 _pmem_memcpy(dst, src, nbytes); 113 } else { 114 memcpy(dst, src, nbytes); 115 } 116 } 117 118 static void 119 _sw_accel_copyv(void *dst, struct iovec *iov, uint32_t iovcnt, int flags) 120 { 121 uint32_t i; 122 123 for (i = 0; i < iovcnt; i++) { 124 assert(iov[i].iov_base != NULL); 125 if (flags & ACCEL_FLAG_PERSISTENT) { 126 _pmem_memcpy(dst, iov[i].iov_base, (size_t)iov[i].iov_len); 127 } else { 128 memcpy(dst, iov[i].iov_base, (size_t)iov[i].iov_len); 129 } 130 dst += iov[i].iov_len; 131 } 132 } 133 134 static int 135 _sw_accel_compare(void *src1, void *src2, size_t nbytes) 136 { 137 return memcmp(src1, src2, nbytes); 138 } 139 140 static void 141 _sw_accel_fill(void *dst, uint8_t fill, size_t nbytes, int flags) 142 { 143 if (flags & ACCEL_FLAG_PERSISTENT) { 144 #ifdef SPDK_CONFIG_PMDK 145 int is_pmem = pmem_is_pmem(dst, nbytes); 146 147 if (is_pmem) { 148 pmem_memset_persist(dst, fill, nbytes); 149 } else { 150 memset(dst, fill, nbytes); 151 pmem_msync(dst, nbytes); 152 } 153 #else 154 SPDK_ERRLOG("Function not defined without SPDK_CONFIG_PMDK enabled.\n"); 155 assert(0); 156 #endif 157 } else { 158 memset(dst, fill, nbytes); 159 } 160 } 161 162 static void 163 _sw_accel_crc32c(uint32_t *crc_dst, void *src, uint32_t seed, uint64_t nbytes) 164 { 165 *crc_dst = spdk_crc32c_update(src, nbytes, ~seed); 166 } 167 168 static void 169 _sw_accel_crc32cv(uint32_t *crc_dst, struct iovec *iov, uint32_t iovcnt, uint32_t seed) 170 { 171 *crc_dst = spdk_crc32c_iov_update(iov, iovcnt, ~seed); 172 } 173 174 static int 175 _sw_accel_compress(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 176 { 177 #ifdef SPDK_CONFIG_ISAL 178 size_t last_seglen = accel_task->s.iovs[accel_task->s.iovcnt - 1].iov_len; 179 struct iovec *siov = accel_task->s.iovs; 180 struct iovec *diov = accel_task->d.iovs; 181 size_t remaining = accel_task->nbytes; 182 uint32_t s = 0, d = 0; 183 int rc = 0; 184 185 accel_task->d.iovcnt = 1; 186 diov[0].iov_base = accel_task->dst; 187 diov[0].iov_len = accel_task->nbytes_dst; 188 189 isal_deflate_reset(&sw_ch->stream); 190 sw_ch->stream.end_of_stream = 0; 191 sw_ch->stream.next_out = diov[d].iov_base; 192 sw_ch->stream.avail_out = diov[d].iov_len; 193 sw_ch->stream.next_in = siov[s].iov_base; 194 sw_ch->stream.avail_in = siov[s].iov_len; 195 196 do { 197 /* if isal has exhausted the current dst iovec, move to the next 198 * one if there is one */ 199 if (sw_ch->stream.avail_out == 0) { 200 if (++d < accel_task->d.iovcnt) { 201 sw_ch->stream.next_out = diov[d].iov_base; 202 sw_ch->stream.avail_out = diov[d].iov_len; 203 assert(sw_ch->stream.avail_out > 0); 204 } else { 205 /* we have no avail_out but also no more iovecs left so this is 206 * the case where either the output buffer was a perfect fit 207 * or not enough was provided. Check the ISAL state to determine 208 * which. */ 209 if (sw_ch->stream.internal_state.state != ZSTATE_END) { 210 SPDK_ERRLOG("Not enough destination buffer provided.\n"); 211 rc = -ENOMEM; 212 } 213 break; 214 } 215 } 216 217 /* if isal has exhausted the current src iovec, move to the next 218 * one if there is one */ 219 if (sw_ch->stream.avail_in == 0 && ((s + 1) < accel_task->s.iovcnt)) { 220 s++; 221 sw_ch->stream.next_in = siov[s].iov_base; 222 sw_ch->stream.avail_in = siov[s].iov_len; 223 assert(sw_ch->stream.avail_in > 0); 224 } 225 226 if (remaining <= last_seglen) { 227 /* Need to set end of stream on last block */ 228 sw_ch->stream.end_of_stream = 1; 229 } 230 231 rc = isal_deflate(&sw_ch->stream); 232 if (rc) { 233 SPDK_ERRLOG("isal_deflate retunred error %d.\n", rc); 234 } 235 236 if (remaining > 0) { 237 assert(siov[s].iov_len > sw_ch->stream.avail_in); 238 remaining -= (siov[s].iov_len - sw_ch->stream.avail_in); 239 } 240 241 } while (remaining > 0 || sw_ch->stream.avail_out == 0); 242 assert(sw_ch->stream.avail_in == 0); 243 244 /* Get our total output size */ 245 if (accel_task->output_size != NULL) { 246 assert(sw_ch->stream.total_out > 0); 247 *accel_task->output_size = sw_ch->stream.total_out; 248 } 249 250 return rc; 251 #else 252 SPDK_ERRLOG("ISAL option is required to use software compression.\n"); 253 return -EINVAL; 254 #endif 255 } 256 257 static int 258 _sw_accel_decompress(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 259 { 260 #ifdef SPDK_CONFIG_ISAL 261 struct iovec *siov = accel_task->s.iovs; 262 struct iovec *diov = accel_task->d.iovs; 263 uint32_t s = 0, d = 0; 264 int rc = 0; 265 266 isal_inflate_reset(&sw_ch->state); 267 sw_ch->state.next_out = diov[d].iov_base; 268 sw_ch->state.avail_out = diov[d].iov_len; 269 sw_ch->state.next_in = siov[s].iov_base; 270 sw_ch->state.avail_in = siov[s].iov_len; 271 272 do { 273 /* if isal has exhausted the current dst iovec, move to the next 274 * one if there is one */ 275 if (sw_ch->state.avail_out == 0 && ((d + 1) < accel_task->d.iovcnt)) { 276 d++; 277 sw_ch->state.next_out = diov[d].iov_base; 278 sw_ch->state.avail_out = diov[d].iov_len; 279 assert(sw_ch->state.avail_out > 0); 280 } 281 282 /* if isal has exhausted the current src iovec, move to the next 283 * one if there is one */ 284 if (sw_ch->state.avail_in == 0 && ((s + 1) < accel_task->s.iovcnt)) { 285 s++; 286 sw_ch->state.next_in = siov[s].iov_base; 287 sw_ch->state.avail_in = siov[s].iov_len; 288 assert(sw_ch->state.avail_in > 0); 289 } 290 291 rc = isal_inflate(&sw_ch->state); 292 if (rc) { 293 SPDK_ERRLOG("isal_inflate retunred error %d.\n", rc); 294 } 295 296 } while (sw_ch->state.block_state < ISAL_BLOCK_FINISH); 297 assert(sw_ch->state.avail_in == 0); 298 299 return rc; 300 #else 301 SPDK_ERRLOG("ISAL option is required to use software decompression.\n"); 302 return -EINVAL; 303 #endif 304 } 305 306 static int 307 sw_accel_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task) 308 { 309 struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch); 310 struct spdk_accel_task *tmp; 311 int rc = 0; 312 313 do { 314 switch (accel_task->op_code) { 315 case ACCEL_OPC_COPY: 316 rc = _check_flags(accel_task->flags); 317 if (rc == 0) { 318 _sw_accel_copy(accel_task->dst, accel_task->src, accel_task->nbytes, accel_task->flags); 319 } 320 break; 321 case ACCEL_OPC_FILL: 322 rc = _check_flags(accel_task->flags); 323 if (rc == 0) { 324 _sw_accel_fill(accel_task->dst, accel_task->fill_pattern, accel_task->nbytes, accel_task->flags); 325 } 326 break; 327 case ACCEL_OPC_DUALCAST: 328 rc = _check_flags(accel_task->flags); 329 if (rc == 0) { 330 _sw_accel_dualcast(accel_task->dst, accel_task->dst2, accel_task->src, accel_task->nbytes, 331 accel_task->flags); 332 } 333 break; 334 case ACCEL_OPC_COMPARE: 335 rc = _sw_accel_compare(accel_task->src, accel_task->src2, accel_task->nbytes); 336 break; 337 case ACCEL_OPC_CRC32C: 338 if (accel_task->s.iovcnt == 0) { 339 _sw_accel_crc32c(accel_task->crc_dst, accel_task->src, accel_task->seed, accel_task->nbytes); 340 } else { 341 _sw_accel_crc32cv(accel_task->crc_dst, accel_task->s.iovs, accel_task->s.iovcnt, accel_task->seed); 342 } 343 break; 344 case ACCEL_OPC_COPY_CRC32C: 345 rc = _check_flags(accel_task->flags); 346 if (rc == 0) { 347 if (accel_task->s.iovcnt == 0) { 348 _sw_accel_copy(accel_task->dst, accel_task->src, accel_task->nbytes, accel_task->flags); 349 _sw_accel_crc32c(accel_task->crc_dst, accel_task->src, accel_task->seed, accel_task->nbytes); 350 } else { 351 _sw_accel_copyv(accel_task->dst, accel_task->s.iovs, accel_task->s.iovcnt, accel_task->flags); 352 _sw_accel_crc32cv(accel_task->crc_dst, accel_task->s.iovs, accel_task->s.iovcnt, accel_task->seed); 353 } 354 } 355 break; 356 case ACCEL_OPC_COMPRESS: 357 rc = _sw_accel_compress(sw_ch, accel_task); 358 break; 359 case ACCEL_OPC_DECOMPRESS: 360 rc = _sw_accel_decompress(sw_ch, accel_task); 361 break; 362 default: 363 assert(false); 364 break; 365 } 366 367 tmp = TAILQ_NEXT(accel_task, link); 368 369 _add_to_comp_list(sw_ch, accel_task, rc); 370 371 accel_task = tmp; 372 } while (accel_task); 373 374 return 0; 375 } 376 377 static struct spdk_io_channel *sw_accel_get_io_channel(void); 378 static int sw_accel_module_init(void); 379 static void sw_accel_module_fini(void *ctxt); 380 static size_t sw_accel_module_get_ctx_size(void); 381 382 static struct spdk_accel_module_if g_sw_module = { 383 .module_init = sw_accel_module_init, 384 .module_fini = sw_accel_module_fini, 385 .write_config_json = NULL, 386 .get_ctx_size = sw_accel_module_get_ctx_size, 387 .name = "software", 388 .supports_opcode = sw_accel_supports_opcode, 389 .get_io_channel = sw_accel_get_io_channel, 390 .submit_tasks = sw_accel_submit_tasks 391 }; 392 393 static int 394 accel_comp_poll(void *arg) 395 { 396 struct sw_accel_io_channel *sw_ch = arg; 397 TAILQ_HEAD(, spdk_accel_task) tasks_to_complete; 398 struct spdk_accel_task *accel_task; 399 400 if (TAILQ_EMPTY(&sw_ch->tasks_to_complete)) { 401 return SPDK_POLLER_IDLE; 402 } 403 404 TAILQ_INIT(&tasks_to_complete); 405 TAILQ_SWAP(&tasks_to_complete, &sw_ch->tasks_to_complete, spdk_accel_task, link); 406 407 while ((accel_task = TAILQ_FIRST(&tasks_to_complete))) { 408 TAILQ_REMOVE(&tasks_to_complete, accel_task, link); 409 spdk_accel_task_complete(accel_task, accel_task->status); 410 } 411 412 return SPDK_POLLER_BUSY; 413 } 414 415 static int 416 sw_accel_create_cb(void *io_device, void *ctx_buf) 417 { 418 struct sw_accel_io_channel *sw_ch = ctx_buf; 419 420 TAILQ_INIT(&sw_ch->tasks_to_complete); 421 sw_ch->completion_poller = SPDK_POLLER_REGISTER(accel_comp_poll, sw_ch, 0); 422 423 #ifdef SPDK_CONFIG_ISAL 424 isal_deflate_init(&sw_ch->stream); 425 sw_ch->stream.flush = NO_FLUSH; 426 sw_ch->stream.level = 1; 427 sw_ch->stream.level_buf = calloc(1, ISAL_DEF_LVL1_DEFAULT); 428 if (sw_ch->stream.level_buf == NULL) { 429 SPDK_ERRLOG("Could not allocate isal internal buffer\n"); 430 return -ENOMEM; 431 } 432 sw_ch->stream.level_buf_size = ISAL_DEF_LVL1_DEFAULT; 433 isal_inflate_init(&sw_ch->state); 434 #endif 435 436 return 0; 437 } 438 439 static void 440 sw_accel_destroy_cb(void *io_device, void *ctx_buf) 441 { 442 struct sw_accel_io_channel *sw_ch = ctx_buf; 443 444 #ifdef SPDK_CONFIG_ISAL 445 free(sw_ch->stream.level_buf); 446 #endif 447 448 spdk_poller_unregister(&sw_ch->completion_poller); 449 } 450 451 static struct spdk_io_channel * 452 sw_accel_get_io_channel(void) 453 { 454 return spdk_get_io_channel(&g_sw_module); 455 } 456 457 static size_t 458 sw_accel_module_get_ctx_size(void) 459 { 460 return sizeof(struct spdk_accel_task); 461 } 462 463 static int 464 sw_accel_module_init(void) 465 { 466 SPDK_NOTICELOG("Accel framework software module initialized.\n"); 467 spdk_io_device_register(&g_sw_module, sw_accel_create_cb, sw_accel_destroy_cb, 468 sizeof(struct sw_accel_io_channel), "sw_accel_module"); 469 470 return 0; 471 } 472 473 static void 474 sw_accel_module_fini(void *ctxt) 475 { 476 spdk_io_device_unregister(&g_sw_module, NULL); 477 spdk_accel_module_finish(); 478 } 479 480 SPDK_ACCEL_MODULE_REGISTER(sw, &g_sw_module) 481