1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include "spdk_internal/accel_module.h" 9 10 #include "spdk/env.h" 11 #include "spdk/likely.h" 12 #include "spdk/log.h" 13 #include "spdk/thread.h" 14 #include "spdk/json.h" 15 #include "spdk/crc32.h" 16 #include "spdk/util.h" 17 18 #ifdef SPDK_CONFIG_PMDK 19 #include "libpmem.h" 20 #endif 21 22 #ifdef SPDK_CONFIG_ISAL 23 #include "../isa-l/include/igzip_lib.h" 24 #endif 25 26 struct sw_accel_io_channel { 27 /* for ISAL */ 28 #ifdef SPDK_CONFIG_ISAL 29 struct isal_zstream stream; 30 struct inflate_state state; 31 #endif 32 struct spdk_poller *completion_poller; 33 TAILQ_HEAD(, spdk_accel_task) tasks_to_complete; 34 }; 35 36 /* Post SW completions to a list and complete in a poller as we don't want to 37 * complete them on the caller's stack as they'll likely submit another. */ 38 inline static void 39 _add_to_comp_list(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task, int status) 40 { 41 accel_task->status = status; 42 TAILQ_INSERT_TAIL(&sw_ch->tasks_to_complete, accel_task, link); 43 } 44 45 /* Used when the SW engine is selected and the durable flag is set. */ 46 inline static int 47 _check_flags(int flags) 48 { 49 if (flags & ACCEL_FLAG_PERSISTENT) { 50 #ifndef SPDK_CONFIG_PMDK 51 /* PMDK is required to use this flag. */ 52 SPDK_ERRLOG("ACCEL_FLAG_PERSISTENT set but PMDK not configured. Configure PMDK or do not use this flag.\n"); 53 return -EINVAL; 54 #endif 55 } 56 return 0; 57 } 58 59 static bool 60 sw_accel_supports_opcode(enum accel_opcode opc) 61 { 62 switch (opc) { 63 case ACCEL_OPC_COPY: 64 case ACCEL_OPC_FILL: 65 case ACCEL_OPC_DUALCAST: 66 case ACCEL_OPC_COMPARE: 67 case ACCEL_OPC_CRC32C: 68 case ACCEL_OPC_COPY_CRC32C: 69 case ACCEL_OPC_COMPRESS: 70 case ACCEL_OPC_DECOMPRESS: 71 return true; 72 default: 73 return false; 74 } 75 } 76 77 static inline void 78 _pmem_memcpy(void *dst, const void *src, size_t len) 79 { 80 #ifdef SPDK_CONFIG_PMDK 81 int is_pmem = pmem_is_pmem(dst, len); 82 83 if (is_pmem) { 84 pmem_memcpy_persist(dst, src, len); 85 } else { 86 memcpy(dst, src, len); 87 pmem_msync(dst, len); 88 } 89 #else 90 SPDK_ERRLOG("Function not defined without SPDK_CONFIG_PMDK enabled.\n"); 91 assert(0); 92 #endif 93 } 94 95 static void 96 _sw_accel_dualcast(void *dst1, void *dst2, void *src, size_t nbytes, int flags) 97 { 98 if (flags & ACCEL_FLAG_PERSISTENT) { 99 _pmem_memcpy(dst1, src, nbytes); 100 _pmem_memcpy(dst2, src, nbytes); 101 } else { 102 memcpy(dst1, src, nbytes); 103 memcpy(dst2, src, nbytes); 104 } 105 } 106 107 static void 108 _sw_accel_copy(void *dst, void *src, size_t nbytes, int flags) 109 { 110 111 if (flags & ACCEL_FLAG_PERSISTENT) { 112 _pmem_memcpy(dst, src, nbytes); 113 } else { 114 memcpy(dst, src, nbytes); 115 } 116 } 117 118 static void 119 _sw_accel_copyv(void *dst, struct iovec *iov, uint32_t iovcnt, int flags) 120 { 121 uint32_t i; 122 123 for (i = 0; i < iovcnt; i++) { 124 assert(iov[i].iov_base != NULL); 125 if (flags & ACCEL_FLAG_PERSISTENT) { 126 _pmem_memcpy(dst, iov[i].iov_base, (size_t)iov[i].iov_len); 127 } else { 128 memcpy(dst, iov[i].iov_base, (size_t)iov[i].iov_len); 129 } 130 dst += iov[i].iov_len; 131 } 132 } 133 134 static int 135 _sw_accel_compare(void *src1, void *src2, size_t nbytes) 136 { 137 return memcmp(src1, src2, nbytes); 138 } 139 140 static void 141 _sw_accel_fill(void *dst, uint8_t fill, size_t nbytes, int flags) 142 { 143 if (flags & ACCEL_FLAG_PERSISTENT) { 144 #ifdef SPDK_CONFIG_PMDK 145 int is_pmem = pmem_is_pmem(dst, nbytes); 146 147 if (is_pmem) { 148 pmem_memset_persist(dst, fill, nbytes); 149 } else { 150 memset(dst, fill, nbytes); 151 pmem_msync(dst, nbytes); 152 } 153 #else 154 SPDK_ERRLOG("Function not defined without SPDK_CONFIG_PMDK enabled.\n"); 155 assert(0); 156 #endif 157 } else { 158 memset(dst, fill, nbytes); 159 } 160 } 161 162 static void 163 _sw_accel_crc32c(uint32_t *crc_dst, void *src, uint32_t seed, uint64_t nbytes) 164 { 165 *crc_dst = spdk_crc32c_update(src, nbytes, ~seed); 166 } 167 168 static void 169 _sw_accel_crc32cv(uint32_t *crc_dst, struct iovec *iov, uint32_t iovcnt, uint32_t seed) 170 { 171 *crc_dst = spdk_crc32c_iov_update(iov, iovcnt, ~seed); 172 } 173 174 static int 175 _sw_accel_compress(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 176 { 177 #ifdef SPDK_CONFIG_ISAL 178 size_t last_seglen = accel_task->s.iovs[accel_task->s.iovcnt - 1].iov_len; 179 struct iovec *siov = accel_task->s.iovs; 180 struct iovec *diov = accel_task->d.iovs; 181 size_t remaining = accel_task->nbytes; 182 uint32_t s = 0, d = 0; 183 int rc = 0; 184 185 accel_task->d.iovcnt = 1; 186 diov[0].iov_base = accel_task->dst; 187 diov[0].iov_len = accel_task->nbytes_dst; 188 189 isal_deflate_reset(&sw_ch->stream); 190 sw_ch->stream.end_of_stream = 0; 191 sw_ch->stream.next_out = diov[d].iov_base; 192 sw_ch->stream.avail_out = diov[d].iov_len; 193 sw_ch->stream.next_in = siov[s].iov_base; 194 sw_ch->stream.avail_in = siov[s].iov_len; 195 196 do { 197 /* if isal has exhausted the current dst iovec, move to the next 198 * one if there is one */ 199 if (sw_ch->stream.avail_out == 0) { 200 if (++d < accel_task->d.iovcnt) { 201 sw_ch->stream.next_out = diov[d].iov_base; 202 sw_ch->stream.avail_out = diov[d].iov_len; 203 assert(sw_ch->stream.avail_out > 0); 204 } else { 205 /* we have no avail_out but also no more iovecs left so this is 206 * the case where the output buffer was a perfect fit for the 207 * compressed data and we're done. */ 208 break; 209 } 210 } 211 212 /* if isal has exhausted the current src iovec, move to the next 213 * one if there is one */ 214 if (sw_ch->stream.avail_in == 0 && ((s + 1) < accel_task->s.iovcnt)) { 215 s++; 216 sw_ch->stream.next_in = siov[s].iov_base; 217 sw_ch->stream.avail_in = siov[s].iov_len; 218 assert(sw_ch->stream.avail_in > 0); 219 } 220 221 if (remaining <= last_seglen) { 222 /* Need to set end of stream on last block */ 223 sw_ch->stream.end_of_stream = 1; 224 } 225 226 rc = isal_deflate(&sw_ch->stream); 227 if (rc) { 228 SPDK_ERRLOG("isal_deflate retunred error %d.\n", rc); 229 } 230 231 if (remaining > 0) { 232 assert(siov[s].iov_len > sw_ch->stream.avail_in); 233 remaining -= (siov[s].iov_len - sw_ch->stream.avail_in); 234 } 235 236 } while (remaining > 0 || sw_ch->stream.avail_out == 0); 237 assert(sw_ch->stream.avail_in == 0); 238 239 /* Get our total output size */ 240 if (accel_task->output_size != NULL) { 241 assert(sw_ch->stream.total_out > 0); 242 *accel_task->output_size = sw_ch->stream.total_out; 243 } 244 245 return rc; 246 #else 247 SPDK_ERRLOG("ISAL option is required to use software compression.\n"); 248 return -EINVAL; 249 #endif 250 } 251 252 static int 253 _sw_accel_decompress(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 254 { 255 #ifdef SPDK_CONFIG_ISAL 256 struct iovec *siov = accel_task->s.iovs; 257 struct iovec *diov = accel_task->d.iovs; 258 uint32_t s = 0, d = 0; 259 int rc = 0; 260 261 isal_inflate_reset(&sw_ch->state); 262 sw_ch->state.next_out = diov[d].iov_base; 263 sw_ch->state.avail_out = diov[d].iov_len; 264 sw_ch->state.next_in = siov[s].iov_base; 265 sw_ch->state.avail_in = siov[s].iov_len; 266 267 do { 268 /* if isal has exhausted the current dst iovec, move to the next 269 * one if there is one */ 270 if (sw_ch->state.avail_out == 0 && ((d + 1) < accel_task->d.iovcnt)) { 271 d++; 272 sw_ch->state.next_out = diov[d].iov_base; 273 sw_ch->state.avail_out = diov[d].iov_len; 274 assert(sw_ch->state.avail_out > 0); 275 } 276 277 /* if isal has exhausted the current src iovec, move to the next 278 * one if there is one */ 279 if (sw_ch->state.avail_in == 0 && ((s + 1) < accel_task->s.iovcnt)) { 280 s++; 281 sw_ch->state.next_in = siov[s].iov_base; 282 sw_ch->state.avail_in = siov[s].iov_len; 283 assert(sw_ch->state.avail_in > 0); 284 } 285 286 rc = isal_inflate(&sw_ch->state); 287 if (rc) { 288 SPDK_ERRLOG("isal_inflate retunred error %d.\n", rc); 289 } 290 291 } while (sw_ch->state.block_state < ISAL_BLOCK_FINISH); 292 assert(sw_ch->state.avail_in == 0); 293 294 return rc; 295 #else 296 SPDK_ERRLOG("ISAL option is required to use software decompression.\n"); 297 return -EINVAL; 298 #endif 299 } 300 301 static int 302 sw_accel_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task) 303 { 304 struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch); 305 struct spdk_accel_task *tmp; 306 int rc = 0; 307 308 do { 309 switch (accel_task->op_code) { 310 case ACCEL_OPC_COPY: 311 rc = _check_flags(accel_task->flags); 312 if (rc == 0) { 313 _sw_accel_copy(accel_task->dst, accel_task->src, accel_task->nbytes, accel_task->flags); 314 } 315 break; 316 case ACCEL_OPC_FILL: 317 rc = _check_flags(accel_task->flags); 318 if (rc == 0) { 319 _sw_accel_fill(accel_task->dst, accel_task->fill_pattern, accel_task->nbytes, accel_task->flags); 320 } 321 break; 322 case ACCEL_OPC_DUALCAST: 323 rc = _check_flags(accel_task->flags); 324 if (rc == 0) { 325 _sw_accel_dualcast(accel_task->dst, accel_task->dst2, accel_task->src, accel_task->nbytes, 326 accel_task->flags); 327 } 328 break; 329 case ACCEL_OPC_COMPARE: 330 rc = _sw_accel_compare(accel_task->src, accel_task->src2, accel_task->nbytes); 331 break; 332 case ACCEL_OPC_CRC32C: 333 if (accel_task->s.iovcnt == 0) { 334 _sw_accel_crc32c(accel_task->crc_dst, accel_task->src, accel_task->seed, accel_task->nbytes); 335 } else { 336 _sw_accel_crc32cv(accel_task->crc_dst, accel_task->s.iovs, accel_task->s.iovcnt, accel_task->seed); 337 } 338 break; 339 case ACCEL_OPC_COPY_CRC32C: 340 rc = _check_flags(accel_task->flags); 341 if (rc == 0) { 342 if (accel_task->s.iovcnt == 0) { 343 _sw_accel_copy(accel_task->dst, accel_task->src, accel_task->nbytes, accel_task->flags); 344 _sw_accel_crc32c(accel_task->crc_dst, accel_task->src, accel_task->seed, accel_task->nbytes); 345 } else { 346 _sw_accel_copyv(accel_task->dst, accel_task->s.iovs, accel_task->s.iovcnt, accel_task->flags); 347 _sw_accel_crc32cv(accel_task->crc_dst, accel_task->s.iovs, accel_task->s.iovcnt, accel_task->seed); 348 } 349 } 350 break; 351 case ACCEL_OPC_COMPRESS: 352 rc = _sw_accel_compress(sw_ch, accel_task); 353 break; 354 case ACCEL_OPC_DECOMPRESS: 355 rc = _sw_accel_decompress(sw_ch, accel_task); 356 break; 357 default: 358 assert(false); 359 break; 360 } 361 362 tmp = TAILQ_NEXT(accel_task, link); 363 364 _add_to_comp_list(sw_ch, accel_task, rc); 365 366 accel_task = tmp; 367 } while (accel_task); 368 369 return 0; 370 } 371 372 static struct spdk_io_channel *sw_accel_get_io_channel(void); 373 static int sw_accel_module_init(void); 374 static void sw_accel_module_fini(void *ctxt); 375 static size_t sw_accel_module_get_ctx_size(void); 376 377 static struct spdk_accel_module_if g_sw_module = { 378 .module_init = sw_accel_module_init, 379 .module_fini = sw_accel_module_fini, 380 .write_config_json = NULL, 381 .get_ctx_size = sw_accel_module_get_ctx_size, 382 .name = "software", 383 .supports_opcode = sw_accel_supports_opcode, 384 .get_io_channel = sw_accel_get_io_channel, 385 .submit_tasks = sw_accel_submit_tasks 386 }; 387 388 static int 389 accel_comp_poll(void *arg) 390 { 391 struct sw_accel_io_channel *sw_ch = arg; 392 TAILQ_HEAD(, spdk_accel_task) tasks_to_complete; 393 struct spdk_accel_task *accel_task; 394 395 if (TAILQ_EMPTY(&sw_ch->tasks_to_complete)) { 396 return SPDK_POLLER_IDLE; 397 } 398 399 TAILQ_INIT(&tasks_to_complete); 400 TAILQ_SWAP(&tasks_to_complete, &sw_ch->tasks_to_complete, spdk_accel_task, link); 401 402 while ((accel_task = TAILQ_FIRST(&tasks_to_complete))) { 403 TAILQ_REMOVE(&tasks_to_complete, accel_task, link); 404 spdk_accel_task_complete(accel_task, accel_task->status); 405 } 406 407 return SPDK_POLLER_BUSY; 408 } 409 410 static int 411 sw_accel_create_cb(void *io_device, void *ctx_buf) 412 { 413 struct sw_accel_io_channel *sw_ch = ctx_buf; 414 415 TAILQ_INIT(&sw_ch->tasks_to_complete); 416 sw_ch->completion_poller = SPDK_POLLER_REGISTER(accel_comp_poll, sw_ch, 0); 417 418 #ifdef SPDK_CONFIG_ISAL 419 isal_deflate_init(&sw_ch->stream); 420 sw_ch->stream.flush = NO_FLUSH; 421 sw_ch->stream.level = 1; 422 sw_ch->stream.level_buf = calloc(1, ISAL_DEF_LVL1_DEFAULT); 423 if (sw_ch->stream.level_buf == NULL) { 424 SPDK_ERRLOG("Could not allocate isal internal buffer\n"); 425 return -ENOMEM; 426 } 427 sw_ch->stream.level_buf_size = ISAL_DEF_LVL1_DEFAULT; 428 isal_inflate_init(&sw_ch->state); 429 #endif 430 431 return 0; 432 } 433 434 static void 435 sw_accel_destroy_cb(void *io_device, void *ctx_buf) 436 { 437 struct sw_accel_io_channel *sw_ch = ctx_buf; 438 439 #ifdef SPDK_CONFIG_ISAL 440 free(sw_ch->stream.level_buf); 441 #endif 442 443 spdk_poller_unregister(&sw_ch->completion_poller); 444 } 445 446 static struct spdk_io_channel * 447 sw_accel_get_io_channel(void) 448 { 449 return spdk_get_io_channel(&g_sw_module); 450 } 451 452 static size_t 453 sw_accel_module_get_ctx_size(void) 454 { 455 return sizeof(struct spdk_accel_task); 456 } 457 458 static int 459 sw_accel_module_init(void) 460 { 461 SPDK_NOTICELOG("Accel framework software module initialized.\n"); 462 spdk_io_device_register(&g_sw_module, sw_accel_create_cb, sw_accel_destroy_cb, 463 sizeof(struct sw_accel_io_channel), "sw_accel_module"); 464 465 return 0; 466 } 467 468 static void 469 sw_accel_module_fini(void *ctxt) 470 { 471 spdk_io_device_unregister(&g_sw_module, NULL); 472 spdk_accel_module_finish(); 473 } 474 475 SPDK_ACCEL_MODULE_REGISTER(sw, &g_sw_module) 476