1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include "spdk/stdinc.h" 7 8 #include "spdk_internal/accel_module.h" 9 10 #include "spdk/env.h" 11 #include "spdk/likely.h" 12 #include "spdk/log.h" 13 #include "spdk/thread.h" 14 #include "spdk/json.h" 15 #include "spdk/crc32.h" 16 #include "spdk/util.h" 17 18 #ifdef SPDK_CONFIG_PMDK 19 #include "libpmem.h" 20 #endif 21 22 #ifdef SPDK_CONFIG_ISAL 23 #include "../isa-l/include/igzip_lib.h" 24 #endif 25 26 struct sw_accel_io_channel { 27 /* for ISAL */ 28 #ifdef SPDK_CONFIG_ISAL 29 struct isal_zstream stream; 30 struct inflate_state state; 31 #endif 32 struct spdk_poller *completion_poller; 33 TAILQ_HEAD(, spdk_accel_task) tasks_to_complete; 34 }; 35 36 /* Post SW completions to a list and complete in a poller as we don't want to 37 * complete them on the caller's stack as they'll likely submit another. */ 38 inline static void 39 _add_to_comp_list(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task, int status) 40 { 41 accel_task->status = status; 42 TAILQ_INSERT_TAIL(&sw_ch->tasks_to_complete, accel_task, link); 43 } 44 45 /* Used when the SW engine is selected and the durable flag is set. */ 46 inline static int 47 _check_flags(int flags) 48 { 49 if (flags & ACCEL_FLAG_PERSISTENT) { 50 #ifndef SPDK_CONFIG_PMDK 51 /* PMDK is required to use this flag. */ 52 SPDK_ERRLOG("ACCEL_FLAG_PERSISTENT set but PMDK not configured. Configure PMDK or do not use this flag.\n"); 53 return -EINVAL; 54 #endif 55 } 56 return 0; 57 } 58 59 static bool 60 sw_accel_supports_opcode(enum accel_opcode opc) 61 { 62 switch (opc) { 63 case ACCEL_OPC_COPY: 64 case ACCEL_OPC_FILL: 65 case ACCEL_OPC_DUALCAST: 66 case ACCEL_OPC_COMPARE: 67 case ACCEL_OPC_CRC32C: 68 case ACCEL_OPC_COPY_CRC32C: 69 case ACCEL_OPC_COMPRESS: 70 case ACCEL_OPC_DECOMPRESS: 71 return true; 72 default: 73 return false; 74 } 75 } 76 77 static inline void 78 _pmem_memcpy(void *dst, const void *src, size_t len) 79 { 80 #ifdef SPDK_CONFIG_PMDK 81 int is_pmem = pmem_is_pmem(dst, len); 82 83 if (is_pmem) { 84 pmem_memcpy_persist(dst, src, len); 85 } else { 86 memcpy(dst, src, len); 87 pmem_msync(dst, len); 88 } 89 #else 90 SPDK_ERRLOG("Function not defined without SPDK_CONFIG_PMDK enabled.\n"); 91 assert(0); 92 #endif 93 } 94 95 static void 96 _sw_accel_dualcast(void *dst1, void *dst2, void *src, size_t nbytes, int flags) 97 { 98 if (flags & ACCEL_FLAG_PERSISTENT) { 99 _pmem_memcpy(dst1, src, nbytes); 100 _pmem_memcpy(dst2, src, nbytes); 101 } else { 102 memcpy(dst1, src, nbytes); 103 memcpy(dst2, src, nbytes); 104 } 105 } 106 107 static void 108 _sw_accel_copy(void *dst, void *src, size_t nbytes, int flags) 109 { 110 111 if (flags & ACCEL_FLAG_PERSISTENT) { 112 _pmem_memcpy(dst, src, nbytes); 113 } else { 114 memcpy(dst, src, nbytes); 115 } 116 } 117 118 static void 119 _sw_accel_copyv(void *dst, struct iovec *iov, uint32_t iovcnt, int flags) 120 { 121 uint32_t i; 122 123 for (i = 0; i < iovcnt; i++) { 124 assert(iov[i].iov_base != NULL); 125 if (flags & ACCEL_FLAG_PERSISTENT) { 126 _pmem_memcpy(dst, iov[i].iov_base, (size_t)iov[i].iov_len); 127 } else { 128 memcpy(dst, iov[i].iov_base, (size_t)iov[i].iov_len); 129 } 130 dst += iov[i].iov_len; 131 } 132 } 133 134 static int 135 _sw_accel_compare(void *src1, void *src2, size_t nbytes) 136 { 137 return memcmp(src1, src2, nbytes); 138 } 139 140 static void 141 _sw_accel_fill(void *dst, uint8_t fill, size_t nbytes, int flags) 142 { 143 if (flags & ACCEL_FLAG_PERSISTENT) { 144 #ifdef SPDK_CONFIG_PMDK 145 int is_pmem = pmem_is_pmem(dst, nbytes); 146 147 if (is_pmem) { 148 pmem_memset_persist(dst, fill, nbytes); 149 } else { 150 memset(dst, fill, nbytes); 151 pmem_msync(dst, nbytes); 152 } 153 #else 154 SPDK_ERRLOG("Function not defined without SPDK_CONFIG_PMDK enabled.\n"); 155 assert(0); 156 #endif 157 } else { 158 memset(dst, fill, nbytes); 159 } 160 } 161 162 static void 163 _sw_accel_crc32c(uint32_t *crc_dst, void *src, uint32_t seed, uint64_t nbytes) 164 { 165 *crc_dst = spdk_crc32c_update(src, nbytes, ~seed); 166 } 167 168 static void 169 _sw_accel_crc32cv(uint32_t *crc_dst, struct iovec *iov, uint32_t iovcnt, uint32_t seed) 170 { 171 *crc_dst = spdk_crc32c_iov_update(iov, iovcnt, ~seed); 172 } 173 174 static int 175 _sw_accel_compress(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 176 { 177 #ifdef SPDK_CONFIG_ISAL 178 sw_ch->stream.next_in = accel_task->src; 179 sw_ch->stream.next_out = accel_task->dst; 180 sw_ch->stream.avail_in = accel_task->nbytes; 181 sw_ch->stream.avail_out = accel_task->nbytes_dst; 182 183 isal_deflate_stateless(&sw_ch->stream); 184 if (accel_task->output_size != NULL) { 185 assert(accel_task->nbytes_dst > sw_ch->stream.avail_out); 186 *accel_task->output_size = accel_task->nbytes_dst - sw_ch->stream.avail_out; 187 } 188 189 return 0; 190 #else 191 SPDK_ERRLOG("ISAL option is required to use software compression.\n"); 192 return -EINVAL; 193 #endif 194 } 195 196 static int 197 _sw_accel_decompress(struct sw_accel_io_channel *sw_ch, struct spdk_accel_task *accel_task) 198 { 199 #ifdef SPDK_CONFIG_ISAL 200 int rc; 201 202 sw_ch->state.next_in = accel_task->src; 203 sw_ch->state.avail_in = accel_task->nbytes; 204 sw_ch->state.next_out = accel_task->dst; 205 sw_ch->state.avail_out = accel_task->nbytes_dst; 206 207 rc = isal_inflate_stateless(&sw_ch->state); 208 if (rc) { 209 SPDK_ERRLOG("isal_inflate_stateless retunred error %d.\n", rc); 210 } 211 return rc; 212 #else 213 SPDK_ERRLOG("ISAL option is required to use software decompression.\n"); 214 return -EINVAL; 215 #endif 216 } 217 218 static int 219 sw_accel_submit_tasks(struct spdk_io_channel *ch, struct spdk_accel_task *accel_task) 220 { 221 struct sw_accel_io_channel *sw_ch = spdk_io_channel_get_ctx(ch); 222 struct spdk_accel_task *tmp; 223 int rc = 0; 224 225 do { 226 switch (accel_task->op_code) { 227 case ACCEL_OPC_COPY: 228 rc = _check_flags(accel_task->flags); 229 if (rc == 0) { 230 _sw_accel_copy(accel_task->dst, accel_task->src, accel_task->nbytes, accel_task->flags); 231 } 232 break; 233 case ACCEL_OPC_FILL: 234 rc = _check_flags(accel_task->flags); 235 if (rc == 0) { 236 _sw_accel_fill(accel_task->dst, accel_task->fill_pattern, accel_task->nbytes, accel_task->flags); 237 } 238 break; 239 case ACCEL_OPC_DUALCAST: 240 rc = _check_flags(accel_task->flags); 241 if (rc == 0) { 242 _sw_accel_dualcast(accel_task->dst, accel_task->dst2, accel_task->src, accel_task->nbytes, 243 accel_task->flags); 244 } 245 break; 246 case ACCEL_OPC_COMPARE: 247 rc = _sw_accel_compare(accel_task->src, accel_task->src2, accel_task->nbytes); 248 break; 249 case ACCEL_OPC_CRC32C: 250 if (accel_task->v.iovcnt == 0) { 251 _sw_accel_crc32c(accel_task->crc_dst, accel_task->src, accel_task->seed, accel_task->nbytes); 252 } else { 253 _sw_accel_crc32cv(accel_task->crc_dst, accel_task->v.iovs, accel_task->v.iovcnt, accel_task->seed); 254 } 255 break; 256 case ACCEL_OPC_COPY_CRC32C: 257 rc = _check_flags(accel_task->flags); 258 if (rc == 0) { 259 if (accel_task->v.iovcnt == 0) { 260 _sw_accel_copy(accel_task->dst, accel_task->src, accel_task->nbytes, accel_task->flags); 261 _sw_accel_crc32c(accel_task->crc_dst, accel_task->src, accel_task->seed, accel_task->nbytes); 262 } else { 263 _sw_accel_copyv(accel_task->dst, accel_task->v.iovs, accel_task->v.iovcnt, accel_task->flags); 264 _sw_accel_crc32cv(accel_task->crc_dst, accel_task->v.iovs, accel_task->v.iovcnt, accel_task->seed); 265 } 266 } 267 break; 268 case ACCEL_OPC_COMPRESS: 269 rc = _sw_accel_compress(sw_ch, accel_task); 270 break; 271 case ACCEL_OPC_DECOMPRESS: 272 rc = _sw_accel_decompress(sw_ch, accel_task); 273 break; 274 default: 275 assert(false); 276 break; 277 } 278 279 tmp = TAILQ_NEXT(accel_task, link); 280 281 _add_to_comp_list(sw_ch, accel_task, rc); 282 283 accel_task = tmp; 284 } while (accel_task); 285 286 return 0; 287 } 288 289 static struct spdk_io_channel *sw_accel_get_io_channel(void); 290 static int sw_accel_module_init(void); 291 static void sw_accel_module_fini(void *ctxt); 292 static size_t sw_accel_module_get_ctx_size(void); 293 294 static struct spdk_accel_module_if g_sw_module = { 295 .module_init = sw_accel_module_init, 296 .module_fini = sw_accel_module_fini, 297 .write_config_json = NULL, 298 .get_ctx_size = sw_accel_module_get_ctx_size, 299 .name = "software", 300 .supports_opcode = sw_accel_supports_opcode, 301 .get_io_channel = sw_accel_get_io_channel, 302 .submit_tasks = sw_accel_submit_tasks 303 }; 304 305 static int 306 accel_comp_poll(void *arg) 307 { 308 struct sw_accel_io_channel *sw_ch = arg; 309 TAILQ_HEAD(, spdk_accel_task) tasks_to_complete; 310 struct spdk_accel_task *accel_task; 311 312 if (TAILQ_EMPTY(&sw_ch->tasks_to_complete)) { 313 return SPDK_POLLER_IDLE; 314 } 315 316 TAILQ_INIT(&tasks_to_complete); 317 TAILQ_SWAP(&tasks_to_complete, &sw_ch->tasks_to_complete, spdk_accel_task, link); 318 319 while ((accel_task = TAILQ_FIRST(&tasks_to_complete))) { 320 TAILQ_REMOVE(&tasks_to_complete, accel_task, link); 321 spdk_accel_task_complete(accel_task, accel_task->status); 322 } 323 324 return SPDK_POLLER_BUSY; 325 } 326 327 static int 328 sw_accel_create_cb(void *io_device, void *ctx_buf) 329 { 330 struct sw_accel_io_channel *sw_ch = ctx_buf; 331 332 TAILQ_INIT(&sw_ch->tasks_to_complete); 333 sw_ch->completion_poller = SPDK_POLLER_REGISTER(accel_comp_poll, sw_ch, 0); 334 335 #ifdef SPDK_CONFIG_ISAL 336 isal_deflate_stateless_init(&sw_ch->stream); 337 sw_ch->stream.level = 1; 338 sw_ch->stream.level_buf = calloc(1, ISAL_DEF_LVL1_DEFAULT); 339 if (sw_ch->stream.level_buf == NULL) { 340 SPDK_ERRLOG("Could not allocate isal internal buffer\n"); 341 return -ENOMEM; 342 } 343 sw_ch->stream.level_buf_size = ISAL_DEF_LVL1_DEFAULT; 344 isal_inflate_init(&sw_ch->state); 345 #endif 346 347 return 0; 348 } 349 350 static void 351 sw_accel_destroy_cb(void *io_device, void *ctx_buf) 352 { 353 struct sw_accel_io_channel *sw_ch = ctx_buf; 354 355 #ifdef SPDK_CONFIG_ISAL 356 free(sw_ch->stream.level_buf); 357 #endif 358 359 spdk_poller_unregister(&sw_ch->completion_poller); 360 } 361 362 static struct spdk_io_channel * 363 sw_accel_get_io_channel(void) 364 { 365 return spdk_get_io_channel(&g_sw_module); 366 } 367 368 static size_t 369 sw_accel_module_get_ctx_size(void) 370 { 371 return sizeof(struct spdk_accel_task); 372 } 373 374 static int 375 sw_accel_module_init(void) 376 { 377 SPDK_NOTICELOG("Accel framework software module initialized.\n"); 378 spdk_io_device_register(&g_sw_module, sw_accel_create_cb, sw_accel_destroy_cb, 379 sizeof(struct sw_accel_io_channel), "sw_accel_module"); 380 381 return 0; 382 } 383 384 static void 385 sw_accel_module_fini(void *ctxt) 386 { 387 spdk_io_device_unregister(&g_sw_module, NULL); 388 spdk_accel_module_finish(); 389 } 390 391 SPDK_ACCEL_MODULE_REGISTER(sw, &g_sw_module) 392