1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (c) Intel Corporation. 3 * All rights reserved. 4 */ 5 #include <ocf/ocf.h> 6 #include <execinfo.h> 7 8 #include "spdk/env.h" 9 #include "spdk/log.h" 10 11 #include "ctx.h" 12 #include "data.h" 13 14 ocf_ctx_t vbdev_ocf_ctx; 15 16 static ctx_data_t * 17 vbdev_ocf_ctx_data_alloc(uint32_t pages) 18 { 19 struct bdev_ocf_data *data; 20 void *buf; 21 uint32_t sz; 22 23 data = vbdev_ocf_data_alloc(1); 24 25 sz = pages * PAGE_SIZE; 26 buf = spdk_malloc(sz, PAGE_SIZE, NULL, 27 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 28 if (buf == NULL) { 29 return NULL; 30 } 31 32 vbdev_ocf_iovs_add(data, buf, sz); 33 34 data->size = sz; 35 36 return data; 37 } 38 39 static void 40 vbdev_ocf_ctx_data_free(ctx_data_t *ctx_data) 41 { 42 struct bdev_ocf_data *data = ctx_data; 43 int i; 44 45 if (!data) { 46 return; 47 } 48 49 for (i = 0; i < data->iovcnt; i++) { 50 spdk_free(data->iovs[i].iov_base); 51 } 52 53 vbdev_ocf_data_free(data); 54 } 55 56 static int 57 vbdev_ocf_ctx_data_mlock(ctx_data_t *ctx_data) 58 { 59 /* TODO [mlock]: add mlock option */ 60 return 0; 61 } 62 63 static void 64 vbdev_ocf_ctx_data_munlock(ctx_data_t *ctx_data) 65 { 66 /* TODO [mlock]: add mlock option */ 67 } 68 69 static size_t 70 iovec_flatten(struct iovec *iov, size_t iovcnt, void *buf, size_t size, size_t offset) 71 { 72 size_t i, len, done = 0; 73 74 for (i = 0; i < iovcnt; i++) { 75 if (offset >= iov[i].iov_len) { 76 offset -= iov[i].iov_len; 77 continue; 78 } 79 80 if (iov[i].iov_base == NULL) { 81 continue; 82 } 83 84 if (done >= size) { 85 break; 86 } 87 88 len = MIN(size - done, iov[i].iov_len - offset); 89 memcpy(buf, iov[i].iov_base + offset, len); 90 buf += len; 91 done += len; 92 offset = 0; 93 } 94 95 return done; 96 } 97 98 static uint32_t 99 vbdev_ocf_ctx_data_rd(void *dst, ctx_data_t *src, uint32_t size) 100 { 101 struct bdev_ocf_data *s = src; 102 uint32_t size_local; 103 104 size_local = iovec_flatten(s->iovs, s->iovcnt, dst, size, s->seek); 105 s->seek += size_local; 106 107 return size_local; 108 } 109 110 static size_t 111 buf_to_iovec(const void *buf, size_t size, struct iovec *iov, size_t iovcnt, size_t offset) 112 { 113 size_t i, len, done = 0; 114 115 for (i = 0; i < iovcnt; i++) { 116 if (offset >= iov[i].iov_len) { 117 offset -= iov[i].iov_len; 118 continue; 119 } 120 121 if (iov[i].iov_base == NULL) { 122 continue; 123 } 124 125 if (done >= size) { 126 break; 127 } 128 129 len = MIN(size - done, iov[i].iov_len - offset); 130 memcpy(iov[i].iov_base + offset, buf, len); 131 buf += len; 132 done += len; 133 offset = 0; 134 } 135 136 return done; 137 } 138 139 static uint32_t 140 vbdev_ocf_ctx_data_wr(ctx_data_t *dst, const void *src, uint32_t size) 141 { 142 struct bdev_ocf_data *d = dst; 143 uint32_t size_local; 144 145 size_local = buf_to_iovec(src, size, d->iovs, d->iovcnt, d->seek); 146 d->seek += size_local; 147 148 return size_local; 149 } 150 151 static size_t 152 iovset(struct iovec *iov, size_t iovcnt, int byte, size_t size, size_t offset) 153 { 154 size_t i, len, done = 0; 155 156 for (i = 0; i < iovcnt; i++) { 157 if (offset >= iov[i].iov_len) { 158 offset -= iov[i].iov_len; 159 continue; 160 } 161 162 if (iov[i].iov_base == NULL) { 163 continue; 164 } 165 166 if (done >= size) { 167 break; 168 } 169 170 len = MIN(size - done, iov[i].iov_len - offset); 171 memset(iov[i].iov_base + offset, byte, len); 172 done += len; 173 offset = 0; 174 } 175 176 return done; 177 } 178 179 static uint32_t 180 vbdev_ocf_ctx_data_zero(ctx_data_t *dst, uint32_t size) 181 { 182 struct bdev_ocf_data *d = dst; 183 uint32_t size_local; 184 185 size_local = iovset(d->iovs, d->iovcnt, 0, size, d->seek); 186 d->seek += size_local; 187 188 return size_local; 189 } 190 191 static uint32_t 192 vbdev_ocf_ctx_data_seek(ctx_data_t *dst, ctx_data_seek_t seek, uint32_t offset) 193 { 194 struct bdev_ocf_data *d = dst; 195 uint32_t off = 0; 196 197 switch (seek) { 198 case ctx_data_seek_begin: 199 off = MIN(offset, d->size); 200 d->seek = off; 201 break; 202 case ctx_data_seek_current: 203 off = MIN(offset, d->size - d->seek); 204 d->seek += off; 205 break; 206 } 207 208 return off; 209 } 210 211 static uint64_t 212 vbdev_ocf_ctx_data_cpy(ctx_data_t *dst, ctx_data_t *src, uint64_t to, 213 uint64_t from, uint64_t bytes) 214 { 215 struct bdev_ocf_data *s = src; 216 struct bdev_ocf_data *d = dst; 217 uint32_t it_iov = 0; 218 uint32_t it_off = 0; 219 uint32_t n, sz; 220 221 bytes = MIN(bytes, s->size - from); 222 bytes = MIN(bytes, d->size - to); 223 sz = bytes; 224 225 while (from || bytes) { 226 if (s->iovs[it_iov].iov_len == it_off) { 227 it_iov++; 228 it_off = 0; 229 continue; 230 } 231 232 if (from) { 233 n = MIN(from, s->iovs[it_iov].iov_len); 234 from -= n; 235 } else { 236 n = MIN(bytes, s->iovs[it_iov].iov_len); 237 buf_to_iovec(s->iovs[it_iov].iov_base + it_off, n, d->iovs, d->iovcnt, to); 238 bytes -= n; 239 to += n; 240 } 241 242 it_off += n; 243 } 244 245 return sz; 246 } 247 248 static void 249 vbdev_ocf_ctx_data_secure_erase(ctx_data_t *ctx_data) 250 { 251 struct bdev_ocf_data *data = ctx_data; 252 struct iovec *iovs = data->iovs; 253 int i; 254 255 for (i = 0; i < data->iovcnt; i++) { 256 if (env_memset(iovs[i].iov_base, iovs[i].iov_len, 0)) { 257 assert(false); 258 } 259 } 260 } 261 262 int 263 vbdev_ocf_queue_create(ocf_cache_t cache, ocf_queue_t *queue, const struct ocf_queue_ops *ops) 264 { 265 int rc; 266 struct vbdev_ocf_cache_ctx *ctx = ocf_cache_get_priv(cache); 267 268 pthread_mutex_lock(&ctx->lock); 269 rc = ocf_queue_create(cache, queue, ops); 270 pthread_mutex_unlock(&ctx->lock); 271 return rc; 272 } 273 274 void 275 vbdev_ocf_queue_put(ocf_queue_t queue) 276 { 277 ocf_cache_t cache = ocf_queue_get_cache(queue); 278 struct vbdev_ocf_cache_ctx *ctx = ocf_cache_get_priv(cache); 279 280 pthread_mutex_lock(&ctx->lock); 281 ocf_queue_put(queue); 282 pthread_mutex_unlock(&ctx->lock); 283 } 284 285 void 286 vbdev_ocf_cache_ctx_put(struct vbdev_ocf_cache_ctx *ctx) 287 { 288 if (env_atomic_dec_return(&ctx->refcnt) == 0) { 289 pthread_mutex_destroy(&ctx->lock); 290 free(ctx); 291 } 292 } 293 294 void 295 vbdev_ocf_cache_ctx_get(struct vbdev_ocf_cache_ctx *ctx) 296 { 297 env_atomic_inc(&ctx->refcnt); 298 } 299 300 struct cleaner_priv { 301 struct spdk_poller *poller; 302 ocf_queue_t queue; 303 uint64_t next_run; 304 }; 305 306 static int 307 cleaner_poll(void *arg) 308 { 309 ocf_cleaner_t cleaner = arg; 310 struct cleaner_priv *priv = ocf_cleaner_get_priv(cleaner); 311 uint32_t iono = ocf_queue_pending_io(priv->queue); 312 int i, max = spdk_min(32, iono); 313 314 for (i = 0; i < max; i++) { 315 ocf_queue_run_single(priv->queue); 316 } 317 318 if (spdk_get_ticks() >= priv->next_run) { 319 ocf_cleaner_run(cleaner, priv->queue); 320 return SPDK_POLLER_BUSY; 321 } 322 323 if (iono > 0) { 324 return SPDK_POLLER_BUSY; 325 } else { 326 return SPDK_POLLER_IDLE; 327 } 328 } 329 330 static void 331 cleaner_cmpl(ocf_cleaner_t c, uint32_t interval) 332 { 333 struct cleaner_priv *priv = ocf_cleaner_get_priv(c); 334 335 priv->next_run = spdk_get_ticks() + ((interval * spdk_get_ticks_hz()) / 1000); 336 } 337 338 static void 339 cleaner_queue_kick(ocf_queue_t q) 340 { 341 } 342 343 static void 344 cleaner_queue_stop(ocf_queue_t q) 345 { 346 struct cleaner_priv *cpriv = ocf_queue_get_priv(q); 347 348 if (cpriv) { 349 spdk_poller_unregister(&cpriv->poller); 350 free(cpriv); 351 } 352 } 353 354 const struct ocf_queue_ops cleaner_queue_ops = { 355 .kick_sync = cleaner_queue_kick, 356 .kick = cleaner_queue_kick, 357 .stop = cleaner_queue_stop, 358 }; 359 360 static int 361 vbdev_ocf_ctx_cleaner_init(ocf_cleaner_t c) 362 { 363 int rc; 364 struct cleaner_priv *priv = calloc(1, sizeof(*priv)); 365 ocf_cache_t cache = ocf_cleaner_get_cache(c); 366 struct vbdev_ocf_cache_ctx *cctx = ocf_cache_get_priv(cache); 367 368 if (priv == NULL) { 369 return -ENOMEM; 370 } 371 372 rc = vbdev_ocf_queue_create(cache, &priv->queue, &cleaner_queue_ops); 373 if (rc) { 374 free(priv); 375 return rc; 376 } 377 378 ocf_queue_set_priv(priv->queue, priv); 379 380 cctx->cleaner_queue = priv->queue; 381 382 ocf_cleaner_set_cmpl(c, cleaner_cmpl); 383 ocf_cleaner_set_priv(c, priv); 384 385 return 0; 386 } 387 388 static void 389 vbdev_ocf_ctx_cleaner_stop(ocf_cleaner_t c) 390 { 391 struct cleaner_priv *priv = ocf_cleaner_get_priv(c); 392 393 vbdev_ocf_queue_put(priv->queue); 394 } 395 396 static void 397 vbdev_ocf_ctx_cleaner_kick(ocf_cleaner_t cleaner) 398 { 399 struct cleaner_priv *priv = ocf_cleaner_get_priv(cleaner); 400 401 if (priv->poller) { 402 return; 403 } 404 405 /* We start cleaner poller at the same thread where cache was created 406 * TODO: allow user to specify core at which cleaner should run */ 407 priv->poller = SPDK_POLLER_REGISTER(cleaner_poll, cleaner, 0); 408 } 409 410 /* This function is main way by which OCF communicates with user 411 * We don't want to use SPDK_LOG here because debugging information that is 412 * associated with every print message is not helpful in callback that only prints info 413 * while the real source is somewhere in OCF code */ 414 static int 415 vbdev_ocf_ctx_log_printf(ocf_logger_t logger, ocf_logger_lvl_t lvl, 416 const char *fmt, va_list args) 417 { 418 int spdk_lvl; 419 420 switch (lvl) { 421 case log_emerg: 422 case log_alert: 423 case log_crit: 424 case log_err: 425 spdk_lvl = SPDK_LOG_ERROR; 426 break; 427 428 case log_warn: 429 spdk_lvl = SPDK_LOG_WARN; 430 break; 431 432 case log_notice: 433 spdk_lvl = SPDK_LOG_NOTICE; 434 break; 435 436 case log_info: 437 case log_debug: 438 default: 439 spdk_lvl = SPDK_LOG_INFO; 440 } 441 442 spdk_vlog(spdk_lvl, NULL, -1, NULL, fmt, args); 443 return 0; 444 } 445 446 static const struct ocf_ctx_config vbdev_ocf_ctx_cfg = { 447 .name = "OCF SPDK", 448 449 .ops = { 450 .data = { 451 .alloc = vbdev_ocf_ctx_data_alloc, 452 .free = vbdev_ocf_ctx_data_free, 453 .mlock = vbdev_ocf_ctx_data_mlock, 454 .munlock = vbdev_ocf_ctx_data_munlock, 455 .read = vbdev_ocf_ctx_data_rd, 456 .write = vbdev_ocf_ctx_data_wr, 457 .zero = vbdev_ocf_ctx_data_zero, 458 .seek = vbdev_ocf_ctx_data_seek, 459 .copy = vbdev_ocf_ctx_data_cpy, 460 .secure_erase = vbdev_ocf_ctx_data_secure_erase, 461 }, 462 463 .cleaner = { 464 .init = vbdev_ocf_ctx_cleaner_init, 465 .stop = vbdev_ocf_ctx_cleaner_stop, 466 .kick = vbdev_ocf_ctx_cleaner_kick, 467 }, 468 469 .logger = { 470 .print = vbdev_ocf_ctx_log_printf, 471 .dump_stack = NULL, 472 }, 473 474 }, 475 }; 476 477 int 478 vbdev_ocf_ctx_init(void) 479 { 480 int ret; 481 482 ret = ocf_ctx_create(&vbdev_ocf_ctx, &vbdev_ocf_ctx_cfg); 483 if (ret < 0) { 484 return ret; 485 } 486 487 return 0; 488 } 489 490 void 491 vbdev_ocf_ctx_cleanup(void) 492 { 493 ocf_ctx_put(vbdev_ocf_ctx); 494 vbdev_ocf_ctx = NULL; 495 } 496