1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2018 Intel Corporation. 3 * All rights reserved. 4 */ 5 6 #include <ocf/ocf.h> 7 #include <execinfo.h> 8 9 #include "spdk/env.h" 10 #include "spdk/log.h" 11 12 #include "ctx.h" 13 #include "data.h" 14 15 ocf_ctx_t vbdev_ocf_ctx; 16 17 static ctx_data_t * 18 vbdev_ocf_ctx_data_alloc(uint32_t pages) 19 { 20 struct bdev_ocf_data *data; 21 void *buf; 22 uint32_t sz; 23 24 data = vbdev_ocf_data_alloc(1); 25 26 sz = pages * PAGE_SIZE; 27 buf = spdk_malloc(sz, PAGE_SIZE, NULL, 28 SPDK_ENV_LCORE_ID_ANY, SPDK_MALLOC_DMA); 29 if (buf == NULL) { 30 return NULL; 31 } 32 33 vbdev_ocf_iovs_add(data, buf, sz); 34 35 data->size = sz; 36 37 return data; 38 } 39 40 static void 41 vbdev_ocf_ctx_data_free(ctx_data_t *ctx_data) 42 { 43 struct bdev_ocf_data *data = ctx_data; 44 int i; 45 46 if (!data) { 47 return; 48 } 49 50 for (i = 0; i < data->iovcnt; i++) { 51 spdk_free(data->iovs[i].iov_base); 52 } 53 54 vbdev_ocf_data_free(data); 55 } 56 57 static int 58 vbdev_ocf_ctx_data_mlock(ctx_data_t *ctx_data) 59 { 60 /* TODO [mlock]: add mlock option */ 61 return 0; 62 } 63 64 static void 65 vbdev_ocf_ctx_data_munlock(ctx_data_t *ctx_data) 66 { 67 /* TODO [mlock]: add mlock option */ 68 } 69 70 static size_t 71 iovec_flatten(struct iovec *iov, size_t iovcnt, void *buf, size_t size, size_t offset) 72 { 73 size_t i, len, done = 0; 74 75 for (i = 0; i < iovcnt; i++) { 76 if (offset >= iov[i].iov_len) { 77 offset -= iov[i].iov_len; 78 continue; 79 } 80 81 if (iov[i].iov_base == NULL) { 82 continue; 83 } 84 85 if (done >= size) { 86 break; 87 } 88 89 len = MIN(size - done, iov[i].iov_len - offset); 90 memcpy(buf, iov[i].iov_base + offset, len); 91 buf += len; 92 done += len; 93 offset = 0; 94 } 95 96 return done; 97 } 98 99 static uint32_t 100 vbdev_ocf_ctx_data_rd(void *dst, ctx_data_t *src, uint32_t size) 101 { 102 struct bdev_ocf_data *s = src; 103 uint32_t size_local; 104 105 size_local = iovec_flatten(s->iovs, s->iovcnt, dst, size, s->seek); 106 s->seek += size_local; 107 108 return size_local; 109 } 110 111 static size_t 112 buf_to_iovec(const void *buf, size_t size, struct iovec *iov, size_t iovcnt, size_t offset) 113 { 114 size_t i, len, done = 0; 115 116 for (i = 0; i < iovcnt; i++) { 117 if (offset >= iov[i].iov_len) { 118 offset -= iov[i].iov_len; 119 continue; 120 } 121 122 if (iov[i].iov_base == NULL) { 123 continue; 124 } 125 126 if (done >= size) { 127 break; 128 } 129 130 len = MIN(size - done, iov[i].iov_len - offset); 131 memcpy(iov[i].iov_base + offset, buf, len); 132 buf += len; 133 done += len; 134 offset = 0; 135 } 136 137 return done; 138 } 139 140 static uint32_t 141 vbdev_ocf_ctx_data_wr(ctx_data_t *dst, const void *src, uint32_t size) 142 { 143 struct bdev_ocf_data *d = dst; 144 uint32_t size_local; 145 146 size_local = buf_to_iovec(src, size, d->iovs, d->iovcnt, d->seek); 147 d->seek += size_local; 148 149 return size_local; 150 } 151 152 static size_t 153 iovset(struct iovec *iov, size_t iovcnt, int byte, size_t size, size_t offset) 154 { 155 size_t i, len, done = 0; 156 157 for (i = 0; i < iovcnt; i++) { 158 if (offset >= iov[i].iov_len) { 159 offset -= iov[i].iov_len; 160 continue; 161 } 162 163 if (iov[i].iov_base == NULL) { 164 continue; 165 } 166 167 if (done >= size) { 168 break; 169 } 170 171 len = MIN(size - done, iov[i].iov_len - offset); 172 memset(iov[i].iov_base + offset, byte, len); 173 done += len; 174 offset = 0; 175 } 176 177 return done; 178 } 179 180 static uint32_t 181 vbdev_ocf_ctx_data_zero(ctx_data_t *dst, uint32_t size) 182 { 183 struct bdev_ocf_data *d = dst; 184 uint32_t size_local; 185 186 size_local = iovset(d->iovs, d->iovcnt, 0, size, d->seek); 187 d->seek += size_local; 188 189 return size_local; 190 } 191 192 static uint32_t 193 vbdev_ocf_ctx_data_seek(ctx_data_t *dst, ctx_data_seek_t seek, uint32_t offset) 194 { 195 struct bdev_ocf_data *d = dst; 196 uint32_t off = 0; 197 198 switch (seek) { 199 case ctx_data_seek_begin: 200 off = MIN(offset, d->size); 201 d->seek = off; 202 break; 203 case ctx_data_seek_current: 204 off = MIN(offset, d->size - d->seek); 205 d->seek += off; 206 break; 207 } 208 209 return off; 210 } 211 212 static uint64_t 213 vbdev_ocf_ctx_data_cpy(ctx_data_t *dst, ctx_data_t *src, uint64_t to, 214 uint64_t from, uint64_t bytes) 215 { 216 struct bdev_ocf_data *s = src; 217 struct bdev_ocf_data *d = dst; 218 uint32_t it_iov = 0; 219 uint32_t it_off = 0; 220 uint32_t n, sz; 221 222 bytes = MIN(bytes, s->size - from); 223 bytes = MIN(bytes, d->size - to); 224 sz = bytes; 225 226 while (from || bytes) { 227 if (s->iovs[it_iov].iov_len == it_off) { 228 it_iov++; 229 it_off = 0; 230 continue; 231 } 232 233 if (from) { 234 n = MIN(from, s->iovs[it_iov].iov_len); 235 from -= n; 236 } else { 237 n = MIN(bytes, s->iovs[it_iov].iov_len); 238 buf_to_iovec(s->iovs[it_iov].iov_base + it_off, n, d->iovs, d->iovcnt, to); 239 bytes -= n; 240 to += n; 241 } 242 243 it_off += n; 244 } 245 246 return sz; 247 } 248 249 static void 250 vbdev_ocf_ctx_data_secure_erase(ctx_data_t *ctx_data) 251 { 252 struct bdev_ocf_data *data = ctx_data; 253 struct iovec *iovs = data->iovs; 254 int i; 255 256 for (i = 0; i < data->iovcnt; i++) { 257 if (env_memset(iovs[i].iov_base, iovs[i].iov_len, 0)) { 258 assert(false); 259 } 260 } 261 } 262 263 int 264 vbdev_ocf_queue_create(ocf_cache_t cache, ocf_queue_t *queue, const struct ocf_queue_ops *ops) 265 { 266 int rc; 267 struct vbdev_ocf_cache_ctx *ctx = ocf_cache_get_priv(cache); 268 269 pthread_mutex_lock(&ctx->lock); 270 rc = ocf_queue_create(cache, queue, ops); 271 pthread_mutex_unlock(&ctx->lock); 272 return rc; 273 } 274 275 void 276 vbdev_ocf_queue_put(ocf_queue_t queue) 277 { 278 ocf_cache_t cache = ocf_queue_get_cache(queue); 279 struct vbdev_ocf_cache_ctx *ctx = ocf_cache_get_priv(cache); 280 281 pthread_mutex_lock(&ctx->lock); 282 ocf_queue_put(queue); 283 pthread_mutex_unlock(&ctx->lock); 284 } 285 286 void 287 vbdev_ocf_cache_ctx_put(struct vbdev_ocf_cache_ctx *ctx) 288 { 289 if (env_atomic_dec_return(&ctx->refcnt) == 0) { 290 pthread_mutex_destroy(&ctx->lock); 291 free(ctx); 292 } 293 } 294 295 void 296 vbdev_ocf_cache_ctx_get(struct vbdev_ocf_cache_ctx *ctx) 297 { 298 env_atomic_inc(&ctx->refcnt); 299 } 300 301 struct cleaner_priv { 302 struct spdk_poller *poller; 303 ocf_queue_t queue; 304 uint64_t next_run; 305 }; 306 307 static int 308 cleaner_poll(void *arg) 309 { 310 ocf_cleaner_t cleaner = arg; 311 struct cleaner_priv *priv = ocf_cleaner_get_priv(cleaner); 312 uint32_t iono = ocf_queue_pending_io(priv->queue); 313 int i, max = spdk_min(32, iono); 314 315 for (i = 0; i < max; i++) { 316 ocf_queue_run_single(priv->queue); 317 } 318 319 if (spdk_get_ticks() >= priv->next_run) { 320 ocf_cleaner_run(cleaner, priv->queue); 321 return SPDK_POLLER_BUSY; 322 } 323 324 if (iono > 0) { 325 return SPDK_POLLER_BUSY; 326 } else { 327 return SPDK_POLLER_IDLE; 328 } 329 } 330 331 static void 332 cleaner_cmpl(ocf_cleaner_t c, uint32_t interval) 333 { 334 struct cleaner_priv *priv = ocf_cleaner_get_priv(c); 335 336 priv->next_run = spdk_get_ticks() + ((interval * spdk_get_ticks_hz()) / 1000); 337 } 338 339 static void 340 cleaner_queue_kick(ocf_queue_t q) 341 { 342 } 343 344 static void 345 cleaner_queue_stop(ocf_queue_t q) 346 { 347 struct cleaner_priv *cpriv = ocf_queue_get_priv(q); 348 349 if (cpriv) { 350 spdk_poller_unregister(&cpriv->poller); 351 free(cpriv); 352 } 353 } 354 355 const struct ocf_queue_ops cleaner_queue_ops = { 356 .kick_sync = cleaner_queue_kick, 357 .kick = cleaner_queue_kick, 358 .stop = cleaner_queue_stop, 359 }; 360 361 static int 362 vbdev_ocf_ctx_cleaner_init(ocf_cleaner_t c) 363 { 364 int rc; 365 struct cleaner_priv *priv = calloc(1, sizeof(*priv)); 366 ocf_cache_t cache = ocf_cleaner_get_cache(c); 367 struct vbdev_ocf_cache_ctx *cctx = ocf_cache_get_priv(cache); 368 369 if (priv == NULL) { 370 return -ENOMEM; 371 } 372 373 rc = vbdev_ocf_queue_create(cache, &priv->queue, &cleaner_queue_ops); 374 if (rc) { 375 free(priv); 376 return rc; 377 } 378 379 ocf_queue_set_priv(priv->queue, priv); 380 381 cctx->cleaner_queue = priv->queue; 382 383 ocf_cleaner_set_cmpl(c, cleaner_cmpl); 384 ocf_cleaner_set_priv(c, priv); 385 386 return 0; 387 } 388 389 static void 390 vbdev_ocf_ctx_cleaner_stop(ocf_cleaner_t c) 391 { 392 struct cleaner_priv *priv = ocf_cleaner_get_priv(c); 393 394 vbdev_ocf_queue_put(priv->queue); 395 } 396 397 static void 398 vbdev_ocf_ctx_cleaner_kick(ocf_cleaner_t cleaner) 399 { 400 struct cleaner_priv *priv = ocf_cleaner_get_priv(cleaner); 401 402 if (priv->poller) { 403 return; 404 } 405 406 /* We start cleaner poller at the same thread where cache was created 407 * TODO: allow user to specify core at which cleaner should run */ 408 priv->poller = SPDK_POLLER_REGISTER(cleaner_poll, cleaner, 0); 409 } 410 411 /* This function is main way by which OCF communicates with user 412 * We don't want to use SPDK_LOG here because debugging information that is 413 * associated with every print message is not helpful in callback that only prints info 414 * while the real source is somewhere in OCF code */ 415 static int 416 vbdev_ocf_ctx_log_printf(ocf_logger_t logger, ocf_logger_lvl_t lvl, 417 const char *fmt, va_list args) 418 { 419 int spdk_lvl; 420 421 switch (lvl) { 422 case log_emerg: 423 case log_alert: 424 case log_crit: 425 case log_err: 426 spdk_lvl = SPDK_LOG_ERROR; 427 break; 428 429 case log_warn: 430 spdk_lvl = SPDK_LOG_WARN; 431 break; 432 433 case log_notice: 434 spdk_lvl = SPDK_LOG_NOTICE; 435 break; 436 437 case log_info: 438 case log_debug: 439 default: 440 spdk_lvl = SPDK_LOG_INFO; 441 } 442 443 spdk_vlog(spdk_lvl, NULL, -1, NULL, fmt, args); 444 return 0; 445 } 446 447 static const struct ocf_ctx_config vbdev_ocf_ctx_cfg = { 448 .name = "OCF SPDK", 449 450 .ops = { 451 .data = { 452 .alloc = vbdev_ocf_ctx_data_alloc, 453 .free = vbdev_ocf_ctx_data_free, 454 .mlock = vbdev_ocf_ctx_data_mlock, 455 .munlock = vbdev_ocf_ctx_data_munlock, 456 .read = vbdev_ocf_ctx_data_rd, 457 .write = vbdev_ocf_ctx_data_wr, 458 .zero = vbdev_ocf_ctx_data_zero, 459 .seek = vbdev_ocf_ctx_data_seek, 460 .copy = vbdev_ocf_ctx_data_cpy, 461 .secure_erase = vbdev_ocf_ctx_data_secure_erase, 462 }, 463 464 .cleaner = { 465 .init = vbdev_ocf_ctx_cleaner_init, 466 .stop = vbdev_ocf_ctx_cleaner_stop, 467 .kick = vbdev_ocf_ctx_cleaner_kick, 468 }, 469 470 .logger = { 471 .print = vbdev_ocf_ctx_log_printf, 472 .dump_stack = NULL, 473 }, 474 475 }, 476 }; 477 478 int 479 vbdev_ocf_ctx_init(void) 480 { 481 int ret; 482 483 ret = ocf_ctx_create(&vbdev_ocf_ctx, &vbdev_ocf_ctx_cfg); 484 if (ret < 0) { 485 return ret; 486 } 487 488 return 0; 489 } 490 491 void 492 vbdev_ocf_ctx_cleanup(void) 493 { 494 ocf_ctx_put(vbdev_ocf_ctx); 495 vbdev_ocf_ctx = NULL; 496 } 497