1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. 3 * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. 4 * All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk_internal/mock.h" 10 11 #include "spdk/env.h" 12 #include "spdk/queue.h" 13 #include "spdk/util.h" 14 #include "spdk/string.h" 15 16 static uint32_t g_ut_num_cores; 17 static bool *g_ut_cores; 18 19 void allocate_cores(uint32_t num_cores); 20 void free_cores(void); 21 22 DEFINE_STUB(spdk_process_is_primary, bool, (void), true) 23 DEFINE_STUB(spdk_memzone_lookup, void *, (const char *name), NULL) 24 DEFINE_STUB_V(spdk_pci_driver_register, (const char *name, struct spdk_pci_id *id_table, 25 uint32_t flags)); 26 DEFINE_STUB(spdk_pci_nvme_get_driver, struct spdk_pci_driver *, (void), NULL) 27 DEFINE_STUB(spdk_pci_ioat_get_driver, struct spdk_pci_driver *, (void), NULL) 28 DEFINE_STUB(spdk_pci_virtio_get_driver, struct spdk_pci_driver *, (void), NULL) 29 DEFINE_STUB(spdk_env_thread_launch_pinned, int, (uint32_t core, thread_start_fn fn, void *arg), 0); 30 DEFINE_STUB_V(spdk_env_thread_wait_all, (void)); 31 DEFINE_STUB_V(spdk_env_opts_init, (struct spdk_env_opts *opts)); 32 DEFINE_STUB(spdk_env_init, int, (const struct spdk_env_opts *opts), 0); 33 DEFINE_STUB_V(spdk_env_fini, (void)); 34 DEFINE_STUB(spdk_env_get_first_numa_id, int32_t, (void), 0); 35 DEFINE_STUB(spdk_env_get_next_numa_id, int32_t, (int32_t prev_numa_id), INT32_MAX); 36 DEFINE_STUB(spdk_env_get_last_numa_id, int32_t, (void), 0); 37 38 void 39 allocate_cores(uint32_t num_cores) 40 { 41 uint32_t i; 42 43 g_ut_num_cores = num_cores; 44 45 g_ut_cores = calloc(num_cores, sizeof(bool)); 46 assert(g_ut_cores != NULL); 47 48 for (i = 0; i < num_cores; i++) { 49 g_ut_cores[i] = true; 50 } 51 } 52 53 void 54 free_cores(void) 55 { 56 free(g_ut_cores); 57 g_ut_cores = NULL; 58 g_ut_num_cores = 0; 59 } 60 61 static uint32_t 62 ut_get_next_core(uint32_t i) 63 { 64 i++; 65 66 while (i < g_ut_num_cores) { 67 if (!g_ut_cores[i]) { 68 i++; 69 continue; 70 } 71 break; 72 } 73 74 if (i < g_ut_num_cores) { 75 return i; 76 } else { 77 return UINT32_MAX; 78 } 79 } 80 81 uint32_t 82 spdk_env_get_first_core(void) 83 { 84 return ut_get_next_core(-1); 85 } 86 87 uint32_t 88 spdk_env_get_next_core(uint32_t prev_core) 89 { 90 return ut_get_next_core(prev_core); 91 } 92 93 uint32_t 94 spdk_env_get_core_count(void) 95 { 96 return g_ut_num_cores; 97 } 98 99 uint32_t 100 spdk_env_get_last_core(void) 101 { 102 uint32_t i; 103 uint32_t last_core = UINT32_MAX; 104 105 SPDK_ENV_FOREACH_CORE(i) { 106 last_core = i; 107 } 108 109 return last_core; 110 } 111 112 DEFINE_RETURN_MOCK(spdk_env_get_current_core, uint32_t); 113 uint32_t 114 spdk_env_get_current_core(void) 115 { 116 HANDLE_RETURN_MOCK(spdk_env_get_current_core); 117 118 return UINT32_MAX; 119 } 120 121 DEFINE_RETURN_MOCK(spdk_env_get_numa_id, int32_t); 122 int32_t 123 spdk_env_get_numa_id(uint32_t core) 124 { 125 HANDLE_RETURN_MOCK(spdk_env_get_numa_id); 126 127 return SPDK_ENV_NUMA_ID_ANY; 128 } 129 130 /* 131 * These mocks don't use the DEFINE_STUB macros because 132 * their default implementation is more complex. 133 */ 134 135 DEFINE_RETURN_MOCK(spdk_memzone_reserve, void *); 136 void * 137 spdk_memzone_reserve(const char *name, size_t len, int numa_id, unsigned flags) 138 { 139 HANDLE_RETURN_MOCK(spdk_memzone_reserve); 140 141 return malloc(len); 142 } 143 144 DEFINE_RETURN_MOCK(spdk_memzone_reserve_aligned, void *); 145 void * 146 spdk_memzone_reserve_aligned(const char *name, size_t len, int numa_id, 147 unsigned flags, unsigned align) 148 { 149 HANDLE_RETURN_MOCK(spdk_memzone_reserve_aligned); 150 151 return malloc(len); 152 } 153 154 DEFINE_RETURN_MOCK(spdk_malloc, void *); 155 void * 156 spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int numa_id, uint32_t flags) 157 { 158 HANDLE_RETURN_MOCK(spdk_malloc); 159 160 void *buf = NULL; 161 162 if (size == 0) { 163 /* Align how mock handles 0 size with rte functions - return NULL. 164 * According to posix_memalig docs, if size is 0, then the 165 * value placed in *memptr is either NULL or a unique pointer value. */ 166 return NULL; 167 } 168 169 if (align == 0) { 170 align = 8; 171 } 172 173 if (posix_memalign(&buf, align, size)) { 174 return NULL; 175 } 176 if (phys_addr) { 177 *phys_addr = (uint64_t)buf; 178 } 179 180 return buf; 181 } 182 183 DEFINE_RETURN_MOCK(spdk_zmalloc, void *); 184 void * 185 spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int numa_id, uint32_t flags) 186 { 187 HANDLE_RETURN_MOCK(spdk_zmalloc); 188 189 void *buf = spdk_malloc(size, align, phys_addr, -1, 1); 190 191 if (buf != NULL) { 192 memset(buf, 0, size); 193 } 194 return buf; 195 } 196 197 DEFINE_RETURN_MOCK(spdk_dma_malloc, void *); 198 void * 199 spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr) 200 { 201 HANDLE_RETURN_MOCK(spdk_dma_malloc); 202 203 return spdk_malloc(size, align, phys_addr, -1, 1); 204 } 205 206 DEFINE_RETURN_MOCK(spdk_realloc, void *); 207 void * 208 spdk_realloc(void *buf, size_t size, size_t align) 209 { 210 HANDLE_RETURN_MOCK(spdk_realloc); 211 212 return realloc(buf, size); 213 } 214 215 DEFINE_RETURN_MOCK(spdk_dma_zmalloc, void *); 216 void * 217 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr) 218 { 219 HANDLE_RETURN_MOCK(spdk_dma_zmalloc); 220 221 return spdk_zmalloc(size, align, phys_addr, -1, 1); 222 } 223 224 DEFINE_RETURN_MOCK(spdk_dma_malloc_socket, void *); 225 void * 226 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int numa_id) 227 { 228 HANDLE_RETURN_MOCK(spdk_dma_malloc_socket); 229 230 return spdk_dma_malloc(size, align, phys_addr); 231 } 232 233 DEFINE_RETURN_MOCK(spdk_dma_zmalloc_socket, void *); 234 void * 235 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, int numa_id) 236 { 237 HANDLE_RETURN_MOCK(spdk_dma_zmalloc_socket); 238 239 return spdk_dma_zmalloc(size, align, phys_addr); 240 } 241 242 DEFINE_RETURN_MOCK(spdk_dma_realloc, void *); 243 void * 244 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr) 245 { 246 HANDLE_RETURN_MOCK(spdk_dma_realloc); 247 248 return realloc(buf, size); 249 } 250 251 void 252 spdk_free(void *buf) 253 { 254 /* fix for false-positives in *certain* static analysis tools. */ 255 assert((uintptr_t)buf != UINTPTR_MAX); 256 free(buf); 257 } 258 259 void 260 spdk_dma_free(void *buf) 261 { 262 return spdk_free(buf); 263 } 264 265 #ifndef UNIT_TEST_NO_VTOPHYS 266 DEFINE_RETURN_MOCK(spdk_vtophys, uint64_t); 267 uint64_t 268 spdk_vtophys(const void *buf, uint64_t *size) 269 { 270 HANDLE_RETURN_MOCK(spdk_vtophys); 271 272 return (uintptr_t)buf; 273 } 274 #endif 275 276 #ifndef UNIT_TEST_NO_ENV_MEMORY 277 DEFINE_STUB(spdk_mem_get_numa_id, int32_t, (const void *buf, uint64_t *size), 0); 278 #endif 279 280 void 281 spdk_memzone_dump(FILE *f) 282 { 283 return; 284 } 285 286 DEFINE_RETURN_MOCK(spdk_memzone_free, int); 287 int 288 spdk_memzone_free(const char *name) 289 { 290 HANDLE_RETURN_MOCK(spdk_memzone_free); 291 292 return 0; 293 } 294 295 struct test_mempool { 296 size_t count; 297 size_t ele_size; 298 }; 299 300 DEFINE_RETURN_MOCK(spdk_mempool_create, struct spdk_mempool *); 301 struct spdk_mempool * 302 spdk_mempool_create(const char *name, size_t count, 303 size_t ele_size, size_t cache_size, int numa_id) 304 { 305 struct test_mempool *mp; 306 307 HANDLE_RETURN_MOCK(spdk_mempool_create); 308 309 mp = calloc(1, sizeof(*mp)); 310 if (mp == NULL) { 311 return NULL; 312 } 313 314 mp->count = count; 315 mp->ele_size = ele_size; 316 317 return (struct spdk_mempool *)mp; 318 } 319 320 void 321 spdk_mempool_free(struct spdk_mempool *_mp) 322 { 323 struct test_mempool *mp = (struct test_mempool *)_mp; 324 325 free(mp); 326 } 327 328 DEFINE_RETURN_MOCK(spdk_mempool_get, void *); 329 void * 330 spdk_mempool_get(struct spdk_mempool *_mp) 331 { 332 struct test_mempool *mp = (struct test_mempool *)_mp; 333 size_t ele_size = 0x10000; 334 void *buf; 335 336 HANDLE_RETURN_MOCK(spdk_mempool_get); 337 338 if (mp && mp->count == 0) { 339 return NULL; 340 } 341 342 if (mp) { 343 ele_size = mp->ele_size; 344 } 345 346 if (posix_memalign(&buf, 64, spdk_align32pow2(ele_size))) { 347 return NULL; 348 } else { 349 if (mp) { 350 mp->count--; 351 } 352 return buf; 353 } 354 } 355 356 int 357 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 358 { 359 struct test_mempool *test_mp = (struct test_mempool *)mp; 360 361 if (test_mp && test_mp->count < count) { 362 return -1; 363 } 364 365 for (size_t i = 0; i < count; i++) { 366 ele_arr[i] = spdk_mempool_get(mp); 367 if (ele_arr[i] == NULL) { 368 return -1; 369 } 370 } 371 return 0; 372 } 373 374 void 375 spdk_mempool_put(struct spdk_mempool *_mp, void *ele) 376 { 377 struct test_mempool *mp = (struct test_mempool *)_mp; 378 379 if (mp) { 380 mp->count++; 381 } 382 free(ele); 383 } 384 385 void 386 spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 387 { 388 for (size_t i = 0; i < count; i++) { 389 spdk_mempool_put(mp, ele_arr[i]); 390 } 391 } 392 393 DEFINE_RETURN_MOCK(spdk_mempool_count, size_t); 394 size_t 395 spdk_mempool_count(const struct spdk_mempool *_mp) 396 { 397 struct test_mempool *mp = (struct test_mempool *)_mp; 398 399 HANDLE_RETURN_MOCK(spdk_mempool_count); 400 401 if (mp) { 402 return mp->count; 403 } else { 404 return 1024; 405 } 406 } 407 408 struct spdk_ring_ele { 409 void *ele; 410 TAILQ_ENTRY(spdk_ring_ele) link; 411 }; 412 413 struct spdk_ring { 414 TAILQ_HEAD(, spdk_ring_ele) elements; 415 pthread_mutex_t lock; 416 size_t count; 417 }; 418 419 DEFINE_RETURN_MOCK(spdk_ring_create, struct spdk_ring *); 420 struct spdk_ring * 421 spdk_ring_create(enum spdk_ring_type type, size_t count, int numa_id) 422 { 423 struct spdk_ring *ring; 424 425 HANDLE_RETURN_MOCK(spdk_ring_create); 426 427 ring = calloc(1, sizeof(*ring)); 428 if (!ring) { 429 return NULL; 430 } 431 432 if (pthread_mutex_init(&ring->lock, NULL)) { 433 free(ring); 434 return NULL; 435 } 436 437 TAILQ_INIT(&ring->elements); 438 return ring; 439 } 440 441 void 442 spdk_ring_free(struct spdk_ring *ring) 443 { 444 struct spdk_ring_ele *ele, *tmp; 445 446 if (!ring) { 447 return; 448 } 449 450 TAILQ_FOREACH_SAFE(ele, &ring->elements, link, tmp) { 451 free(ele); 452 } 453 454 pthread_mutex_destroy(&ring->lock); 455 free(ring); 456 } 457 458 DEFINE_RETURN_MOCK(spdk_ring_enqueue, size_t); 459 size_t 460 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count, 461 size_t *free_space) 462 { 463 struct spdk_ring_ele *ele; 464 size_t i; 465 466 HANDLE_RETURN_MOCK(spdk_ring_enqueue); 467 468 pthread_mutex_lock(&ring->lock); 469 470 for (i = 0; i < count; i++) { 471 ele = calloc(1, sizeof(*ele)); 472 if (!ele) { 473 break; 474 } 475 476 ele->ele = objs[i]; 477 TAILQ_INSERT_TAIL(&ring->elements, ele, link); 478 ring->count++; 479 } 480 481 pthread_mutex_unlock(&ring->lock); 482 return i; 483 } 484 485 DEFINE_RETURN_MOCK(spdk_ring_dequeue, size_t); 486 size_t 487 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count) 488 { 489 struct spdk_ring_ele *ele, *tmp; 490 size_t i = 0; 491 492 HANDLE_RETURN_MOCK(spdk_ring_dequeue); 493 494 if (count == 0) { 495 return 0; 496 } 497 498 pthread_mutex_lock(&ring->lock); 499 500 TAILQ_FOREACH_SAFE(ele, &ring->elements, link, tmp) { 501 TAILQ_REMOVE(&ring->elements, ele, link); 502 ring->count--; 503 objs[i] = ele->ele; 504 free(ele); 505 i++; 506 if (i >= count) { 507 break; 508 } 509 } 510 511 pthread_mutex_unlock(&ring->lock); 512 return i; 513 514 } 515 516 517 DEFINE_RETURN_MOCK(spdk_ring_count, size_t); 518 size_t 519 spdk_ring_count(struct spdk_ring *ring) 520 { 521 HANDLE_RETURN_MOCK(spdk_ring_count); 522 return ring->count; 523 } 524 525 DEFINE_RETURN_MOCK(spdk_get_ticks, uint64_t); 526 uint64_t 527 spdk_get_ticks(void) 528 { 529 HANDLE_RETURN_MOCK(spdk_get_ticks); 530 531 return ut_spdk_get_ticks; 532 } 533 534 DEFINE_RETURN_MOCK(spdk_get_ticks_hz, uint64_t); 535 uint64_t 536 spdk_get_ticks_hz(void) 537 { 538 HANDLE_RETURN_MOCK(spdk_get_ticks_hz); 539 540 return 1000000; 541 } 542 543 void 544 spdk_delay_us(unsigned int us) 545 { 546 /* spdk_get_ticks_hz is 1000000, meaning 1 tick per us. */ 547 ut_spdk_get_ticks += us; 548 } 549 550 DEFINE_RETURN_MOCK(spdk_pci_addr_parse, int); 551 int 552 spdk_pci_addr_parse(struct spdk_pci_addr *addr, const char *bdf) 553 { 554 unsigned domain, bus, dev, func; 555 556 HANDLE_RETURN_MOCK(spdk_pci_addr_parse); 557 558 if (addr == NULL || bdf == NULL) { 559 return -EINVAL; 560 } 561 562 if ((sscanf(bdf, "%x:%x:%x.%x", &domain, &bus, &dev, &func) == 4) || 563 (sscanf(bdf, "%x.%x.%x.%x", &domain, &bus, &dev, &func) == 4)) { 564 /* Matched a full address - all variables are initialized */ 565 } else if (sscanf(bdf, "%x:%x:%x", &domain, &bus, &dev) == 3) { 566 func = 0; 567 } else if ((sscanf(bdf, "%x:%x.%x", &bus, &dev, &func) == 3) || 568 (sscanf(bdf, "%x.%x.%x", &bus, &dev, &func) == 3)) { 569 domain = 0; 570 } else if ((sscanf(bdf, "%x:%x", &bus, &dev) == 2) || 571 (sscanf(bdf, "%x.%x", &bus, &dev) == 2)) { 572 domain = 0; 573 func = 0; 574 } else { 575 return -EINVAL; 576 } 577 578 if (bus > 0xFF || dev > 0x1F || func > 7) { 579 return -EINVAL; 580 } 581 582 addr->domain = domain; 583 addr->bus = bus; 584 addr->dev = dev; 585 addr->func = func; 586 587 return 0; 588 } 589 590 DEFINE_RETURN_MOCK(spdk_pci_addr_fmt, int); 591 int 592 spdk_pci_addr_fmt(char *bdf, size_t sz, const struct spdk_pci_addr *addr) 593 { 594 int rc; 595 596 HANDLE_RETURN_MOCK(spdk_pci_addr_fmt); 597 598 rc = snprintf(bdf, sz, "%04x:%02x:%02x.%x", 599 addr->domain, addr->bus, 600 addr->dev, addr->func); 601 602 if (rc > 0 && (size_t)rc < sz) { 603 return 0; 604 } 605 606 return -1; 607 } 608 609 DEFINE_RETURN_MOCK(spdk_pci_addr_compare, int); 610 int 611 spdk_pci_addr_compare(const struct spdk_pci_addr *a1, const struct spdk_pci_addr *a2) 612 { 613 HANDLE_RETURN_MOCK(spdk_pci_addr_compare); 614 615 if (a1->domain > a2->domain) { 616 return 1; 617 } else if (a1->domain < a2->domain) { 618 return -1; 619 } else if (a1->bus > a2->bus) { 620 return 1; 621 } else if (a1->bus < a2->bus) { 622 return -1; 623 } else if (a1->dev > a2->dev) { 624 return 1; 625 } else if (a1->dev < a2->dev) { 626 return -1; 627 } else if (a1->func > a2->func) { 628 return 1; 629 } else if (a1->func < a2->func) { 630 return -1; 631 } 632 633 return 0; 634 } 635