1 /* SPDX-License-Identifier: BSD-3-Clause 2 * Copyright (C) 2016 Intel Corporation. 3 * Copyright (c) 2022 NVIDIA CORPORATION & AFFILIATES. 4 * All rights reserved. 5 */ 6 7 #include "spdk/stdinc.h" 8 9 #include "spdk_internal/mock.h" 10 11 #include "spdk/env.h" 12 #include "spdk/queue.h" 13 #include "spdk/util.h" 14 #include "spdk/string.h" 15 16 static uint32_t g_ut_num_cores; 17 static bool *g_ut_cores; 18 19 void allocate_cores(uint32_t num_cores); 20 void free_cores(void); 21 22 DEFINE_STUB(spdk_process_is_primary, bool, (void), true) 23 DEFINE_STUB(spdk_memzone_lookup, void *, (const char *name), NULL) 24 DEFINE_STUB_V(spdk_pci_driver_register, (const char *name, struct spdk_pci_id *id_table, 25 uint32_t flags)); 26 DEFINE_STUB(spdk_pci_nvme_get_driver, struct spdk_pci_driver *, (void), NULL) 27 DEFINE_STUB(spdk_pci_ioat_get_driver, struct spdk_pci_driver *, (void), NULL) 28 DEFINE_STUB(spdk_pci_virtio_get_driver, struct spdk_pci_driver *, (void), NULL) 29 DEFINE_STUB(spdk_env_thread_launch_pinned, int, (uint32_t core, thread_start_fn fn, void *arg), 0); 30 DEFINE_STUB_V(spdk_env_thread_wait_all, (void)); 31 DEFINE_STUB_V(spdk_env_opts_init, (struct spdk_env_opts *opts)); 32 DEFINE_STUB(spdk_env_init, int, (const struct spdk_env_opts *opts), 0); 33 DEFINE_STUB_V(spdk_env_fini, (void)); 34 DEFINE_STUB(spdk_env_get_last_numa_id, int32_t, (void), 0); 35 36 void 37 allocate_cores(uint32_t num_cores) 38 { 39 uint32_t i; 40 41 g_ut_num_cores = num_cores; 42 43 g_ut_cores = calloc(num_cores, sizeof(bool)); 44 assert(g_ut_cores != NULL); 45 46 for (i = 0; i < num_cores; i++) { 47 g_ut_cores[i] = true; 48 } 49 } 50 51 void 52 free_cores(void) 53 { 54 free(g_ut_cores); 55 g_ut_cores = NULL; 56 g_ut_num_cores = 0; 57 } 58 59 static uint32_t 60 ut_get_next_core(uint32_t i) 61 { 62 i++; 63 64 while (i < g_ut_num_cores) { 65 if (!g_ut_cores[i]) { 66 i++; 67 continue; 68 } 69 break; 70 } 71 72 if (i < g_ut_num_cores) { 73 return i; 74 } else { 75 return UINT32_MAX; 76 } 77 } 78 79 uint32_t 80 spdk_env_get_first_core(void) 81 { 82 return ut_get_next_core(-1); 83 } 84 85 uint32_t 86 spdk_env_get_next_core(uint32_t prev_core) 87 { 88 return ut_get_next_core(prev_core); 89 } 90 91 uint32_t 92 spdk_env_get_core_count(void) 93 { 94 return g_ut_num_cores; 95 } 96 97 uint32_t 98 spdk_env_get_last_core(void) 99 { 100 uint32_t i; 101 uint32_t last_core = UINT32_MAX; 102 103 SPDK_ENV_FOREACH_CORE(i) { 104 last_core = i; 105 } 106 107 return last_core; 108 } 109 110 DEFINE_RETURN_MOCK(spdk_env_get_current_core, uint32_t); 111 uint32_t 112 spdk_env_get_current_core(void) 113 { 114 HANDLE_RETURN_MOCK(spdk_env_get_current_core); 115 116 return UINT32_MAX; 117 } 118 119 DEFINE_RETURN_MOCK(spdk_env_get_numa_id, int32_t); 120 int32_t 121 spdk_env_get_numa_id(uint32_t core) 122 { 123 HANDLE_RETURN_MOCK(spdk_env_get_numa_id); 124 125 return SPDK_ENV_NUMA_ID_ANY; 126 } 127 128 /* 129 * These mocks don't use the DEFINE_STUB macros because 130 * their default implementation is more complex. 131 */ 132 133 DEFINE_RETURN_MOCK(spdk_memzone_reserve, void *); 134 void * 135 spdk_memzone_reserve(const char *name, size_t len, int numa_id, unsigned flags) 136 { 137 HANDLE_RETURN_MOCK(spdk_memzone_reserve); 138 139 return malloc(len); 140 } 141 142 DEFINE_RETURN_MOCK(spdk_memzone_reserve_aligned, void *); 143 void * 144 spdk_memzone_reserve_aligned(const char *name, size_t len, int numa_id, 145 unsigned flags, unsigned align) 146 { 147 HANDLE_RETURN_MOCK(spdk_memzone_reserve_aligned); 148 149 return malloc(len); 150 } 151 152 DEFINE_RETURN_MOCK(spdk_malloc, void *); 153 void * 154 spdk_malloc(size_t size, size_t align, uint64_t *phys_addr, int numa_id, uint32_t flags) 155 { 156 HANDLE_RETURN_MOCK(spdk_malloc); 157 158 void *buf = NULL; 159 160 if (size == 0) { 161 /* Align how mock handles 0 size with rte functions - return NULL. 162 * According to posix_memalig docs, if size is 0, then the 163 * value placed in *memptr is either NULL or a unique pointer value. */ 164 return NULL; 165 } 166 167 if (align == 0) { 168 align = 8; 169 } 170 171 if (posix_memalign(&buf, align, size)) { 172 return NULL; 173 } 174 if (phys_addr) { 175 *phys_addr = (uint64_t)buf; 176 } 177 178 return buf; 179 } 180 181 DEFINE_RETURN_MOCK(spdk_zmalloc, void *); 182 void * 183 spdk_zmalloc(size_t size, size_t align, uint64_t *phys_addr, int numa_id, uint32_t flags) 184 { 185 HANDLE_RETURN_MOCK(spdk_zmalloc); 186 187 void *buf = spdk_malloc(size, align, phys_addr, -1, 1); 188 189 if (buf != NULL) { 190 memset(buf, 0, size); 191 } 192 return buf; 193 } 194 195 DEFINE_RETURN_MOCK(spdk_dma_malloc, void *); 196 void * 197 spdk_dma_malloc(size_t size, size_t align, uint64_t *phys_addr) 198 { 199 HANDLE_RETURN_MOCK(spdk_dma_malloc); 200 201 return spdk_malloc(size, align, phys_addr, -1, 1); 202 } 203 204 DEFINE_RETURN_MOCK(spdk_realloc, void *); 205 void * 206 spdk_realloc(void *buf, size_t size, size_t align) 207 { 208 HANDLE_RETURN_MOCK(spdk_realloc); 209 210 return realloc(buf, size); 211 } 212 213 DEFINE_RETURN_MOCK(spdk_dma_zmalloc, void *); 214 void * 215 spdk_dma_zmalloc(size_t size, size_t align, uint64_t *phys_addr) 216 { 217 HANDLE_RETURN_MOCK(spdk_dma_zmalloc); 218 219 return spdk_zmalloc(size, align, phys_addr, -1, 1); 220 } 221 222 DEFINE_RETURN_MOCK(spdk_dma_malloc_socket, void *); 223 void * 224 spdk_dma_malloc_socket(size_t size, size_t align, uint64_t *phys_addr, int numa_id) 225 { 226 HANDLE_RETURN_MOCK(spdk_dma_malloc_socket); 227 228 return spdk_dma_malloc(size, align, phys_addr); 229 } 230 231 DEFINE_RETURN_MOCK(spdk_dma_zmalloc_socket, void *); 232 void * 233 spdk_dma_zmalloc_socket(size_t size, size_t align, uint64_t *phys_addr, int numa_id) 234 { 235 HANDLE_RETURN_MOCK(spdk_dma_zmalloc_socket); 236 237 return spdk_dma_zmalloc(size, align, phys_addr); 238 } 239 240 DEFINE_RETURN_MOCK(spdk_dma_realloc, void *); 241 void * 242 spdk_dma_realloc(void *buf, size_t size, size_t align, uint64_t *phys_addr) 243 { 244 HANDLE_RETURN_MOCK(spdk_dma_realloc); 245 246 return realloc(buf, size); 247 } 248 249 void 250 spdk_free(void *buf) 251 { 252 /* fix for false-positives in *certain* static analysis tools. */ 253 assert((uintptr_t)buf != UINTPTR_MAX); 254 free(buf); 255 } 256 257 void 258 spdk_dma_free(void *buf) 259 { 260 return spdk_free(buf); 261 } 262 263 #ifndef UNIT_TEST_NO_VTOPHYS 264 DEFINE_RETURN_MOCK(spdk_vtophys, uint64_t); 265 uint64_t 266 spdk_vtophys(const void *buf, uint64_t *size) 267 { 268 HANDLE_RETURN_MOCK(spdk_vtophys); 269 270 return (uintptr_t)buf; 271 } 272 #endif 273 274 void 275 spdk_memzone_dump(FILE *f) 276 { 277 return; 278 } 279 280 DEFINE_RETURN_MOCK(spdk_memzone_free, int); 281 int 282 spdk_memzone_free(const char *name) 283 { 284 HANDLE_RETURN_MOCK(spdk_memzone_free); 285 286 return 0; 287 } 288 289 struct test_mempool { 290 size_t count; 291 size_t ele_size; 292 }; 293 294 DEFINE_RETURN_MOCK(spdk_mempool_create, struct spdk_mempool *); 295 struct spdk_mempool * 296 spdk_mempool_create(const char *name, size_t count, 297 size_t ele_size, size_t cache_size, int numa_id) 298 { 299 struct test_mempool *mp; 300 301 HANDLE_RETURN_MOCK(spdk_mempool_create); 302 303 mp = calloc(1, sizeof(*mp)); 304 if (mp == NULL) { 305 return NULL; 306 } 307 308 mp->count = count; 309 mp->ele_size = ele_size; 310 311 return (struct spdk_mempool *)mp; 312 } 313 314 void 315 spdk_mempool_free(struct spdk_mempool *_mp) 316 { 317 struct test_mempool *mp = (struct test_mempool *)_mp; 318 319 free(mp); 320 } 321 322 DEFINE_RETURN_MOCK(spdk_mempool_get, void *); 323 void * 324 spdk_mempool_get(struct spdk_mempool *_mp) 325 { 326 struct test_mempool *mp = (struct test_mempool *)_mp; 327 size_t ele_size = 0x10000; 328 void *buf; 329 330 HANDLE_RETURN_MOCK(spdk_mempool_get); 331 332 if (mp && mp->count == 0) { 333 return NULL; 334 } 335 336 if (mp) { 337 ele_size = mp->ele_size; 338 } 339 340 if (posix_memalign(&buf, 64, spdk_align32pow2(ele_size))) { 341 return NULL; 342 } else { 343 if (mp) { 344 mp->count--; 345 } 346 return buf; 347 } 348 } 349 350 int 351 spdk_mempool_get_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 352 { 353 struct test_mempool *test_mp = (struct test_mempool *)mp; 354 355 if (test_mp && test_mp->count < count) { 356 return -1; 357 } 358 359 for (size_t i = 0; i < count; i++) { 360 ele_arr[i] = spdk_mempool_get(mp); 361 if (ele_arr[i] == NULL) { 362 return -1; 363 } 364 } 365 return 0; 366 } 367 368 void 369 spdk_mempool_put(struct spdk_mempool *_mp, void *ele) 370 { 371 struct test_mempool *mp = (struct test_mempool *)_mp; 372 373 if (mp) { 374 mp->count++; 375 } 376 free(ele); 377 } 378 379 void 380 spdk_mempool_put_bulk(struct spdk_mempool *mp, void **ele_arr, size_t count) 381 { 382 for (size_t i = 0; i < count; i++) { 383 spdk_mempool_put(mp, ele_arr[i]); 384 } 385 } 386 387 DEFINE_RETURN_MOCK(spdk_mempool_count, size_t); 388 size_t 389 spdk_mempool_count(const struct spdk_mempool *_mp) 390 { 391 struct test_mempool *mp = (struct test_mempool *)_mp; 392 393 HANDLE_RETURN_MOCK(spdk_mempool_count); 394 395 if (mp) { 396 return mp->count; 397 } else { 398 return 1024; 399 } 400 } 401 402 struct spdk_ring_ele { 403 void *ele; 404 TAILQ_ENTRY(spdk_ring_ele) link; 405 }; 406 407 struct spdk_ring { 408 TAILQ_HEAD(, spdk_ring_ele) elements; 409 pthread_mutex_t lock; 410 size_t count; 411 }; 412 413 DEFINE_RETURN_MOCK(spdk_ring_create, struct spdk_ring *); 414 struct spdk_ring * 415 spdk_ring_create(enum spdk_ring_type type, size_t count, int numa_id) 416 { 417 struct spdk_ring *ring; 418 419 HANDLE_RETURN_MOCK(spdk_ring_create); 420 421 ring = calloc(1, sizeof(*ring)); 422 if (!ring) { 423 return NULL; 424 } 425 426 if (pthread_mutex_init(&ring->lock, NULL)) { 427 free(ring); 428 return NULL; 429 } 430 431 TAILQ_INIT(&ring->elements); 432 return ring; 433 } 434 435 void 436 spdk_ring_free(struct spdk_ring *ring) 437 { 438 struct spdk_ring_ele *ele, *tmp; 439 440 if (!ring) { 441 return; 442 } 443 444 TAILQ_FOREACH_SAFE(ele, &ring->elements, link, tmp) { 445 free(ele); 446 } 447 448 pthread_mutex_destroy(&ring->lock); 449 free(ring); 450 } 451 452 DEFINE_RETURN_MOCK(spdk_ring_enqueue, size_t); 453 size_t 454 spdk_ring_enqueue(struct spdk_ring *ring, void **objs, size_t count, 455 size_t *free_space) 456 { 457 struct spdk_ring_ele *ele; 458 size_t i; 459 460 HANDLE_RETURN_MOCK(spdk_ring_enqueue); 461 462 pthread_mutex_lock(&ring->lock); 463 464 for (i = 0; i < count; i++) { 465 ele = calloc(1, sizeof(*ele)); 466 if (!ele) { 467 break; 468 } 469 470 ele->ele = objs[i]; 471 TAILQ_INSERT_TAIL(&ring->elements, ele, link); 472 ring->count++; 473 } 474 475 pthread_mutex_unlock(&ring->lock); 476 return i; 477 } 478 479 DEFINE_RETURN_MOCK(spdk_ring_dequeue, size_t); 480 size_t 481 spdk_ring_dequeue(struct spdk_ring *ring, void **objs, size_t count) 482 { 483 struct spdk_ring_ele *ele, *tmp; 484 size_t i = 0; 485 486 HANDLE_RETURN_MOCK(spdk_ring_dequeue); 487 488 if (count == 0) { 489 return 0; 490 } 491 492 pthread_mutex_lock(&ring->lock); 493 494 TAILQ_FOREACH_SAFE(ele, &ring->elements, link, tmp) { 495 TAILQ_REMOVE(&ring->elements, ele, link); 496 ring->count--; 497 objs[i] = ele->ele; 498 free(ele); 499 i++; 500 if (i >= count) { 501 break; 502 } 503 } 504 505 pthread_mutex_unlock(&ring->lock); 506 return i; 507 508 } 509 510 511 DEFINE_RETURN_MOCK(spdk_ring_count, size_t); 512 size_t 513 spdk_ring_count(struct spdk_ring *ring) 514 { 515 HANDLE_RETURN_MOCK(spdk_ring_count); 516 return ring->count; 517 } 518 519 DEFINE_RETURN_MOCK(spdk_get_ticks, uint64_t); 520 uint64_t 521 spdk_get_ticks(void) 522 { 523 HANDLE_RETURN_MOCK(spdk_get_ticks); 524 525 return ut_spdk_get_ticks; 526 } 527 528 DEFINE_RETURN_MOCK(spdk_get_ticks_hz, uint64_t); 529 uint64_t 530 spdk_get_ticks_hz(void) 531 { 532 HANDLE_RETURN_MOCK(spdk_get_ticks_hz); 533 534 return 1000000; 535 } 536 537 void 538 spdk_delay_us(unsigned int us) 539 { 540 /* spdk_get_ticks_hz is 1000000, meaning 1 tick per us. */ 541 ut_spdk_get_ticks += us; 542 } 543 544 DEFINE_RETURN_MOCK(spdk_pci_addr_parse, int); 545 int 546 spdk_pci_addr_parse(struct spdk_pci_addr *addr, const char *bdf) 547 { 548 unsigned domain, bus, dev, func; 549 550 HANDLE_RETURN_MOCK(spdk_pci_addr_parse); 551 552 if (addr == NULL || bdf == NULL) { 553 return -EINVAL; 554 } 555 556 if ((sscanf(bdf, "%x:%x:%x.%x", &domain, &bus, &dev, &func) == 4) || 557 (sscanf(bdf, "%x.%x.%x.%x", &domain, &bus, &dev, &func) == 4)) { 558 /* Matched a full address - all variables are initialized */ 559 } else if (sscanf(bdf, "%x:%x:%x", &domain, &bus, &dev) == 3) { 560 func = 0; 561 } else if ((sscanf(bdf, "%x:%x.%x", &bus, &dev, &func) == 3) || 562 (sscanf(bdf, "%x.%x.%x", &bus, &dev, &func) == 3)) { 563 domain = 0; 564 } else if ((sscanf(bdf, "%x:%x", &bus, &dev) == 2) || 565 (sscanf(bdf, "%x.%x", &bus, &dev) == 2)) { 566 domain = 0; 567 func = 0; 568 } else { 569 return -EINVAL; 570 } 571 572 if (bus > 0xFF || dev > 0x1F || func > 7) { 573 return -EINVAL; 574 } 575 576 addr->domain = domain; 577 addr->bus = bus; 578 addr->dev = dev; 579 addr->func = func; 580 581 return 0; 582 } 583 584 DEFINE_RETURN_MOCK(spdk_pci_addr_fmt, int); 585 int 586 spdk_pci_addr_fmt(char *bdf, size_t sz, const struct spdk_pci_addr *addr) 587 { 588 int rc; 589 590 HANDLE_RETURN_MOCK(spdk_pci_addr_fmt); 591 592 rc = snprintf(bdf, sz, "%04x:%02x:%02x.%x", 593 addr->domain, addr->bus, 594 addr->dev, addr->func); 595 596 if (rc > 0 && (size_t)rc < sz) { 597 return 0; 598 } 599 600 return -1; 601 } 602 603 DEFINE_RETURN_MOCK(spdk_pci_addr_compare, int); 604 int 605 spdk_pci_addr_compare(const struct spdk_pci_addr *a1, const struct spdk_pci_addr *a2) 606 { 607 HANDLE_RETURN_MOCK(spdk_pci_addr_compare); 608 609 if (a1->domain > a2->domain) { 610 return 1; 611 } else if (a1->domain < a2->domain) { 612 return -1; 613 } else if (a1->bus > a2->bus) { 614 return 1; 615 } else if (a1->bus < a2->bus) { 616 return -1; 617 } else if (a1->dev > a2->dev) { 618 return 1; 619 } else if (a1->dev < a2->dev) { 620 return -1; 621 } else if (a1->func > a2->func) { 622 return 1; 623 } else if (a1->func < a2->func) { 624 return -1; 625 } 626 627 return 0; 628 } 629