1 /* 2 * Copyright (c) Meta Platforms, Inc. and affiliates. 3 * All rights reserved. 4 * 5 * This source code is licensed under both the BSD-style license (found in the 6 * LICENSE file in the root directory of this source tree) and the GPLv2 (found 7 * in the COPYING file in the root directory of this source tree). 8 * You may select, at your option, one of the above-listed licenses. 9 */ 10 11 #if defined (__cplusplus) 12 extern "C" { 13 #endif 14 15 16 /*-**************************************** 17 * Dependencies 18 ******************************************/ 19 #include "util.h" /* note : ensure that platform.h is included first ! */ 20 #include <stdlib.h> /* malloc, realloc, free */ 21 #include <stdio.h> /* fprintf */ 22 #include <time.h> /* clock_t, clock, CLOCKS_PER_SEC, nanosleep */ 23 #include <errno.h> 24 #include <assert.h> 25 26 #if defined(__FreeBSD__) 27 #include <sys/param.h> /* __FreeBSD_version */ 28 #endif /* #ifdef __FreeBSD__ */ 29 30 #if defined(_WIN32) 31 # include <sys/utime.h> /* utime */ 32 # include <io.h> /* _chmod */ 33 # define ZSTD_USE_UTIMENSAT 0 34 #else 35 # include <unistd.h> /* chown, stat */ 36 # include <sys/stat.h> /* utimensat, st_mtime */ 37 # if (PLATFORM_POSIX_VERSION >= 200809L && defined(st_mtime)) \ 38 || (defined(__FreeBSD__) && __FreeBSD_version >= 1100056) 39 # define ZSTD_USE_UTIMENSAT 1 40 # else 41 # define ZSTD_USE_UTIMENSAT 0 42 # endif 43 # if ZSTD_USE_UTIMENSAT 44 # include <fcntl.h> /* AT_FDCWD */ 45 # else 46 # include <utime.h> /* utime */ 47 # endif 48 #endif 49 50 #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) 51 #include <direct.h> /* needed for _mkdir in windows */ 52 #endif 53 54 #if defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ 55 # include <dirent.h> /* opendir, readdir */ 56 # include <string.h> /* strerror, memcpy */ 57 #endif /* #ifdef _WIN32 */ 58 59 /*-**************************************** 60 * Internal Macros 61 ******************************************/ 62 63 /* CONTROL is almost like an assert(), but is never disabled. 64 * It's designed for failures that may happen rarely, 65 * but we don't want to maintain a specific error code path for them, 66 * such as a malloc() returning NULL for example. 67 * Since it's always active, this macro can trigger side effects. 68 */ 69 #define CONTROL(c) { \ 70 if (!(c)) { \ 71 UTIL_DISPLAYLEVEL(1, "Error : %s, %i : %s", \ 72 __FILE__, __LINE__, #c); \ 73 exit(1); \ 74 } } 75 76 /* console log */ 77 #define UTIL_DISPLAY(...) fprintf(stderr, __VA_ARGS__) 78 #define UTIL_DISPLAYLEVEL(l, ...) { if (g_utilDisplayLevel>=l) { UTIL_DISPLAY(__VA_ARGS__); } } 79 80 static int g_traceDepth = 0; 81 int g_traceFileStat = 0; 82 83 #define UTIL_TRACE_CALL(...) \ 84 { \ 85 if (g_traceFileStat) { \ 86 UTIL_DISPLAY("Trace:FileStat: %*s> ", g_traceDepth, ""); \ 87 UTIL_DISPLAY(__VA_ARGS__); \ 88 UTIL_DISPLAY("\n"); \ 89 ++g_traceDepth; \ 90 } \ 91 } 92 93 #define UTIL_TRACE_RET(ret) \ 94 { \ 95 if (g_traceFileStat) { \ 96 --g_traceDepth; \ 97 UTIL_DISPLAY("Trace:FileStat: %*s< %d\n", g_traceDepth, "", (ret)); \ 98 } \ 99 } 100 101 /* A modified version of realloc(). 102 * If UTIL_realloc() fails the original block is freed. 103 */ 104 UTIL_STATIC void* UTIL_realloc(void *ptr, size_t size) 105 { 106 void *newptr = realloc(ptr, size); 107 if (newptr) return newptr; 108 free(ptr); 109 return NULL; 110 } 111 112 #if defined(_MSC_VER) 113 #define chmod _chmod 114 #endif 115 116 #ifndef ZSTD_HAVE_FCHMOD 117 #if PLATFORM_POSIX_VERSION >= 199309L 118 #define ZSTD_HAVE_FCHMOD 119 #endif 120 #endif 121 122 #ifndef ZSTD_HAVE_FCHOWN 123 #if PLATFORM_POSIX_VERSION >= 200809L 124 #define ZSTD_HAVE_FCHOWN 125 #endif 126 #endif 127 128 /*-**************************************** 129 * Console log 130 ******************************************/ 131 int g_utilDisplayLevel; 132 133 int UTIL_requireUserConfirmation(const char* prompt, const char* abortMsg, 134 const char* acceptableLetters, int hasStdinInput) { 135 int ch, result; 136 137 if (hasStdinInput) { 138 UTIL_DISPLAY("stdin is an input - not proceeding.\n"); 139 return 1; 140 } 141 142 UTIL_DISPLAY("%s", prompt); 143 ch = getchar(); 144 result = 0; 145 if (strchr(acceptableLetters, ch) == NULL) { 146 UTIL_DISPLAY("%s \n", abortMsg); 147 result = 1; 148 } 149 /* flush the rest */ 150 while ((ch!=EOF) && (ch!='\n')) 151 ch = getchar(); 152 return result; 153 } 154 155 156 /*-************************************* 157 * Constants 158 ***************************************/ 159 #define LIST_SIZE_INCREASE (8*1024) 160 #define MAX_FILE_OF_FILE_NAMES_SIZE (1<<20)*50 161 162 163 /*-************************************* 164 * Functions 165 ***************************************/ 166 167 void UTIL_traceFileStat(void) 168 { 169 g_traceFileStat = 1; 170 } 171 172 int UTIL_fstat(const int fd, const char* filename, stat_t* statbuf) 173 { 174 int ret; 175 UTIL_TRACE_CALL("UTIL_stat(%d, %s)", fd, filename); 176 #if defined(_MSC_VER) 177 if (fd >= 0) { 178 ret = !_fstat64(fd, statbuf); 179 } else { 180 ret = !_stat64(filename, statbuf); 181 } 182 #elif defined(__MINGW32__) && defined (__MSVCRT__) 183 if (fd >= 0) { 184 ret = !_fstati64(fd, statbuf); 185 } else { 186 ret = !_stati64(filename, statbuf); 187 } 188 #else 189 if (fd >= 0) { 190 ret = !fstat(fd, statbuf); 191 } else { 192 ret = !stat(filename, statbuf); 193 } 194 #endif 195 UTIL_TRACE_RET(ret); 196 return ret; 197 } 198 199 int UTIL_stat(const char* filename, stat_t* statbuf) 200 { 201 return UTIL_fstat(-1, filename, statbuf); 202 } 203 204 int UTIL_isRegularFile(const char* infilename) 205 { 206 stat_t statbuf; 207 int ret; 208 UTIL_TRACE_CALL("UTIL_isRegularFile(%s)", infilename); 209 ret = UTIL_stat(infilename, &statbuf) && UTIL_isRegularFileStat(&statbuf); 210 UTIL_TRACE_RET(ret); 211 return ret; 212 } 213 214 int UTIL_isRegularFileStat(const stat_t* statbuf) 215 { 216 #if defined(_MSC_VER) 217 return (statbuf->st_mode & S_IFREG) != 0; 218 #else 219 return S_ISREG(statbuf->st_mode) != 0; 220 #endif 221 } 222 223 /* like chmod, but avoid changing permission of /dev/null */ 224 int UTIL_chmod(char const* filename, const stat_t* statbuf, mode_t permissions) 225 { 226 return UTIL_fchmod(-1, filename, statbuf, permissions); 227 } 228 229 int UTIL_fchmod(const int fd, char const* filename, const stat_t* statbuf, mode_t permissions) 230 { 231 stat_t localStatBuf; 232 UTIL_TRACE_CALL("UTIL_chmod(%s, %#4o)", filename, (unsigned)permissions); 233 if (statbuf == NULL) { 234 if (!UTIL_fstat(fd, filename, &localStatBuf)) { 235 UTIL_TRACE_RET(0); 236 return 0; 237 } 238 statbuf = &localStatBuf; 239 } 240 if (!UTIL_isRegularFileStat(statbuf)) { 241 UTIL_TRACE_RET(0); 242 return 0; /* pretend success, but don't change anything */ 243 } 244 #ifdef ZSTD_HAVE_FCHMOD 245 if (fd >= 0) { 246 int ret; 247 UTIL_TRACE_CALL("fchmod"); 248 ret = fchmod(fd, permissions); 249 UTIL_TRACE_RET(ret); 250 UTIL_TRACE_RET(ret); 251 return ret; 252 } else 253 #endif 254 { 255 int ret; 256 UTIL_TRACE_CALL("chmod"); 257 ret = chmod(filename, permissions); 258 UTIL_TRACE_RET(ret); 259 UTIL_TRACE_RET(ret); 260 return ret; 261 } 262 } 263 264 /* set access and modification times */ 265 int UTIL_utime(const char* filename, const stat_t *statbuf) 266 { 267 int ret; 268 UTIL_TRACE_CALL("UTIL_utime(%s)", filename); 269 /* We check that st_mtime is a macro here in order to give us confidence 270 * that struct stat has a struct timespec st_mtim member. We need this 271 * check because there are some platforms that claim to be POSIX 2008 272 * compliant but which do not have st_mtim... */ 273 /* FreeBSD has implemented POSIX 2008 for a long time but still only 274 * advertises support for POSIX 2001. They have a version macro that 275 * lets us safely gate them in. 276 * See https://docs.freebsd.org/en/books/porters-handbook/versions/. 277 */ 278 #if ZSTD_USE_UTIMENSAT 279 { 280 /* (atime, mtime) */ 281 struct timespec timebuf[2] = { {0, UTIME_NOW} }; 282 timebuf[1] = statbuf->st_mtim; 283 ret = utimensat(AT_FDCWD, filename, timebuf, 0); 284 } 285 #else 286 { 287 struct utimbuf timebuf; 288 timebuf.actime = time(NULL); 289 timebuf.modtime = statbuf->st_mtime; 290 ret = utime(filename, &timebuf); 291 } 292 #endif 293 errno = 0; 294 UTIL_TRACE_RET(ret); 295 return ret; 296 } 297 298 int UTIL_setFileStat(const char *filename, const stat_t *statbuf) 299 { 300 return UTIL_setFDStat(-1, filename, statbuf); 301 } 302 303 int UTIL_setFDStat(const int fd, const char *filename, const stat_t *statbuf) 304 { 305 int res = 0; 306 stat_t curStatBuf; 307 UTIL_TRACE_CALL("UTIL_setFileStat(%d, %s)", fd, filename); 308 309 if (!UTIL_fstat(fd, filename, &curStatBuf) || !UTIL_isRegularFileStat(&curStatBuf)) { 310 UTIL_TRACE_RET(-1); 311 return -1; 312 } 313 314 /* Mimic gzip's behavior: 315 * 316 * "Change the group first, then the permissions, then the owner. 317 * That way, the permissions will be correct on systems that allow 318 * users to give away files, without introducing a security hole. 319 * Security depends on permissions not containing the setuid or 320 * setgid bits." */ 321 322 #if !defined(_WIN32) 323 #ifdef ZSTD_HAVE_FCHOWN 324 if (fd >= 0) { 325 res += fchown(fd, -1, statbuf->st_gid); /* Apply group ownership */ 326 } else 327 #endif 328 { 329 res += chown(filename, -1, statbuf->st_gid); /* Apply group ownership */ 330 } 331 #endif 332 333 res += UTIL_fchmod(fd, filename, &curStatBuf, statbuf->st_mode & 0777); /* Copy file permissions */ 334 335 #if !defined(_WIN32) 336 #ifdef ZSTD_HAVE_FCHOWN 337 if (fd >= 0) { 338 res += fchown(fd, statbuf->st_uid, -1); /* Apply user ownership */ 339 } else 340 #endif 341 { 342 res += chown(filename, statbuf->st_uid, -1); /* Apply user ownership */ 343 } 344 #endif 345 346 errno = 0; 347 UTIL_TRACE_RET(-res); 348 return -res; /* number of errors is returned */ 349 } 350 351 int UTIL_isDirectory(const char* infilename) 352 { 353 stat_t statbuf; 354 int ret; 355 UTIL_TRACE_CALL("UTIL_isDirectory(%s)", infilename); 356 ret = UTIL_stat(infilename, &statbuf) && UTIL_isDirectoryStat(&statbuf); 357 UTIL_TRACE_RET(ret); 358 return ret; 359 } 360 361 int UTIL_isDirectoryStat(const stat_t* statbuf) 362 { 363 int ret; 364 UTIL_TRACE_CALL("UTIL_isDirectoryStat()"); 365 #if defined(_MSC_VER) 366 ret = (statbuf->st_mode & _S_IFDIR) != 0; 367 #else 368 ret = S_ISDIR(statbuf->st_mode) != 0; 369 #endif 370 UTIL_TRACE_RET(ret); 371 return ret; 372 } 373 374 int UTIL_compareStr(const void *p1, const void *p2) { 375 return strcmp(* (char * const *) p1, * (char * const *) p2); 376 } 377 378 int UTIL_isSameFile(const char* fName1, const char* fName2) 379 { 380 int ret; 381 assert(fName1 != NULL); assert(fName2 != NULL); 382 UTIL_TRACE_CALL("UTIL_isSameFile(%s, %s)", fName1, fName2); 383 #if defined(_MSC_VER) || defined(_WIN32) 384 /* note : Visual does not support file identification by inode. 385 * inode does not work on Windows, even with a posix layer, like msys2. 386 * The following work-around is limited to detecting exact name repetition only, 387 * aka `filename` is considered different from `subdir/../filename` */ 388 ret = !strcmp(fName1, fName2); 389 #else 390 { stat_t file1Stat; 391 stat_t file2Stat; 392 ret = UTIL_stat(fName1, &file1Stat) 393 && UTIL_stat(fName2, &file2Stat) 394 && UTIL_isSameFileStat(fName1, fName2, &file1Stat, &file2Stat); 395 } 396 #endif 397 UTIL_TRACE_RET(ret); 398 return ret; 399 } 400 401 int UTIL_isSameFileStat( 402 const char* fName1, const char* fName2, 403 const stat_t* file1Stat, const stat_t* file2Stat) 404 { 405 int ret; 406 assert(fName1 != NULL); assert(fName2 != NULL); 407 UTIL_TRACE_CALL("UTIL_isSameFileStat(%s, %s)", fName1, fName2); 408 #if defined(_MSC_VER) || defined(_WIN32) 409 /* note : Visual does not support file identification by inode. 410 * inode does not work on Windows, even with a posix layer, like msys2. 411 * The following work-around is limited to detecting exact name repetition only, 412 * aka `filename` is considered different from `subdir/../filename` */ 413 (void)file1Stat; 414 (void)file2Stat; 415 ret = !strcmp(fName1, fName2); 416 #else 417 { 418 ret = (file1Stat->st_dev == file2Stat->st_dev) 419 && (file1Stat->st_ino == file2Stat->st_ino); 420 } 421 #endif 422 UTIL_TRACE_RET(ret); 423 return ret; 424 } 425 426 /* UTIL_isFIFO : distinguish named pipes */ 427 int UTIL_isFIFO(const char* infilename) 428 { 429 UTIL_TRACE_CALL("UTIL_isFIFO(%s)", infilename); 430 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ 431 #if PLATFORM_POSIX_VERSION >= 200112L 432 { 433 stat_t statbuf; 434 if (UTIL_stat(infilename, &statbuf) && UTIL_isFIFOStat(&statbuf)) { 435 UTIL_TRACE_RET(1); 436 return 1; 437 } 438 } 439 #endif 440 (void)infilename; 441 UTIL_TRACE_RET(0); 442 return 0; 443 } 444 445 /* UTIL_isFIFO : distinguish named pipes */ 446 int UTIL_isFIFOStat(const stat_t* statbuf) 447 { 448 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ 449 #if PLATFORM_POSIX_VERSION >= 200112L 450 if (S_ISFIFO(statbuf->st_mode)) return 1; 451 #endif 452 (void)statbuf; 453 return 0; 454 } 455 456 /* UTIL_isBlockDevStat : distinguish named pipes */ 457 int UTIL_isBlockDevStat(const stat_t* statbuf) 458 { 459 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ 460 #if PLATFORM_POSIX_VERSION >= 200112L 461 if (S_ISBLK(statbuf->st_mode)) return 1; 462 #endif 463 (void)statbuf; 464 return 0; 465 } 466 467 int UTIL_isLink(const char* infilename) 468 { 469 UTIL_TRACE_CALL("UTIL_isLink(%s)", infilename); 470 /* macro guards, as defined in : https://linux.die.net/man/2/lstat */ 471 #if PLATFORM_POSIX_VERSION >= 200112L 472 { 473 stat_t statbuf; 474 int const r = lstat(infilename, &statbuf); 475 if (!r && S_ISLNK(statbuf.st_mode)) { 476 UTIL_TRACE_RET(1); 477 return 1; 478 } 479 } 480 #endif 481 (void)infilename; 482 UTIL_TRACE_RET(0); 483 return 0; 484 } 485 486 static int g_fakeStdinIsConsole = 0; 487 static int g_fakeStderrIsConsole = 0; 488 static int g_fakeStdoutIsConsole = 0; 489 490 int UTIL_isConsole(FILE* file) 491 { 492 int ret; 493 UTIL_TRACE_CALL("UTIL_isConsole(%d)", fileno(file)); 494 if (file == stdin && g_fakeStdinIsConsole) 495 ret = 1; 496 else if (file == stderr && g_fakeStderrIsConsole) 497 ret = 1; 498 else if (file == stdout && g_fakeStdoutIsConsole) 499 ret = 1; 500 else 501 ret = IS_CONSOLE(file); 502 UTIL_TRACE_RET(ret); 503 return ret; 504 } 505 506 void UTIL_fakeStdinIsConsole(void) 507 { 508 g_fakeStdinIsConsole = 1; 509 } 510 void UTIL_fakeStdoutIsConsole(void) 511 { 512 g_fakeStdoutIsConsole = 1; 513 } 514 void UTIL_fakeStderrIsConsole(void) 515 { 516 g_fakeStderrIsConsole = 1; 517 } 518 519 U64 UTIL_getFileSize(const char* infilename) 520 { 521 stat_t statbuf; 522 UTIL_TRACE_CALL("UTIL_getFileSize(%s)", infilename); 523 if (!UTIL_stat(infilename, &statbuf)) { 524 UTIL_TRACE_RET(-1); 525 return UTIL_FILESIZE_UNKNOWN; 526 } 527 { 528 U64 const size = UTIL_getFileSizeStat(&statbuf); 529 UTIL_TRACE_RET((int)size); 530 return size; 531 } 532 } 533 534 U64 UTIL_getFileSizeStat(const stat_t* statbuf) 535 { 536 if (!UTIL_isRegularFileStat(statbuf)) return UTIL_FILESIZE_UNKNOWN; 537 #if defined(_MSC_VER) 538 if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; 539 #elif defined(__MINGW32__) && defined (__MSVCRT__) 540 if (!(statbuf->st_mode & S_IFREG)) return UTIL_FILESIZE_UNKNOWN; 541 #else 542 if (!S_ISREG(statbuf->st_mode)) return UTIL_FILESIZE_UNKNOWN; 543 #endif 544 return (U64)statbuf->st_size; 545 } 546 547 UTIL_HumanReadableSize_t UTIL_makeHumanReadableSize(U64 size) 548 { 549 UTIL_HumanReadableSize_t hrs; 550 551 if (g_utilDisplayLevel > 3) { 552 /* In verbose mode, do not scale sizes down, except in the case of 553 * values that exceed the integral precision of a double. */ 554 if (size >= (1ull << 53)) { 555 hrs.value = (double)size / (1ull << 20); 556 hrs.suffix = " MiB"; 557 /* At worst, a double representation of a maximal size will be 558 * accurate to better than tens of kilobytes. */ 559 hrs.precision = 2; 560 } else { 561 hrs.value = (double)size; 562 hrs.suffix = " B"; 563 hrs.precision = 0; 564 } 565 } else { 566 /* In regular mode, scale sizes down and use suffixes. */ 567 if (size >= (1ull << 60)) { 568 hrs.value = (double)size / (1ull << 60); 569 hrs.suffix = " EiB"; 570 } else if (size >= (1ull << 50)) { 571 hrs.value = (double)size / (1ull << 50); 572 hrs.suffix = " PiB"; 573 } else if (size >= (1ull << 40)) { 574 hrs.value = (double)size / (1ull << 40); 575 hrs.suffix = " TiB"; 576 } else if (size >= (1ull << 30)) { 577 hrs.value = (double)size / (1ull << 30); 578 hrs.suffix = " GiB"; 579 } else if (size >= (1ull << 20)) { 580 hrs.value = (double)size / (1ull << 20); 581 hrs.suffix = " MiB"; 582 } else if (size >= (1ull << 10)) { 583 hrs.value = (double)size / (1ull << 10); 584 hrs.suffix = " KiB"; 585 } else { 586 hrs.value = (double)size; 587 hrs.suffix = " B"; 588 } 589 590 if (hrs.value >= 100 || (U64)hrs.value == size) { 591 hrs.precision = 0; 592 } else if (hrs.value >= 10) { 593 hrs.precision = 1; 594 } else if (hrs.value > 1) { 595 hrs.precision = 2; 596 } else { 597 hrs.precision = 3; 598 } 599 } 600 601 return hrs; 602 } 603 604 U64 UTIL_getTotalFileSize(const char* const * fileNamesTable, unsigned nbFiles) 605 { 606 U64 total = 0; 607 unsigned n; 608 UTIL_TRACE_CALL("UTIL_getTotalFileSize(%u)", nbFiles); 609 for (n=0; n<nbFiles; n++) { 610 U64 const size = UTIL_getFileSize(fileNamesTable[n]); 611 if (size == UTIL_FILESIZE_UNKNOWN) { 612 UTIL_TRACE_RET(-1); 613 return UTIL_FILESIZE_UNKNOWN; 614 } 615 total += size; 616 } 617 UTIL_TRACE_RET((int)total); 618 return total; 619 } 620 621 622 /* condition : @file must be valid, and not have reached its end. 623 * @return : length of line written into @buf, ended with `\0` instead of '\n', 624 * or 0, if there is no new line */ 625 static size_t readLineFromFile(char* buf, size_t len, FILE* file) 626 { 627 assert(!feof(file)); 628 if ( fgets(buf, (int) len, file) == NULL ) return 0; 629 { size_t linelen = strlen(buf); 630 if (strlen(buf)==0) return 0; 631 if (buf[linelen-1] == '\n') linelen--; 632 buf[linelen] = '\0'; 633 return linelen+1; 634 } 635 } 636 637 /* Conditions : 638 * size of @inputFileName file must be < @dstCapacity 639 * @dst must be initialized 640 * @return : nb of lines 641 * or -1 if there's an error 642 */ 643 static int 644 readLinesFromFile(void* dst, size_t dstCapacity, 645 const char* inputFileName) 646 { 647 int nbFiles = 0; 648 size_t pos = 0; 649 char* const buf = (char*)dst; 650 FILE* const inputFile = fopen(inputFileName, "r"); 651 652 assert(dst != NULL); 653 654 if(!inputFile) { 655 if (g_utilDisplayLevel >= 1) perror("zstd:util:readLinesFromFile"); 656 return -1; 657 } 658 659 while ( !feof(inputFile) ) { 660 size_t const lineLength = readLineFromFile(buf+pos, dstCapacity-pos, inputFile); 661 if (lineLength == 0) break; 662 assert(pos + lineLength <= dstCapacity); /* '=' for inputFile not terminated with '\n' */ 663 pos += lineLength; 664 ++nbFiles; 665 } 666 667 CONTROL( fclose(inputFile) == 0 ); 668 669 return nbFiles; 670 } 671 672 /*Note: buf is not freed in case function successfully created table because filesTable->fileNames[0] = buf*/ 673 FileNamesTable* 674 UTIL_createFileNamesTable_fromFileName(const char* inputFileName) 675 { 676 size_t nbFiles = 0; 677 char* buf; 678 size_t bufSize; 679 stat_t statbuf; 680 681 if (!UTIL_stat(inputFileName, &statbuf) || !UTIL_isRegularFileStat(&statbuf)) 682 return NULL; 683 684 { U64 const inputFileSize = UTIL_getFileSizeStat(&statbuf); 685 if(inputFileSize > MAX_FILE_OF_FILE_NAMES_SIZE) 686 return NULL; 687 bufSize = (size_t)(inputFileSize + 1); /* (+1) to add '\0' at the end of last filename */ 688 } 689 690 buf = (char*) malloc(bufSize); 691 CONTROL( buf != NULL ); 692 693 { int const ret_nbFiles = readLinesFromFile(buf, bufSize, inputFileName); 694 695 if (ret_nbFiles <= 0) { 696 free(buf); 697 return NULL; 698 } 699 nbFiles = (size_t)ret_nbFiles; 700 } 701 702 { const char** filenamesTable = (const char**) malloc(nbFiles * sizeof(*filenamesTable)); 703 CONTROL(filenamesTable != NULL); 704 705 { size_t fnb, pos = 0; 706 for (fnb = 0; fnb < nbFiles; fnb++) { 707 filenamesTable[fnb] = buf+pos; 708 pos += strlen(buf+pos)+1; /* +1 for the finishing `\0` */ 709 } 710 assert(pos <= bufSize); 711 } 712 713 return UTIL_assembleFileNamesTable(filenamesTable, nbFiles, buf); 714 } 715 } 716 717 static FileNamesTable* 718 UTIL_assembleFileNamesTable2(const char** filenames, size_t tableSize, size_t tableCapacity, char* buf) 719 { 720 FileNamesTable* const table = (FileNamesTable*) malloc(sizeof(*table)); 721 CONTROL(table != NULL); 722 table->fileNames = filenames; 723 table->buf = buf; 724 table->tableSize = tableSize; 725 table->tableCapacity = tableCapacity; 726 return table; 727 } 728 729 FileNamesTable* 730 UTIL_assembleFileNamesTable(const char** filenames, size_t tableSize, char* buf) 731 { 732 return UTIL_assembleFileNamesTable2(filenames, tableSize, tableSize, buf); 733 } 734 735 void UTIL_freeFileNamesTable(FileNamesTable* table) 736 { 737 if (table==NULL) return; 738 free((void*)table->fileNames); 739 free(table->buf); 740 free(table); 741 } 742 743 FileNamesTable* UTIL_allocateFileNamesTable(size_t tableSize) 744 { 745 const char** const fnTable = (const char**)malloc(tableSize * sizeof(*fnTable)); 746 FileNamesTable* fnt; 747 if (fnTable==NULL) return NULL; 748 fnt = UTIL_assembleFileNamesTable(fnTable, tableSize, NULL); 749 fnt->tableSize = 0; /* the table is empty */ 750 return fnt; 751 } 752 753 int UTIL_searchFileNamesTable(FileNamesTable* table, char const* name) { 754 size_t i; 755 for(i=0 ;i < table->tableSize; i++) { 756 if(!strcmp(table->fileNames[i], name)) { 757 return (int)i; 758 } 759 } 760 return -1; 761 } 762 763 void UTIL_refFilename(FileNamesTable* fnt, const char* filename) 764 { 765 assert(fnt->tableSize < fnt->tableCapacity); 766 fnt->fileNames[fnt->tableSize] = filename; 767 fnt->tableSize++; 768 } 769 770 static size_t getTotalTableSize(FileNamesTable* table) 771 { 772 size_t fnb, totalSize = 0; 773 for(fnb = 0 ; fnb < table->tableSize && table->fileNames[fnb] ; ++fnb) { 774 totalSize += strlen(table->fileNames[fnb]) + 1; /* +1 to add '\0' at the end of each fileName */ 775 } 776 return totalSize; 777 } 778 779 FileNamesTable* 780 UTIL_mergeFileNamesTable(FileNamesTable* table1, FileNamesTable* table2) 781 { 782 unsigned newTableIdx = 0; 783 size_t pos = 0; 784 size_t newTotalTableSize; 785 char* buf; 786 787 FileNamesTable* const newTable = UTIL_assembleFileNamesTable(NULL, 0, NULL); 788 CONTROL( newTable != NULL ); 789 790 newTotalTableSize = getTotalTableSize(table1) + getTotalTableSize(table2); 791 792 buf = (char*) calloc(newTotalTableSize, sizeof(*buf)); 793 CONTROL ( buf != NULL ); 794 795 newTable->buf = buf; 796 newTable->tableSize = table1->tableSize + table2->tableSize; 797 newTable->fileNames = (const char **) calloc(newTable->tableSize, sizeof(*(newTable->fileNames))); 798 CONTROL ( newTable->fileNames != NULL ); 799 800 { unsigned idx1; 801 for( idx1=0 ; (idx1 < table1->tableSize) && table1->fileNames[idx1] && (pos < newTotalTableSize); ++idx1, ++newTableIdx) { 802 size_t const curLen = strlen(table1->fileNames[idx1]); 803 memcpy(buf+pos, table1->fileNames[idx1], curLen); 804 assert(newTableIdx <= newTable->tableSize); 805 newTable->fileNames[newTableIdx] = buf+pos; 806 pos += curLen+1; 807 } } 808 809 { unsigned idx2; 810 for( idx2=0 ; (idx2 < table2->tableSize) && table2->fileNames[idx2] && (pos < newTotalTableSize) ; ++idx2, ++newTableIdx) { 811 size_t const curLen = strlen(table2->fileNames[idx2]); 812 memcpy(buf+pos, table2->fileNames[idx2], curLen); 813 assert(newTableIdx < newTable->tableSize); 814 newTable->fileNames[newTableIdx] = buf+pos; 815 pos += curLen+1; 816 } } 817 assert(pos <= newTotalTableSize); 818 newTable->tableSize = newTableIdx; 819 820 UTIL_freeFileNamesTable(table1); 821 UTIL_freeFileNamesTable(table2); 822 823 return newTable; 824 } 825 826 #ifdef _WIN32 827 static int UTIL_prepareFileList(const char* dirName, 828 char** bufStart, size_t* pos, 829 char** bufEnd, int followLinks) 830 { 831 char* path; 832 size_t dirLength, pathLength; 833 int nbFiles = 0; 834 WIN32_FIND_DATAA cFile; 835 HANDLE hFile; 836 837 dirLength = strlen(dirName); 838 path = (char*) malloc(dirLength + 3); 839 if (!path) return 0; 840 841 memcpy(path, dirName, dirLength); 842 path[dirLength] = '\\'; 843 path[dirLength+1] = '*'; 844 path[dirLength+2] = 0; 845 846 hFile=FindFirstFileA(path, &cFile); 847 if (hFile == INVALID_HANDLE_VALUE) { 848 UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s'\n", dirName); 849 return 0; 850 } 851 free(path); 852 853 do { 854 size_t const fnameLength = strlen(cFile.cFileName); 855 path = (char*) malloc(dirLength + fnameLength + 2); 856 if (!path) { FindClose(hFile); return 0; } 857 memcpy(path, dirName, dirLength); 858 path[dirLength] = '\\'; 859 memcpy(path+dirLength+1, cFile.cFileName, fnameLength); 860 pathLength = dirLength+1+fnameLength; 861 path[pathLength] = 0; 862 if (cFile.dwFileAttributes & FILE_ATTRIBUTE_DIRECTORY) { 863 if ( strcmp (cFile.cFileName, "..") == 0 864 || strcmp (cFile.cFileName, ".") == 0 ) 865 continue; 866 /* Recursively call "UTIL_prepareFileList" with the new path. */ 867 nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); 868 if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } 869 } else if ( (cFile.dwFileAttributes & FILE_ATTRIBUTE_NORMAL) 870 || (cFile.dwFileAttributes & FILE_ATTRIBUTE_ARCHIVE) 871 || (cFile.dwFileAttributes & FILE_ATTRIBUTE_COMPRESSED) ) { 872 if (*bufStart + *pos + pathLength >= *bufEnd) { 873 ptrdiff_t const newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; 874 *bufStart = (char*)UTIL_realloc(*bufStart, newListSize); 875 if (*bufStart == NULL) { free(path); FindClose(hFile); return 0; } 876 *bufEnd = *bufStart + newListSize; 877 } 878 if (*bufStart + *pos + pathLength < *bufEnd) { 879 memcpy(*bufStart + *pos, path, pathLength+1 /* include final \0 */); 880 *pos += pathLength + 1; 881 nbFiles++; 882 } } 883 free(path); 884 } while (FindNextFileA(hFile, &cFile)); 885 886 FindClose(hFile); 887 return nbFiles; 888 } 889 890 #elif defined(__linux__) || (PLATFORM_POSIX_VERSION >= 200112L) /* opendir, readdir require POSIX.1-2001 */ 891 892 static int UTIL_prepareFileList(const char *dirName, 893 char** bufStart, size_t* pos, 894 char** bufEnd, int followLinks) 895 { 896 DIR* dir; 897 struct dirent * entry; 898 size_t dirLength; 899 int nbFiles = 0; 900 901 if (!(dir = opendir(dirName))) { 902 UTIL_DISPLAYLEVEL(1, "Cannot open directory '%s': %s\n", dirName, strerror(errno)); 903 return 0; 904 } 905 906 dirLength = strlen(dirName); 907 errno = 0; 908 while ((entry = readdir(dir)) != NULL) { 909 char* path; 910 size_t fnameLength, pathLength; 911 if (strcmp (entry->d_name, "..") == 0 || 912 strcmp (entry->d_name, ".") == 0) continue; 913 fnameLength = strlen(entry->d_name); 914 path = (char*) malloc(dirLength + fnameLength + 2); 915 if (!path) { closedir(dir); return 0; } 916 memcpy(path, dirName, dirLength); 917 918 path[dirLength] = '/'; 919 memcpy(path+dirLength+1, entry->d_name, fnameLength); 920 pathLength = dirLength+1+fnameLength; 921 path[pathLength] = 0; 922 923 if (!followLinks && UTIL_isLink(path)) { 924 UTIL_DISPLAYLEVEL(2, "Warning : %s is a symbolic link, ignoring\n", path); 925 free(path); 926 continue; 927 } 928 929 if (UTIL_isDirectory(path)) { 930 nbFiles += UTIL_prepareFileList(path, bufStart, pos, bufEnd, followLinks); /* Recursively call "UTIL_prepareFileList" with the new path. */ 931 if (*bufStart == NULL) { free(path); closedir(dir); return 0; } 932 } else { 933 if (*bufStart + *pos + pathLength >= *bufEnd) { 934 ptrdiff_t newListSize = (*bufEnd - *bufStart) + LIST_SIZE_INCREASE; 935 assert(newListSize >= 0); 936 *bufStart = (char*)UTIL_realloc(*bufStart, (size_t)newListSize); 937 if (*bufStart != NULL) { 938 *bufEnd = *bufStart + newListSize; 939 } else { 940 free(path); closedir(dir); return 0; 941 } 942 } 943 if (*bufStart + *pos + pathLength < *bufEnd) { 944 memcpy(*bufStart + *pos, path, pathLength + 1); /* with final \0 */ 945 *pos += pathLength + 1; 946 nbFiles++; 947 } } 948 free(path); 949 errno = 0; /* clear errno after UTIL_isDirectory, UTIL_prepareFileList */ 950 } 951 952 if (errno != 0) { 953 UTIL_DISPLAYLEVEL(1, "readdir(%s) error: %s \n", dirName, strerror(errno)); 954 free(*bufStart); 955 *bufStart = NULL; 956 } 957 closedir(dir); 958 return nbFiles; 959 } 960 961 #else 962 963 static int UTIL_prepareFileList(const char *dirName, 964 char** bufStart, size_t* pos, 965 char** bufEnd, int followLinks) 966 { 967 (void)bufStart; (void)bufEnd; (void)pos; (void)followLinks; 968 UTIL_DISPLAYLEVEL(1, "Directory %s ignored (compiled without _WIN32 or _POSIX_C_SOURCE) \n", dirName); 969 return 0; 970 } 971 972 #endif /* #ifdef _WIN32 */ 973 974 int UTIL_isCompressedFile(const char *inputName, const char *extensionList[]) 975 { 976 const char* ext = UTIL_getFileExtension(inputName); 977 while(*extensionList!=NULL) 978 { 979 const int isCompressedExtension = strcmp(ext,*extensionList); 980 if(isCompressedExtension==0) 981 return 1; 982 ++extensionList; 983 } 984 return 0; 985 } 986 987 /*Utility function to get file extension from file */ 988 const char* UTIL_getFileExtension(const char* infilename) 989 { 990 const char* extension = strrchr(infilename, '.'); 991 if(!extension || extension==infilename) return ""; 992 return extension; 993 } 994 995 static int pathnameHas2Dots(const char *pathname) 996 { 997 /* We need to figure out whether any ".." present in the path is a whole 998 * path token, which is the case if it is bordered on both sides by either 999 * the beginning/end of the path or by a directory separator. 1000 */ 1001 const char *needle = pathname; 1002 while (1) { 1003 needle = strstr(needle, ".."); 1004 1005 if (needle == NULL) { 1006 return 0; 1007 } 1008 1009 if ((needle == pathname || needle[-1] == PATH_SEP) 1010 && (needle[2] == '\0' || needle[2] == PATH_SEP)) { 1011 return 1; 1012 } 1013 1014 /* increment so we search for the next match */ 1015 needle++; 1016 }; 1017 return 0; 1018 } 1019 1020 static int isFileNameValidForMirroredOutput(const char *filename) 1021 { 1022 return !pathnameHas2Dots(filename); 1023 } 1024 1025 1026 #define DIR_DEFAULT_MODE 0755 1027 static mode_t getDirMode(const char *dirName) 1028 { 1029 stat_t st; 1030 if (!UTIL_stat(dirName, &st)) { 1031 UTIL_DISPLAY("zstd: failed to get DIR stats %s: %s\n", dirName, strerror(errno)); 1032 return DIR_DEFAULT_MODE; 1033 } 1034 if (!UTIL_isDirectoryStat(&st)) { 1035 UTIL_DISPLAY("zstd: expected directory: %s\n", dirName); 1036 return DIR_DEFAULT_MODE; 1037 } 1038 return st.st_mode; 1039 } 1040 1041 static int makeDir(const char *dir, mode_t mode) 1042 { 1043 #if defined(_MSC_VER) || defined(__MINGW32__) || defined (__MSVCRT__) 1044 int ret = _mkdir(dir); 1045 (void) mode; 1046 #else 1047 int ret = mkdir(dir, mode); 1048 #endif 1049 if (ret != 0) { 1050 if (errno == EEXIST) 1051 return 0; 1052 UTIL_DISPLAY("zstd: failed to create DIR %s: %s\n", dir, strerror(errno)); 1053 } 1054 return ret; 1055 } 1056 1057 /* this function requires a mutable input string */ 1058 static void convertPathnameToDirName(char *pathname) 1059 { 1060 size_t len = 0; 1061 char* pos = NULL; 1062 /* get dir name from pathname similar to 'dirname()' */ 1063 assert(pathname != NULL); 1064 1065 /* remove trailing '/' chars */ 1066 len = strlen(pathname); 1067 assert(len > 0); 1068 while (pathname[len] == PATH_SEP) { 1069 pathname[len] = '\0'; 1070 len--; 1071 } 1072 if (len == 0) return; 1073 1074 /* if input is a single file, return '.' instead. i.e. 1075 * "xyz/abc/file.txt" => "xyz/abc" 1076 "./file.txt" => "." 1077 "file.txt" => "." 1078 */ 1079 pos = strrchr(pathname, PATH_SEP); 1080 if (pos == NULL) { 1081 pathname[0] = '.'; 1082 pathname[1] = '\0'; 1083 } else { 1084 *pos = '\0'; 1085 } 1086 } 1087 1088 /* pathname must be valid */ 1089 static const char* trimLeadingRootChar(const char *pathname) 1090 { 1091 assert(pathname != NULL); 1092 if (pathname[0] == PATH_SEP) 1093 return pathname + 1; 1094 return pathname; 1095 } 1096 1097 /* pathname must be valid */ 1098 static const char* trimLeadingCurrentDirConst(const char *pathname) 1099 { 1100 assert(pathname != NULL); 1101 if ((pathname[0] == '.') && (pathname[1] == PATH_SEP)) 1102 return pathname + 2; 1103 return pathname; 1104 } 1105 1106 static char* 1107 trimLeadingCurrentDir(char *pathname) 1108 { 1109 /* 'union charunion' can do const-cast without compiler warning */ 1110 union charunion { 1111 char *chr; 1112 const char* cchr; 1113 } ptr; 1114 ptr.cchr = trimLeadingCurrentDirConst(pathname); 1115 return ptr.chr; 1116 } 1117 1118 /* remove leading './' or '/' chars here */ 1119 static const char * trimPath(const char *pathname) 1120 { 1121 return trimLeadingRootChar( 1122 trimLeadingCurrentDirConst(pathname)); 1123 } 1124 1125 static char* mallocAndJoin2Dir(const char *dir1, const char *dir2) 1126 { 1127 assert(dir1 != NULL && dir2 != NULL); 1128 { const size_t dir1Size = strlen(dir1); 1129 const size_t dir2Size = strlen(dir2); 1130 char *outDirBuffer, *buffer; 1131 1132 outDirBuffer = (char *) malloc(dir1Size + dir2Size + 2); 1133 CONTROL(outDirBuffer != NULL); 1134 1135 memcpy(outDirBuffer, dir1, dir1Size); 1136 outDirBuffer[dir1Size] = '\0'; 1137 1138 buffer = outDirBuffer + dir1Size; 1139 if (dir1Size > 0 && *(buffer - 1) != PATH_SEP) { 1140 *buffer = PATH_SEP; 1141 buffer++; 1142 } 1143 memcpy(buffer, dir2, dir2Size); 1144 buffer[dir2Size] = '\0'; 1145 1146 return outDirBuffer; 1147 } 1148 } 1149 1150 /* this function will return NULL if input srcFileName is not valid name for mirrored output path */ 1151 char* UTIL_createMirroredDestDirName(const char* srcFileName, const char* outDirRootName) 1152 { 1153 char* pathname = NULL; 1154 if (!isFileNameValidForMirroredOutput(srcFileName)) 1155 return NULL; 1156 1157 pathname = mallocAndJoin2Dir(outDirRootName, trimPath(srcFileName)); 1158 1159 convertPathnameToDirName(pathname); 1160 return pathname; 1161 } 1162 1163 static int 1164 mirrorSrcDir(char* srcDirName, const char* outDirName) 1165 { 1166 mode_t srcMode; 1167 int status = 0; 1168 char* newDir = mallocAndJoin2Dir(outDirName, trimPath(srcDirName)); 1169 if (!newDir) 1170 return -ENOMEM; 1171 1172 srcMode = getDirMode(srcDirName); 1173 status = makeDir(newDir, srcMode); 1174 free(newDir); 1175 return status; 1176 } 1177 1178 static int 1179 mirrorSrcDirRecursive(char* srcDirName, const char* outDirName) 1180 { 1181 int status = 0; 1182 char* pp = trimLeadingCurrentDir(srcDirName); 1183 char* sp = NULL; 1184 1185 while ((sp = strchr(pp, PATH_SEP)) != NULL) { 1186 if (sp != pp) { 1187 *sp = '\0'; 1188 status = mirrorSrcDir(srcDirName, outDirName); 1189 if (status != 0) 1190 return status; 1191 *sp = PATH_SEP; 1192 } 1193 pp = sp + 1; 1194 } 1195 status = mirrorSrcDir(srcDirName, outDirName); 1196 return status; 1197 } 1198 1199 static void 1200 makeMirroredDestDirsWithSameSrcDirMode(char** srcDirNames, unsigned nbFile, const char* outDirName) 1201 { 1202 unsigned int i = 0; 1203 for (i = 0; i < nbFile; i++) 1204 mirrorSrcDirRecursive(srcDirNames[i], outDirName); 1205 } 1206 1207 static int 1208 firstIsParentOrSameDirOfSecond(const char* firstDir, const char* secondDir) 1209 { 1210 size_t firstDirLen = strlen(firstDir), 1211 secondDirLen = strlen(secondDir); 1212 return firstDirLen <= secondDirLen && 1213 (secondDir[firstDirLen] == PATH_SEP || secondDir[firstDirLen] == '\0') && 1214 0 == strncmp(firstDir, secondDir, firstDirLen); 1215 } 1216 1217 static int compareDir(const void* pathname1, const void* pathname2) { 1218 /* sort it after remove the leading '/' or './'*/ 1219 const char* s1 = trimPath(*(char * const *) pathname1); 1220 const char* s2 = trimPath(*(char * const *) pathname2); 1221 return strcmp(s1, s2); 1222 } 1223 1224 static void 1225 makeUniqueMirroredDestDirs(char** srcDirNames, unsigned nbFile, const char* outDirName) 1226 { 1227 unsigned int i = 0, uniqueDirNr = 0; 1228 char** uniqueDirNames = NULL; 1229 1230 if (nbFile == 0) 1231 return; 1232 1233 uniqueDirNames = (char** ) malloc(nbFile * sizeof (char *)); 1234 CONTROL(uniqueDirNames != NULL); 1235 1236 /* if dirs is "a/b/c" and "a/b/c/d", we only need call: 1237 * we just need "a/b/c/d" */ 1238 qsort((void *)srcDirNames, nbFile, sizeof(char*), compareDir); 1239 1240 uniqueDirNr = 1; 1241 uniqueDirNames[uniqueDirNr - 1] = srcDirNames[0]; 1242 for (i = 1; i < nbFile; i++) { 1243 char* prevDirName = srcDirNames[i - 1]; 1244 char* currDirName = srcDirNames[i]; 1245 1246 /* note: we always compare trimmed path, i.e.: 1247 * src dir of "./foo" and "/foo" will be both saved into: 1248 * "outDirName/foo/" */ 1249 if (!firstIsParentOrSameDirOfSecond(trimPath(prevDirName), 1250 trimPath(currDirName))) 1251 uniqueDirNr++; 1252 1253 /* we need to maintain original src dir name instead of trimmed 1254 * dir, so we can retrieve the original src dir's mode_t */ 1255 uniqueDirNames[uniqueDirNr - 1] = currDirName; 1256 } 1257 1258 makeMirroredDestDirsWithSameSrcDirMode(uniqueDirNames, uniqueDirNr, outDirName); 1259 1260 free(uniqueDirNames); 1261 } 1262 1263 static void 1264 makeMirroredDestDirs(char** srcFileNames, unsigned nbFile, const char* outDirName) 1265 { 1266 unsigned int i = 0; 1267 for (i = 0; i < nbFile; ++i) 1268 convertPathnameToDirName(srcFileNames[i]); 1269 makeUniqueMirroredDestDirs(srcFileNames, nbFile, outDirName); 1270 } 1271 1272 void UTIL_mirrorSourceFilesDirectories(const char** inFileNames, unsigned int nbFile, const char* outDirName) 1273 { 1274 unsigned int i = 0, validFilenamesNr = 0; 1275 char** srcFileNames = (char **) malloc(nbFile * sizeof (char *)); 1276 CONTROL(srcFileNames != NULL); 1277 1278 /* check input filenames is valid */ 1279 for (i = 0; i < nbFile; ++i) { 1280 if (isFileNameValidForMirroredOutput(inFileNames[i])) { 1281 char* fname = STRDUP(inFileNames[i]); 1282 CONTROL(fname != NULL); 1283 srcFileNames[validFilenamesNr++] = fname; 1284 } 1285 } 1286 1287 if (validFilenamesNr > 0) { 1288 makeDir(outDirName, DIR_DEFAULT_MODE); 1289 makeMirroredDestDirs(srcFileNames, validFilenamesNr, outDirName); 1290 } 1291 1292 for (i = 0; i < validFilenamesNr; i++) 1293 free(srcFileNames[i]); 1294 free(srcFileNames); 1295 } 1296 1297 FileNamesTable* 1298 UTIL_createExpandedFNT(const char* const* inputNames, size_t nbIfns, int followLinks) 1299 { 1300 unsigned nbFiles; 1301 char* buf = (char*)malloc(LIST_SIZE_INCREASE); 1302 char* bufend = buf + LIST_SIZE_INCREASE; 1303 1304 if (!buf) return NULL; 1305 1306 { size_t ifnNb, pos; 1307 for (ifnNb=0, pos=0, nbFiles=0; ifnNb<nbIfns; ifnNb++) { 1308 if (!UTIL_isDirectory(inputNames[ifnNb])) { 1309 size_t const len = strlen(inputNames[ifnNb]); 1310 if (buf + pos + len >= bufend) { 1311 ptrdiff_t newListSize = (bufend - buf) + LIST_SIZE_INCREASE; 1312 assert(newListSize >= 0); 1313 buf = (char*)UTIL_realloc(buf, (size_t)newListSize); 1314 if (!buf) return NULL; 1315 bufend = buf + newListSize; 1316 } 1317 if (buf + pos + len < bufend) { 1318 memcpy(buf+pos, inputNames[ifnNb], len+1); /* including final \0 */ 1319 pos += len + 1; 1320 nbFiles++; 1321 } 1322 } else { 1323 nbFiles += (unsigned)UTIL_prepareFileList(inputNames[ifnNb], &buf, &pos, &bufend, followLinks); 1324 if (buf == NULL) return NULL; 1325 } } } 1326 1327 /* note : even if nbFiles==0, function returns a valid, though empty, FileNamesTable* object */ 1328 1329 { size_t ifnNb, pos; 1330 size_t const fntCapacity = nbFiles + 1; /* minimum 1, allows adding one reference, typically stdin */ 1331 const char** const fileNamesTable = (const char**)malloc(fntCapacity * sizeof(*fileNamesTable)); 1332 if (!fileNamesTable) { free(buf); return NULL; } 1333 1334 for (ifnNb = 0, pos = 0; ifnNb < nbFiles; ifnNb++) { 1335 fileNamesTable[ifnNb] = buf + pos; 1336 if (buf + pos > bufend) { free(buf); free((void*)fileNamesTable); return NULL; } 1337 pos += strlen(fileNamesTable[ifnNb]) + 1; 1338 } 1339 return UTIL_assembleFileNamesTable2(fileNamesTable, nbFiles, fntCapacity, buf); 1340 } 1341 } 1342 1343 1344 void UTIL_expandFNT(FileNamesTable** fnt, int followLinks) 1345 { 1346 FileNamesTable* const newFNT = UTIL_createExpandedFNT((*fnt)->fileNames, (*fnt)->tableSize, followLinks); 1347 CONTROL(newFNT != NULL); 1348 UTIL_freeFileNamesTable(*fnt); 1349 *fnt = newFNT; 1350 } 1351 1352 FileNamesTable* UTIL_createFNT_fromROTable(const char** filenames, size_t nbFilenames) 1353 { 1354 size_t const sizeof_FNTable = nbFilenames * sizeof(*filenames); 1355 const char** const newFNTable = (const char**)malloc(sizeof_FNTable); 1356 if (newFNTable==NULL) return NULL; 1357 memcpy((void*)newFNTable, filenames, sizeof_FNTable); /* void* : mitigate a Visual compiler bug or limitation */ 1358 return UTIL_assembleFileNamesTable(newFNTable, nbFilenames, NULL); 1359 } 1360 1361 1362 /*-**************************************** 1363 * count the number of cores 1364 ******************************************/ 1365 1366 #if defined(_WIN32) || defined(WIN32) 1367 1368 #include <windows.h> 1369 1370 typedef BOOL(WINAPI* LPFN_GLPI)(PSYSTEM_LOGICAL_PROCESSOR_INFORMATION, PDWORD); 1371 1372 DWORD CountSetBits(ULONG_PTR bitMask) 1373 { 1374 DWORD LSHIFT = sizeof(ULONG_PTR)*8 - 1; 1375 DWORD bitSetCount = 0; 1376 ULONG_PTR bitTest = (ULONG_PTR)1 << LSHIFT; 1377 DWORD i; 1378 1379 for (i = 0; i <= LSHIFT; ++i) 1380 { 1381 bitSetCount += ((bitMask & bitTest)?1:0); 1382 bitTest/=2; 1383 } 1384 1385 return bitSetCount; 1386 } 1387 1388 int UTIL_countCores(int logical) 1389 { 1390 static int numCores = 0; 1391 if (numCores != 0) return numCores; 1392 1393 { LPFN_GLPI glpi; 1394 BOOL done = FALSE; 1395 PSYSTEM_LOGICAL_PROCESSOR_INFORMATION buffer = NULL; 1396 PSYSTEM_LOGICAL_PROCESSOR_INFORMATION ptr = NULL; 1397 DWORD returnLength = 0; 1398 size_t byteOffset = 0; 1399 1400 #if defined(_MSC_VER) 1401 /* Visual Studio does not like the following cast */ 1402 # pragma warning( disable : 4054 ) /* conversion from function ptr to data ptr */ 1403 # pragma warning( disable : 4055 ) /* conversion from data ptr to function ptr */ 1404 #endif 1405 glpi = (LPFN_GLPI)(void*)GetProcAddress(GetModuleHandle(TEXT("kernel32")), 1406 "GetLogicalProcessorInformation"); 1407 1408 if (glpi == NULL) { 1409 goto failed; 1410 } 1411 1412 while(!done) { 1413 DWORD rc = glpi(buffer, &returnLength); 1414 if (FALSE == rc) { 1415 if (GetLastError() == ERROR_INSUFFICIENT_BUFFER) { 1416 if (buffer) 1417 free(buffer); 1418 buffer = (PSYSTEM_LOGICAL_PROCESSOR_INFORMATION)malloc(returnLength); 1419 1420 if (buffer == NULL) { 1421 perror("zstd"); 1422 exit(1); 1423 } 1424 } else { 1425 /* some other error */ 1426 goto failed; 1427 } 1428 } else { 1429 done = TRUE; 1430 } } 1431 1432 ptr = buffer; 1433 1434 while (byteOffset + sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION) <= returnLength) { 1435 1436 if (ptr->Relationship == RelationProcessorCore) { 1437 if (logical) 1438 numCores += CountSetBits(ptr->ProcessorMask); 1439 else 1440 numCores++; 1441 } 1442 1443 ptr++; 1444 byteOffset += sizeof(SYSTEM_LOGICAL_PROCESSOR_INFORMATION); 1445 } 1446 1447 free(buffer); 1448 1449 return numCores; 1450 } 1451 1452 failed: 1453 /* try to fall back on GetSystemInfo */ 1454 { SYSTEM_INFO sysinfo; 1455 GetSystemInfo(&sysinfo); 1456 numCores = sysinfo.dwNumberOfProcessors; 1457 if (numCores == 0) numCores = 1; /* just in case */ 1458 } 1459 return numCores; 1460 } 1461 1462 #elif defined(__APPLE__) 1463 1464 #include <sys/sysctl.h> 1465 1466 /* Use apple-provided syscall 1467 * see: man 3 sysctl */ 1468 int UTIL_countCores(int logical) 1469 { 1470 static S32 numCores = 0; /* apple specifies int32_t */ 1471 if (numCores != 0) return numCores; 1472 1473 { size_t size = sizeof(S32); 1474 int const ret = sysctlbyname(logical ? "hw.logicalcpu" : "hw.physicalcpu", &numCores, &size, NULL, 0); 1475 if (ret != 0) { 1476 if (errno == ENOENT) { 1477 /* entry not present, fall back on 1 */ 1478 numCores = 1; 1479 } else { 1480 perror("zstd: can't get number of cpus"); 1481 exit(1); 1482 } 1483 } 1484 1485 return numCores; 1486 } 1487 } 1488 1489 #elif defined(__linux__) 1490 1491 /* parse /proc/cpuinfo 1492 * siblings / cpu cores should give hyperthreading ratio 1493 * otherwise fall back on sysconf */ 1494 int UTIL_countCores(int logical) 1495 { 1496 static int numCores = 0; 1497 1498 if (numCores != 0) return numCores; 1499 1500 numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); 1501 if (numCores == -1) { 1502 /* value not queryable, fall back on 1 */ 1503 return numCores = 1; 1504 } 1505 1506 /* try to determine if there's hyperthreading */ 1507 { FILE* const cpuinfo = fopen("/proc/cpuinfo", "r"); 1508 #define BUF_SIZE 80 1509 char buff[BUF_SIZE]; 1510 1511 int siblings = 0; 1512 int cpu_cores = 0; 1513 int ratio = 1; 1514 1515 if (cpuinfo == NULL) { 1516 /* fall back on the sysconf value */ 1517 return numCores; 1518 } 1519 1520 /* assume the cpu cores/siblings values will be constant across all 1521 * present processors */ 1522 while (!feof(cpuinfo)) { 1523 if (fgets(buff, BUF_SIZE, cpuinfo) != NULL) { 1524 if (strncmp(buff, "siblings", 8) == 0) { 1525 const char* const sep = strchr(buff, ':'); 1526 if (sep == NULL || *sep == '\0') { 1527 /* formatting was broken? */ 1528 goto failed; 1529 } 1530 1531 siblings = atoi(sep + 1); 1532 } 1533 if (strncmp(buff, "cpu cores", 9) == 0) { 1534 const char* const sep = strchr(buff, ':'); 1535 if (sep == NULL || *sep == '\0') { 1536 /* formatting was broken? */ 1537 goto failed; 1538 } 1539 1540 cpu_cores = atoi(sep + 1); 1541 } 1542 } else if (ferror(cpuinfo)) { 1543 /* fall back on the sysconf value */ 1544 goto failed; 1545 } } 1546 if (siblings && cpu_cores && siblings > cpu_cores) { 1547 ratio = siblings / cpu_cores; 1548 } 1549 1550 if (ratio && numCores > ratio && !logical) { 1551 numCores = numCores / ratio; 1552 } 1553 1554 failed: 1555 fclose(cpuinfo); 1556 return numCores; 1557 } 1558 } 1559 1560 #elif defined(__FreeBSD__) 1561 1562 #include <sys/sysctl.h> 1563 1564 /* Use physical core sysctl when available 1565 * see: man 4 smp, man 3 sysctl */ 1566 int UTIL_countCores(int logical) 1567 { 1568 static int numCores = 0; /* freebsd sysctl is native int sized */ 1569 #if __FreeBSD_version >= 1300008 1570 static int perCore = 1; 1571 #endif 1572 if (numCores != 0) return numCores; 1573 1574 #if __FreeBSD_version >= 1300008 1575 { size_t size = sizeof(numCores); 1576 int ret = sysctlbyname("kern.smp.cores", &numCores, &size, NULL, 0); 1577 if (ret == 0) { 1578 if (logical) { 1579 ret = sysctlbyname("kern.smp.threads_per_core", &perCore, &size, NULL, 0); 1580 /* default to physical cores if logical cannot be read */ 1581 if (ret == 0) 1582 numCores *= perCore; 1583 } 1584 1585 return numCores; 1586 } 1587 if (errno != ENOENT) { 1588 perror("zstd: can't get number of cpus"); 1589 exit(1); 1590 } 1591 /* sysctl not present, fall through to older sysconf method */ 1592 } 1593 #else 1594 /* suppress unused parameter warning */ 1595 (void) logical; 1596 #endif 1597 1598 numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); 1599 if (numCores == -1) { 1600 /* value not queryable, fall back on 1 */ 1601 numCores = 1; 1602 } 1603 return numCores; 1604 } 1605 1606 #elif defined(__NetBSD__) || defined(__OpenBSD__) || defined(__DragonFly__) || defined(__CYGWIN__) 1607 1608 /* Use POSIX sysconf 1609 * see: man 3 sysconf */ 1610 int UTIL_countCores(int logical) 1611 { 1612 static int numCores = 0; 1613 1614 /* suppress unused parameter warning */ 1615 (void)logical; 1616 1617 if (numCores != 0) return numCores; 1618 1619 numCores = (int)sysconf(_SC_NPROCESSORS_ONLN); 1620 if (numCores == -1) { 1621 /* value not queryable, fall back on 1 */ 1622 return numCores = 1; 1623 } 1624 return numCores; 1625 } 1626 1627 #else 1628 1629 int UTIL_countCores(int logical) 1630 { 1631 /* suppress unused parameter warning */ 1632 (void)logical; 1633 1634 /* assume 1 */ 1635 return 1; 1636 } 1637 1638 #endif 1639 1640 int UTIL_countPhysicalCores(void) 1641 { 1642 return UTIL_countCores(0); 1643 } 1644 1645 int UTIL_countLogicalCores(void) 1646 { 1647 return UTIL_countCores(1); 1648 } 1649 1650 #if defined (__cplusplus) 1651 } 1652 #endif 1653