1from __future__ import print_function 2 3import errno 4import itertools 5import math 6import numbers 7import os 8import platform 9import re 10import signal 11import subprocess 12import sys 13import threading 14 15 16def is_string(value): 17 try: 18 # Python 2 and Python 3 are different here. 19 return isinstance(value, basestring) 20 except NameError: 21 return isinstance(value, str) 22 23 24def pythonize_bool(value): 25 if value is None: 26 return False 27 if type(value) is bool: 28 return value 29 if isinstance(value, numbers.Number): 30 return value != 0 31 if is_string(value): 32 if value.lower() in ("1", "true", "on", "yes"): 33 return True 34 if value.lower() in ("", "0", "false", "off", "no"): 35 return False 36 raise ValueError('"{}" is not a valid boolean'.format(value)) 37 38 39def make_word_regex(word): 40 return r"\b" + word + r"\b" 41 42 43def to_bytes(s): 44 """Return the parameter as type 'bytes', possibly encoding it. 45 46 In Python2, the 'bytes' type is the same as 'str'. In Python3, they 47 are distinct. 48 49 """ 50 if isinstance(s, bytes): 51 # In Python2, this branch is taken for both 'str' and 'bytes'. 52 # In Python3, this branch is taken only for 'bytes'. 53 return s 54 # In Python2, 's' is a 'unicode' object. 55 # In Python3, 's' is a 'str' object. 56 # Encode to UTF-8 to get 'bytes' data. 57 return s.encode("utf-8") 58 59 60def to_string(b): 61 """Return the parameter as type 'str', possibly encoding it. 62 63 In Python2, the 'str' type is the same as 'bytes'. In Python3, the 64 'str' type is (essentially) Python2's 'unicode' type, and 'bytes' is 65 distinct. 66 67 """ 68 if isinstance(b, str): 69 # In Python2, this branch is taken for types 'str' and 'bytes'. 70 # In Python3, this branch is taken only for 'str'. 71 return b 72 if isinstance(b, bytes): 73 # In Python2, this branch is never taken ('bytes' is handled as 'str'). 74 # In Python3, this is true only for 'bytes'. 75 try: 76 return b.decode("utf-8") 77 except UnicodeDecodeError: 78 # If the value is not valid Unicode, return the default 79 # repr-line encoding. 80 return str(b) 81 82 # By this point, here's what we *don't* have: 83 # 84 # - In Python2: 85 # - 'str' or 'bytes' (1st branch above) 86 # - In Python3: 87 # - 'str' (1st branch above) 88 # - 'bytes' (2nd branch above) 89 # 90 # The last type we might expect is the Python2 'unicode' type. There is no 91 # 'unicode' type in Python3 (all the Python3 cases were already handled). In 92 # order to get a 'str' object, we need to encode the 'unicode' object. 93 try: 94 return b.encode("utf-8") 95 except AttributeError: 96 raise TypeError("not sure how to convert %s to %s" % (type(b), str)) 97 98 99def to_unicode(s): 100 """Return the parameter as type which supports unicode, possibly decoding 101 it. 102 103 In Python2, this is the unicode type. In Python3 it's the str type. 104 105 """ 106 if isinstance(s, bytes): 107 # In Python2, this branch is taken for both 'str' and 'bytes'. 108 # In Python3, this branch is taken only for 'bytes'. 109 return s.decode("utf-8") 110 return s 111 112 113def usable_core_count(): 114 """Return the number of cores the current process can use, if supported. 115 Otherwise, return the total number of cores (like `os.cpu_count()`). 116 Default to 1 if undetermined. 117 118 """ 119 try: 120 n = len(os.sched_getaffinity(0)) 121 except AttributeError: 122 n = os.cpu_count() or 1 123 124 # On Windows with more than 60 processes, multiprocessing's call to 125 # _winapi.WaitForMultipleObjects() prints an error and lit hangs. 126 if platform.system() == "Windows": 127 return min(n, 60) 128 129 return n 130 131def abs_path_preserve_drive(path): 132 """Return the absolute path without resolving drive mappings on Windows. 133 134 """ 135 if platform.system() == "Windows": 136 # Windows has limitations on path length (MAX_PATH) that 137 # can be worked around using substitute drives, which map 138 # a drive letter to a longer path on another drive. 139 # Since Python 3.8, os.path.realpath resolves sustitute drives, 140 # so we should not use it. In Python 3.7, os.path.realpath 141 # was implemented as os.path.abspath. 142 return os.path.abspath(path) 143 else: 144 # On UNIX, the current directory always has symbolic links resolved, 145 # so any program accepting relative paths cannot preserve symbolic 146 # links in paths and we should always use os.path.realpath. 147 return os.path.realpath(path) 148 149def mkdir(path): 150 try: 151 if platform.system() == "Windows": 152 from ctypes import windll 153 from ctypes import GetLastError, WinError 154 155 path = os.path.abspath(path) 156 # Make sure that the path uses backslashes here, in case 157 # python would have happened to use forward slashes, as the 158 # NT path format only supports backslashes. 159 path = path.replace("/", "\\") 160 NTPath = to_unicode(r"\\?\%s" % path) 161 if not windll.kernel32.CreateDirectoryW(NTPath, None): 162 raise WinError(GetLastError()) 163 else: 164 os.mkdir(path) 165 except OSError: 166 e = sys.exc_info()[1] 167 # ignore EEXIST, which may occur during a race condition 168 if e.errno != errno.EEXIST: 169 raise 170 171 172def mkdir_p(path): 173 """mkdir_p(path) - Make the "path" directory, if it does not exist; this 174 will also make directories for any missing parent directories.""" 175 if not path or os.path.exists(path): 176 return 177 178 parent = os.path.dirname(path) 179 if parent != path: 180 mkdir_p(parent) 181 182 mkdir(path) 183 184 185def listdir_files(dirname, suffixes=None, exclude_filenames=None): 186 """Yields files in a directory. 187 188 Filenames that are not excluded by rules below are yielded one at a time, as 189 basenames (i.e., without dirname). 190 191 Files starting with '.' are always skipped. 192 193 If 'suffixes' is not None, then only filenames ending with one of its 194 members will be yielded. These can be extensions, like '.exe', or strings, 195 like 'Test'. (It is a lexicographic check; so an empty sequence will yield 196 nothing, but a single empty string will yield all filenames.) 197 198 If 'exclude_filenames' is not None, then none of the file basenames in it 199 will be yielded. 200 201 If specified, the containers for 'suffixes' and 'exclude_filenames' must 202 support membership checking for strs. 203 204 Args: 205 dirname: a directory path. 206 suffixes: (optional) a sequence of strings (set, list, etc.). 207 exclude_filenames: (optional) a sequence of strings. 208 209 Yields: 210 Filenames as returned by os.listdir (generally, str). 211 212 """ 213 if exclude_filenames is None: 214 exclude_filenames = set() 215 if suffixes is None: 216 suffixes = {""} 217 for filename in os.listdir(dirname): 218 if ( 219 os.path.isdir(os.path.join(dirname, filename)) 220 or filename.startswith(".") 221 or filename in exclude_filenames 222 or not any(filename.endswith(sfx) for sfx in suffixes) 223 ): 224 continue 225 yield filename 226 227 228def which(command, paths=None): 229 """which(command, [paths]) - Look up the given command in the paths string 230 (or the PATH environment variable, if unspecified).""" 231 232 if paths is None: 233 paths = os.environ.get("PATH", "") 234 235 # Check for absolute match first. 236 if os.path.isabs(command) and os.path.isfile(command): 237 return os.path.normcase(os.path.normpath(command)) 238 239 # Would be nice if Python had a lib function for this. 240 if not paths: 241 paths = os.defpath 242 243 # Get suffixes to search. 244 # On Cygwin, 'PATHEXT' may exist but it should not be used. 245 if os.pathsep == ";": 246 pathext = os.environ.get("PATHEXT", "").split(";") 247 else: 248 pathext = [""] 249 250 # Search the paths... 251 for path in paths.split(os.pathsep): 252 for ext in pathext: 253 p = os.path.join(path, command + ext) 254 if os.path.exists(p) and not os.path.isdir(p): 255 return os.path.normcase(os.path.abspath(p)) 256 257 return None 258 259 260def checkToolsPath(dir, tools): 261 for tool in tools: 262 if not os.path.exists(os.path.join(dir, tool)): 263 return False 264 return True 265 266 267def whichTools(tools, paths): 268 for path in paths.split(os.pathsep): 269 if checkToolsPath(path, tools): 270 return path 271 return None 272 273 274def printHistogram(items, title="Items"): 275 items.sort(key=lambda item: item[1]) 276 277 maxValue = max([v for _, v in items]) 278 279 # Select first "nice" bar height that produces more than 10 bars. 280 power = int(math.ceil(math.log(maxValue, 10))) 281 for inc in itertools.cycle((5, 2, 2.5, 1)): 282 barH = inc * 10**power 283 N = int(math.ceil(maxValue / barH)) 284 if N > 10: 285 break 286 elif inc == 1: 287 power -= 1 288 289 histo = [set() for i in range(N)] 290 for name, v in items: 291 bin = min(int(N * v / maxValue), N - 1) 292 histo[bin].add(name) 293 294 barW = 40 295 hr = "-" * (barW + 34) 296 print("Slowest %s:" % title) 297 print(hr) 298 for name, value in reversed(items[-20:]): 299 print("%.2fs: %s" % (value, name)) 300 print("\n%s Times:" % title) 301 print(hr) 302 pDigits = int(math.ceil(math.log(maxValue, 10))) 303 pfDigits = max(0, 3 - pDigits) 304 if pfDigits: 305 pDigits += pfDigits + 1 306 cDigits = int(math.ceil(math.log(len(items), 10))) 307 print( 308 "[%s] :: [%s] :: [%s]" 309 % ( 310 "Range".center((pDigits + 1) * 2 + 3), 311 "Percentage".center(barW), 312 "Count".center(cDigits * 2 + 1), 313 ) 314 ) 315 print(hr) 316 for i, row in reversed(list(enumerate(histo))): 317 pct = float(len(row)) / len(items) 318 w = int(barW * pct) 319 print( 320 "[%*.*fs,%*.*fs) :: [%s%s] :: [%*d/%*d]" 321 % ( 322 pDigits, 323 pfDigits, 324 i * barH, 325 pDigits, 326 pfDigits, 327 (i + 1) * barH, 328 "*" * w, 329 " " * (barW - w), 330 cDigits, 331 len(row), 332 cDigits, 333 len(items), 334 ) 335 ) 336 print(hr) 337 338 339class ExecuteCommandTimeoutException(Exception): 340 def __init__(self, msg, out, err, exitCode): 341 assert isinstance(msg, str) 342 assert isinstance(out, str) 343 assert isinstance(err, str) 344 assert isinstance(exitCode, int) 345 self.msg = msg 346 self.out = out 347 self.err = err 348 self.exitCode = exitCode 349 350 351# Close extra file handles on UNIX (on Windows this cannot be done while 352# also redirecting input). 353kUseCloseFDs = not (platform.system() == "Windows") 354 355 356def executeCommand( 357 command, cwd=None, env=None, input=None, timeout=0, redirect_stderr=False 358): 359 """Execute command ``command`` (list of arguments or string) with. 360 361 * working directory ``cwd`` (str), use None to use the current 362 working directory 363 * environment ``env`` (dict), use None for none 364 * Input to the command ``input`` (str), use string to pass 365 no input. 366 * Max execution time ``timeout`` (int) seconds. Use 0 for no timeout. 367 * ``redirect_stderr`` (bool), use True if redirect stderr to stdout 368 369 Returns a tuple (out, err, exitCode) where 370 * ``out`` (str) is the standard output of running the command 371 * ``err`` (str) is the standard error of running the command 372 * ``exitCode`` (int) is the exitCode of running the command 373 374 If the timeout is hit an ``ExecuteCommandTimeoutException`` 375 is raised. 376 377 """ 378 if input is not None: 379 input = to_bytes(input) 380 err_out = subprocess.STDOUT if redirect_stderr else subprocess.PIPE 381 p = subprocess.Popen( 382 command, 383 cwd=cwd, 384 stdin=subprocess.PIPE, 385 stdout=subprocess.PIPE, 386 stderr=err_out, 387 env=env, 388 close_fds=kUseCloseFDs, 389 ) 390 timerObject = None 391 # FIXME: Because of the way nested function scopes work in Python 2.x we 392 # need to use a reference to a mutable object rather than a plain 393 # bool. In Python 3 we could use the "nonlocal" keyword but we need 394 # to support Python 2 as well. 395 hitTimeOut = [False] 396 try: 397 if timeout > 0: 398 399 def killProcess(): 400 # We may be invoking a shell so we need to kill the 401 # process and all its children. 402 hitTimeOut[0] = True 403 killProcessAndChildren(p.pid) 404 405 timerObject = threading.Timer(timeout, killProcess) 406 timerObject.start() 407 408 out, err = p.communicate(input=input) 409 exitCode = p.wait() 410 finally: 411 if timerObject is not None: 412 timerObject.cancel() 413 414 # Ensure the resulting output is always of string type. 415 out = to_string(out) 416 err = "" if redirect_stderr else to_string(err) 417 418 if hitTimeOut[0]: 419 raise ExecuteCommandTimeoutException( 420 msg="Reached timeout of {} seconds".format(timeout), 421 out=out, 422 err=err, 423 exitCode=exitCode, 424 ) 425 426 # Detect Ctrl-C in subprocess. 427 if exitCode == -signal.SIGINT: 428 raise KeyboardInterrupt 429 430 return out, err, exitCode 431 432 433def isAIXTriple(target_triple): 434 """Whether the given target triple is for AIX, 435 e.g. powerpc64-ibm-aix 436 """ 437 return "aix" in target_triple 438 439 440def addAIXVersion(target_triple): 441 """Add the AIX version to the given target triple, 442 e.g. powerpc64-ibm-aix7.2.5.6 443 """ 444 os_cmd = "oslevel -s | awk -F\'-\' \'{printf \"%.1f.%d.%d\", $1/1000, $2, $3}\'" 445 os_version = subprocess.run(os_cmd, capture_output=True, shell=True).stdout.decode() 446 return re.sub("aix", "aix" + os_version, target_triple) 447 448 449def isMacOSTriple(target_triple): 450 """Whether the given target triple is for macOS, 451 e.g. x86_64-apple-darwin, arm64-apple-macos 452 """ 453 return "darwin" in target_triple or "macos" in target_triple 454 455 456def usePlatformSdkOnDarwin(config, lit_config): 457 # On Darwin, support relocatable SDKs by providing Clang with a 458 # default system root path. 459 if isMacOSTriple(config.target_triple): 460 try: 461 cmd = subprocess.Popen( 462 ["xcrun", "--show-sdk-path", "--sdk", "macosx"], 463 stdout=subprocess.PIPE, 464 stderr=subprocess.PIPE, 465 ) 466 out, err = cmd.communicate() 467 out = out.strip() 468 res = cmd.wait() 469 except OSError: 470 res = -1 471 if res == 0 and out: 472 sdk_path = out.decode() 473 lit_config.note("using SDKROOT: %r" % sdk_path) 474 config.environment["SDKROOT"] = sdk_path 475 476 477def findPlatformSdkVersionOnMacOS(config, lit_config): 478 if isMacOSTriple(config.target_triple): 479 try: 480 cmd = subprocess.Popen( 481 ["xcrun", "--show-sdk-version", "--sdk", "macosx"], 482 stdout=subprocess.PIPE, 483 stderr=subprocess.PIPE, 484 ) 485 out, err = cmd.communicate() 486 out = out.strip() 487 res = cmd.wait() 488 except OSError: 489 res = -1 490 if res == 0 and out: 491 return out.decode() 492 return None 493 494 495def killProcessAndChildrenIsSupported(): 496 """ 497 Returns a tuple (<supported> , <error message>) 498 where 499 `<supported>` is True if `killProcessAndChildren()` is supported on 500 the current host, returns False otherwise. 501 `<error message>` is an empty string if `<supported>` is True, 502 otherwise is contains a string describing why the function is 503 not supported. 504 """ 505 if platform.system() == "AIX" or platform.system() == "OS/390": 506 return (True, "") 507 try: 508 import psutil # noqa: F401 509 510 return (True, "") 511 except ImportError: 512 return ( 513 False, 514 "Requires the Python psutil module but it could" 515 " not be found. Try installing it via pip or via" 516 " your operating system's package manager.", 517 ) 518 519 520def killProcessAndChildren(pid): 521 """This function kills a process with ``pid`` and all its running children 522 (recursively). It is currently implemented using the psutil module on some 523 platforms which provides a simple platform neutral implementation. 524 525 TODO: Reimplement this without using psutil on all platforms so we can 526 remove our dependency on it. 527 528 """ 529 if platform.system() == "AIX": 530 subprocess.call("kill -kill $(ps -o pid= -L{})".format(pid), shell=True) 531 elif platform.system() == "OS/390": 532 # FIXME: Only the process is killed. 533 subprocess.call("kill -KILL $(ps -s {} -o pid=)".format(pid), shell=True) 534 else: 535 import psutil 536 537 try: 538 psutilProc = psutil.Process(pid) 539 # Handle the different psutil API versions 540 try: 541 # psutil >= 2.x 542 children_iterator = psutilProc.children(recursive=True) 543 except AttributeError: 544 # psutil 1.x 545 children_iterator = psutilProc.get_children(recursive=True) 546 for child in children_iterator: 547 try: 548 child.kill() 549 except psutil.NoSuchProcess: 550 pass 551 psutilProc.kill() 552 except psutil.NoSuchProcess: 553 pass 554