1from __future__ import absolute_import 2import os, signal, subprocess, sys 3import re 4import platform 5import tempfile 6 7import lit.ShUtil as ShUtil 8import lit.Test as Test 9import lit.util 10from lit.util import to_bytes, to_string 11 12class InternalShellError(Exception): 13 def __init__(self, command, message): 14 self.command = command 15 self.message = message 16 17kIsWindows = platform.system() == 'Windows' 18 19# Don't use close_fds on Windows. 20kUseCloseFDs = not kIsWindows 21 22# Use temporary files to replace /dev/null on Windows. 23kAvoidDevNull = kIsWindows 24 25def executeShCmd(cmd, cfg, cwd, results): 26 if isinstance(cmd, ShUtil.Seq): 27 if cmd.op == ';': 28 res = executeShCmd(cmd.lhs, cfg, cwd, results) 29 return executeShCmd(cmd.rhs, cfg, cwd, results) 30 31 if cmd.op == '&': 32 raise InternalShellError(cmd,"unsupported shell operator: '&'") 33 34 if cmd.op == '||': 35 res = executeShCmd(cmd.lhs, cfg, cwd, results) 36 if res != 0: 37 res = executeShCmd(cmd.rhs, cfg, cwd, results) 38 return res 39 40 if cmd.op == '&&': 41 res = executeShCmd(cmd.lhs, cfg, cwd, results) 42 if res is None: 43 return res 44 45 if res == 0: 46 res = executeShCmd(cmd.rhs, cfg, cwd, results) 47 return res 48 49 raise ValueError('Unknown shell command: %r' % cmd.op) 50 51 assert isinstance(cmd, ShUtil.Pipeline) 52 procs = [] 53 input = subprocess.PIPE 54 stderrTempFiles = [] 55 opened_files = [] 56 named_temp_files = [] 57 # To avoid deadlock, we use a single stderr stream for piped 58 # output. This is null until we have seen some output using 59 # stderr. 60 for i,j in enumerate(cmd.commands): 61 # Apply the redirections, we use (N,) as a sentinel to indicate stdin, 62 # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or 63 # from a file are represented with a list [file, mode, file-object] 64 # where file-object is initially None. 65 redirects = [(0,), (1,), (2,)] 66 for r in j.redirects: 67 if r[0] == ('>',2): 68 redirects[2] = [r[1], 'w', None] 69 elif r[0] == ('>>',2): 70 redirects[2] = [r[1], 'a', None] 71 elif r[0] == ('>&',2) and r[1] in '012': 72 redirects[2] = redirects[int(r[1])] 73 elif r[0] == ('>&',) or r[0] == ('&>',): 74 redirects[1] = redirects[2] = [r[1], 'w', None] 75 elif r[0] == ('>',): 76 redirects[1] = [r[1], 'w', None] 77 elif r[0] == ('>>',): 78 redirects[1] = [r[1], 'a', None] 79 elif r[0] == ('<',): 80 redirects[0] = [r[1], 'r', None] 81 else: 82 raise InternalShellError(j,"Unsupported redirect: %r" % (r,)) 83 84 # Map from the final redirections to something subprocess can handle. 85 final_redirects = [] 86 for index,r in enumerate(redirects): 87 if r == (0,): 88 result = input 89 elif r == (1,): 90 if index == 0: 91 raise InternalShellError(j,"Unsupported redirect for stdin") 92 elif index == 1: 93 result = subprocess.PIPE 94 else: 95 result = subprocess.STDOUT 96 elif r == (2,): 97 if index != 2: 98 raise InternalShellError(j,"Unsupported redirect on stdout") 99 result = subprocess.PIPE 100 else: 101 if r[2] is None: 102 if kAvoidDevNull and r[0] == '/dev/null': 103 r[2] = tempfile.TemporaryFile(mode=r[1]) 104 else: 105 r[2] = open(r[0], r[1]) 106 # Workaround a Win32 and/or subprocess bug when appending. 107 # 108 # FIXME: Actually, this is probably an instance of PR6753. 109 if r[1] == 'a': 110 r[2].seek(0, 2) 111 opened_files.append(r[2]) 112 result = r[2] 113 final_redirects.append(result) 114 115 stdin, stdout, stderr = final_redirects 116 117 # If stderr wants to come from stdout, but stdout isn't a pipe, then put 118 # stderr on a pipe and treat it as stdout. 119 if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE): 120 stderr = subprocess.PIPE 121 stderrIsStdout = True 122 else: 123 stderrIsStdout = False 124 125 # Don't allow stderr on a PIPE except for the last 126 # process, this could deadlock. 127 # 128 # FIXME: This is slow, but so is deadlock. 129 if stderr == subprocess.PIPE and j != cmd.commands[-1]: 130 stderr = tempfile.TemporaryFile(mode='w+b') 131 stderrTempFiles.append((i, stderr)) 132 133 # Resolve the executable path ourselves. 134 args = list(j.args) 135 executable = lit.util.which(args[0], cfg.environment['PATH']) 136 if not executable: 137 raise InternalShellError(j, '%r: command not found' % j.args[0]) 138 139 # Replace uses of /dev/null with temporary files. 140 if kAvoidDevNull: 141 for i,arg in enumerate(args): 142 if arg == "/dev/null": 143 f = tempfile.NamedTemporaryFile(delete=False) 144 f.close() 145 named_temp_files.append(f.name) 146 args[i] = f.name 147 148 try: 149 procs.append(subprocess.Popen(args, cwd=cwd, 150 executable = executable, 151 stdin = stdin, 152 stdout = stdout, 153 stderr = stderr, 154 env = cfg.environment, 155 close_fds = kUseCloseFDs)) 156 except OSError as e: 157 raise InternalShellError(j, 'Could not create process due to {}'.format(e)) 158 159 # Immediately close stdin for any process taking stdin from us. 160 if stdin == subprocess.PIPE: 161 procs[-1].stdin.close() 162 procs[-1].stdin = None 163 164 # Update the current stdin source. 165 if stdout == subprocess.PIPE: 166 input = procs[-1].stdout 167 elif stderrIsStdout: 168 input = procs[-1].stderr 169 else: 170 input = subprocess.PIPE 171 172 # Explicitly close any redirected files. We need to do this now because we 173 # need to release any handles we may have on the temporary files (important 174 # on Win32, for example). Since we have already spawned the subprocess, our 175 # handles have already been transferred so we do not need them anymore. 176 for f in opened_files: 177 f.close() 178 179 # FIXME: There is probably still deadlock potential here. Yawn. 180 procData = [None] * len(procs) 181 procData[-1] = procs[-1].communicate() 182 183 for i in range(len(procs) - 1): 184 if procs[i].stdout is not None: 185 out = procs[i].stdout.read() 186 else: 187 out = '' 188 if procs[i].stderr is not None: 189 err = procs[i].stderr.read() 190 else: 191 err = '' 192 procData[i] = (out,err) 193 194 # Read stderr out of the temp files. 195 for i,f in stderrTempFiles: 196 f.seek(0, 0) 197 procData[i] = (procData[i][0], f.read()) 198 199 def to_string(bytes): 200 if isinstance(bytes, str): 201 return bytes 202 return bytes.encode('utf-8') 203 204 exitCode = None 205 for i,(out,err) in enumerate(procData): 206 res = procs[i].wait() 207 # Detect Ctrl-C in subprocess. 208 if res == -signal.SIGINT: 209 raise KeyboardInterrupt 210 211 # Ensure the resulting output is always of string type. 212 try: 213 out = to_string(out.decode('utf-8')) 214 except: 215 out = str(out) 216 try: 217 err = to_string(err.decode('utf-8')) 218 except: 219 err = str(err) 220 221 results.append((cmd.commands[i], out, err, res)) 222 if cmd.pipe_err: 223 # Python treats the exit code as a signed char. 224 if exitCode is None: 225 exitCode = res 226 elif res < 0: 227 exitCode = min(exitCode, res) 228 else: 229 exitCode = max(exitCode, res) 230 else: 231 exitCode = res 232 233 # Remove any named temporary files we created. 234 for f in named_temp_files: 235 try: 236 os.remove(f) 237 except OSError: 238 pass 239 240 if cmd.negate: 241 exitCode = not exitCode 242 243 return exitCode 244 245def executeScriptInternal(test, litConfig, tmpBase, commands, cwd): 246 cmds = [] 247 for ln in commands: 248 try: 249 cmds.append(ShUtil.ShParser(ln, litConfig.isWindows, 250 test.config.pipefail).parse()) 251 except: 252 return lit.Test.Result(Test.FAIL, "shell parser error on: %r" % ln) 253 254 cmd = cmds[0] 255 for c in cmds[1:]: 256 cmd = ShUtil.Seq(cmd, '&&', c) 257 258 results = [] 259 try: 260 exitCode = executeShCmd(cmd, test.config, cwd, results) 261 except InternalShellError: 262 e = sys.exc_info()[1] 263 exitCode = 127 264 results.append((e.command, '', e.message, exitCode)) 265 266 out = err = '' 267 for i,(cmd, cmd_out,cmd_err,res) in enumerate(results): 268 out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args)) 269 out += 'Command %d Result: %r\n' % (i, res) 270 out += 'Command %d Output:\n%s\n\n' % (i, cmd_out) 271 out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err) 272 273 return out, err, exitCode 274 275def executeScript(test, litConfig, tmpBase, commands, cwd): 276 bashPath = litConfig.getBashPath(); 277 isWin32CMDEXE = (litConfig.isWindows and not bashPath) 278 script = tmpBase + '.script' 279 if isWin32CMDEXE: 280 script += '.bat' 281 282 # Write script file 283 mode = 'w' 284 if litConfig.isWindows and not isWin32CMDEXE: 285 mode += 'b' # Avoid CRLFs when writing bash scripts. 286 f = open(script, mode) 287 if isWin32CMDEXE: 288 f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands)) 289 else: 290 if test.config.pipefail: 291 f.write('set -o pipefail;') 292 f.write('{ ' + '; } &&\n{ '.join(commands) + '; }') 293 f.write('\n') 294 f.close() 295 296 if isWin32CMDEXE: 297 command = ['cmd','/c', script] 298 else: 299 if bashPath: 300 command = [bashPath, script] 301 else: 302 command = ['/bin/sh', script] 303 if litConfig.useValgrind: 304 # FIXME: Running valgrind on sh is overkill. We probably could just 305 # run on clang with no real loss. 306 command = litConfig.valgrindArgs + command 307 308 return lit.util.executeCommand(command, cwd=cwd, 309 env=test.config.environment) 310 311def parseIntegratedTestScriptCommands(source_path): 312 """ 313 parseIntegratedTestScriptCommands(source_path) -> commands 314 315 Parse the commands in an integrated test script file into a list of 316 (line_number, command_type, line). 317 """ 318 319 # This code is carefully written to be dual compatible with Python 2.5+ and 320 # Python 3 without requiring input files to always have valid codings. The 321 # trick we use is to open the file in binary mode and use the regular 322 # expression library to find the commands, with it scanning strings in 323 # Python2 and bytes in Python3. 324 # 325 # Once we find a match, we do require each script line to be decodable to 326 # UTF-8, so we convert the outputs to UTF-8 before returning. This way the 327 # remaining code can work with "strings" agnostic of the executing Python 328 # version. 329 330 keywords = ['RUN:', 'XFAIL:', 'REQUIRES:', 'UNSUPPORTED:', 'END.'] 331 keywords_re = re.compile( 332 to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),))) 333 334 f = open(source_path, 'rb') 335 try: 336 # Read the entire file contents. 337 data = f.read() 338 339 # Ensure the data ends with a newline. 340 if not data.endswith(to_bytes('\n')): 341 data = data + to_bytes('\n') 342 343 # Iterate over the matches. 344 line_number = 1 345 last_match_position = 0 346 for match in keywords_re.finditer(data): 347 # Compute the updated line number by counting the intervening 348 # newlines. 349 match_position = match.start() 350 line_number += data.count(to_bytes('\n'), last_match_position, 351 match_position) 352 last_match_position = match_position 353 354 # Convert the keyword and line to UTF-8 strings and yield the 355 # command. Note that we take care to return regular strings in 356 # Python 2, to avoid other code having to differentiate between the 357 # str and unicode types. 358 keyword,ln = match.groups() 359 yield (line_number, to_string(keyword[:-1].decode('utf-8')), 360 to_string(ln.decode('utf-8'))) 361 finally: 362 f.close() 363 364 365def parseIntegratedTestScript(test, normalize_slashes=False, 366 extra_substitutions=[], require_script=True): 367 """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test 368 script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES' 369 and 'UNSUPPORTED' information. The RUN lines also will have variable 370 substitution performed. If 'require_script' is False an empty script may be 371 returned. This can be used for test formats where the actual script is 372 optional or ignored. 373 """ 374 375 # Get the temporary location, this is always relative to the test suite 376 # root, not test source root. 377 # 378 # FIXME: This should not be here? 379 sourcepath = test.getSourcePath() 380 sourcedir = os.path.dirname(sourcepath) 381 execpath = test.getExecPath() 382 execdir,execbase = os.path.split(execpath) 383 tmpDir = os.path.join(execdir, 'Output') 384 tmpBase = os.path.join(tmpDir, execbase) 385 386 # Normalize slashes, if requested. 387 if normalize_slashes: 388 sourcepath = sourcepath.replace('\\', '/') 389 sourcedir = sourcedir.replace('\\', '/') 390 tmpDir = tmpDir.replace('\\', '/') 391 tmpBase = tmpBase.replace('\\', '/') 392 393 # We use #_MARKER_# to hide %% while we do the other substitutions. 394 substitutions = list(extra_substitutions) 395 substitutions.extend([('%%', '#_MARKER_#')]) 396 substitutions.extend(test.config.substitutions) 397 substitutions.extend([('%s', sourcepath), 398 ('%S', sourcedir), 399 ('%p', sourcedir), 400 ('%{pathsep}', os.pathsep), 401 ('%t', tmpBase + '.tmp'), 402 ('%T', tmpDir), 403 ('#_MARKER_#', '%')]) 404 405 # "%/[STpst]" should be normalized. 406 substitutions.extend([ 407 ('%/s', sourcepath.replace('\\', '/')), 408 ('%/S', sourcedir.replace('\\', '/')), 409 ('%/p', sourcedir.replace('\\', '/')), 410 ('%/t', tmpBase.replace('\\', '/') + '.tmp'), 411 ('%/T', tmpDir.replace('\\', '/')), 412 ]) 413 414 # Collect the test lines from the script. 415 script = [] 416 requires = [] 417 unsupported = [] 418 for line_number, command_type, ln in \ 419 parseIntegratedTestScriptCommands(sourcepath): 420 if command_type == 'RUN': 421 # Trim trailing whitespace. 422 ln = ln.rstrip() 423 424 # Substitute line number expressions 425 ln = re.sub('%\(line\)', str(line_number), ln) 426 def replace_line_number(match): 427 if match.group(1) == '+': 428 return str(line_number + int(match.group(2))) 429 if match.group(1) == '-': 430 return str(line_number - int(match.group(2))) 431 ln = re.sub('%\(line *([\+-]) *(\d+)\)', replace_line_number, ln) 432 433 # Collapse lines with trailing '\\'. 434 if script and script[-1][-1] == '\\': 435 script[-1] = script[-1][:-1] + ln 436 else: 437 script.append(ln) 438 elif command_type == 'XFAIL': 439 test.xfails.extend([s.strip() for s in ln.split(',')]) 440 elif command_type == 'REQUIRES': 441 requires.extend([s.strip() for s in ln.split(',')]) 442 elif command_type == 'UNSUPPORTED': 443 unsupported.extend([s.strip() for s in ln.split(',')]) 444 elif command_type == 'END': 445 # END commands are only honored if the rest of the line is empty. 446 if not ln.strip(): 447 break 448 else: 449 raise ValueError("unknown script command type: %r" % ( 450 command_type,)) 451 452 # Apply substitutions to the script. Allow full regular 453 # expression syntax. Replace each matching occurrence of regular 454 # expression pattern a with substitution b in line ln. 455 def processLine(ln): 456 # Apply substitutions 457 for a,b in substitutions: 458 if kIsWindows: 459 b = b.replace("\\","\\\\") 460 ln = re.sub(a, b, ln) 461 462 # Strip the trailing newline and any extra whitespace. 463 return ln.strip() 464 script = [processLine(ln) 465 for ln in script] 466 467 # Verify the script contains a run line. 468 if require_script and not script: 469 return lit.Test.Result(Test.UNRESOLVED, "Test has no run line!") 470 471 # Check for unterminated run lines. 472 if script and script[-1][-1] == '\\': 473 return lit.Test.Result(Test.UNRESOLVED, 474 "Test has unterminated run lines (with '\\')") 475 476 # Check that we have the required features: 477 missing_required_features = [f for f in requires 478 if f not in test.config.available_features] 479 if missing_required_features: 480 msg = ', '.join(missing_required_features) 481 return lit.Test.Result(Test.UNSUPPORTED, 482 "Test requires the following features: %s" % msg) 483 unsupported_features = [f for f in unsupported 484 if f in test.config.available_features] 485 if unsupported_features: 486 msg = ', '.join(unsupported_features) 487 return lit.Test.Result(Test.UNSUPPORTED, 488 "Test is unsupported with the following features: %s" % msg) 489 490 return script,tmpBase,execdir 491 492def _runShTest(test, litConfig, useExternalSh, 493 script, tmpBase, execdir): 494 # Create the output directory if it does not already exist. 495 lit.util.mkdir_p(os.path.dirname(tmpBase)) 496 497 if useExternalSh: 498 res = executeScript(test, litConfig, tmpBase, script, execdir) 499 else: 500 res = executeScriptInternal(test, litConfig, tmpBase, script, execdir) 501 if isinstance(res, lit.Test.Result): 502 return res 503 504 out,err,exitCode = res 505 if exitCode == 0: 506 status = Test.PASS 507 else: 508 status = Test.FAIL 509 510 # Form the output log. 511 output = """Script:\n--\n%s\n--\nExit Code: %d\n\n""" % ( 512 '\n'.join(script), exitCode) 513 514 # Append the outputs, if present. 515 if out: 516 output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,) 517 if err: 518 output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,) 519 520 return lit.Test.Result(status, output) 521 522 523def executeShTest(test, litConfig, useExternalSh, 524 extra_substitutions=[]): 525 if test.config.unsupported: 526 return (Test.UNSUPPORTED, 'Test is unsupported') 527 528 res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions) 529 if isinstance(res, lit.Test.Result): 530 return res 531 if litConfig.noExecute: 532 return lit.Test.Result(Test.PASS) 533 534 script, tmpBase, execdir = res 535 return _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir) 536 537