xref: /minix3/external/bsd/llvm/dist/llvm/utils/lit/lit/TestRunner.py (revision 0a6a1f1d05b60e214de2f05a7310ddd1f0e590e7)
1from __future__ import absolute_import
2import os, signal, subprocess, sys
3import re
4import platform
5import tempfile
6
7import lit.ShUtil as ShUtil
8import lit.Test as Test
9import lit.util
10from lit.util import to_bytes, to_string
11
12class InternalShellError(Exception):
13    def __init__(self, command, message):
14        self.command = command
15        self.message = message
16
17kIsWindows = platform.system() == 'Windows'
18
19# Don't use close_fds on Windows.
20kUseCloseFDs = not kIsWindows
21
22# Use temporary files to replace /dev/null on Windows.
23kAvoidDevNull = kIsWindows
24
25def executeShCmd(cmd, cfg, cwd, results):
26    if isinstance(cmd, ShUtil.Seq):
27        if cmd.op == ';':
28            res = executeShCmd(cmd.lhs, cfg, cwd, results)
29            return executeShCmd(cmd.rhs, cfg, cwd, results)
30
31        if cmd.op == '&':
32            raise InternalShellError(cmd,"unsupported shell operator: '&'")
33
34        if cmd.op == '||':
35            res = executeShCmd(cmd.lhs, cfg, cwd, results)
36            if res != 0:
37                res = executeShCmd(cmd.rhs, cfg, cwd, results)
38            return res
39
40        if cmd.op == '&&':
41            res = executeShCmd(cmd.lhs, cfg, cwd, results)
42            if res is None:
43                return res
44
45            if res == 0:
46                res = executeShCmd(cmd.rhs, cfg, cwd, results)
47            return res
48
49        raise ValueError('Unknown shell command: %r' % cmd.op)
50
51    assert isinstance(cmd, ShUtil.Pipeline)
52    procs = []
53    input = subprocess.PIPE
54    stderrTempFiles = []
55    opened_files = []
56    named_temp_files = []
57    # To avoid deadlock, we use a single stderr stream for piped
58    # output. This is null until we have seen some output using
59    # stderr.
60    for i,j in enumerate(cmd.commands):
61        # Apply the redirections, we use (N,) as a sentinel to indicate stdin,
62        # stdout, stderr for N equal to 0, 1, or 2 respectively. Redirects to or
63        # from a file are represented with a list [file, mode, file-object]
64        # where file-object is initially None.
65        redirects = [(0,), (1,), (2,)]
66        for r in j.redirects:
67            if r[0] == ('>',2):
68                redirects[2] = [r[1], 'w', None]
69            elif r[0] == ('>>',2):
70                redirects[2] = [r[1], 'a', None]
71            elif r[0] == ('>&',2) and r[1] in '012':
72                redirects[2] = redirects[int(r[1])]
73            elif r[0] == ('>&',) or r[0] == ('&>',):
74                redirects[1] = redirects[2] = [r[1], 'w', None]
75            elif r[0] == ('>',):
76                redirects[1] = [r[1], 'w', None]
77            elif r[0] == ('>>',):
78                redirects[1] = [r[1], 'a', None]
79            elif r[0] == ('<',):
80                redirects[0] = [r[1], 'r', None]
81            else:
82                raise InternalShellError(j,"Unsupported redirect: %r" % (r,))
83
84        # Map from the final redirections to something subprocess can handle.
85        final_redirects = []
86        for index,r in enumerate(redirects):
87            if r == (0,):
88                result = input
89            elif r == (1,):
90                if index == 0:
91                    raise InternalShellError(j,"Unsupported redirect for stdin")
92                elif index == 1:
93                    result = subprocess.PIPE
94                else:
95                    result = subprocess.STDOUT
96            elif r == (2,):
97                if index != 2:
98                    raise InternalShellError(j,"Unsupported redirect on stdout")
99                result = subprocess.PIPE
100            else:
101                if r[2] is None:
102                    if kAvoidDevNull and r[0] == '/dev/null':
103                        r[2] = tempfile.TemporaryFile(mode=r[1])
104                    else:
105                        r[2] = open(r[0], r[1])
106                    # Workaround a Win32 and/or subprocess bug when appending.
107                    #
108                    # FIXME: Actually, this is probably an instance of PR6753.
109                    if r[1] == 'a':
110                        r[2].seek(0, 2)
111                    opened_files.append(r[2])
112                result = r[2]
113            final_redirects.append(result)
114
115        stdin, stdout, stderr = final_redirects
116
117        # If stderr wants to come from stdout, but stdout isn't a pipe, then put
118        # stderr on a pipe and treat it as stdout.
119        if (stderr == subprocess.STDOUT and stdout != subprocess.PIPE):
120            stderr = subprocess.PIPE
121            stderrIsStdout = True
122        else:
123            stderrIsStdout = False
124
125            # Don't allow stderr on a PIPE except for the last
126            # process, this could deadlock.
127            #
128            # FIXME: This is slow, but so is deadlock.
129            if stderr == subprocess.PIPE and j != cmd.commands[-1]:
130                stderr = tempfile.TemporaryFile(mode='w+b')
131                stderrTempFiles.append((i, stderr))
132
133        # Resolve the executable path ourselves.
134        args = list(j.args)
135        executable = lit.util.which(args[0], cfg.environment['PATH'])
136        if not executable:
137            raise InternalShellError(j, '%r: command not found' % j.args[0])
138
139        # Replace uses of /dev/null with temporary files.
140        if kAvoidDevNull:
141            for i,arg in enumerate(args):
142                if arg == "/dev/null":
143                    f = tempfile.NamedTemporaryFile(delete=False)
144                    f.close()
145                    named_temp_files.append(f.name)
146                    args[i] = f.name
147
148        try:
149            procs.append(subprocess.Popen(args, cwd=cwd,
150                                          executable = executable,
151                                          stdin = stdin,
152                                          stdout = stdout,
153                                          stderr = stderr,
154                                          env = cfg.environment,
155                                          close_fds = kUseCloseFDs))
156        except OSError as e:
157            raise InternalShellError(j, 'Could not create process due to {}'.format(e))
158
159        # Immediately close stdin for any process taking stdin from us.
160        if stdin == subprocess.PIPE:
161            procs[-1].stdin.close()
162            procs[-1].stdin = None
163
164        # Update the current stdin source.
165        if stdout == subprocess.PIPE:
166            input = procs[-1].stdout
167        elif stderrIsStdout:
168            input = procs[-1].stderr
169        else:
170            input = subprocess.PIPE
171
172    # Explicitly close any redirected files. We need to do this now because we
173    # need to release any handles we may have on the temporary files (important
174    # on Win32, for example). Since we have already spawned the subprocess, our
175    # handles have already been transferred so we do not need them anymore.
176    for f in opened_files:
177        f.close()
178
179    # FIXME: There is probably still deadlock potential here. Yawn.
180    procData = [None] * len(procs)
181    procData[-1] = procs[-1].communicate()
182
183    for i in range(len(procs) - 1):
184        if procs[i].stdout is not None:
185            out = procs[i].stdout.read()
186        else:
187            out = ''
188        if procs[i].stderr is not None:
189            err = procs[i].stderr.read()
190        else:
191            err = ''
192        procData[i] = (out,err)
193
194    # Read stderr out of the temp files.
195    for i,f in stderrTempFiles:
196        f.seek(0, 0)
197        procData[i] = (procData[i][0], f.read())
198
199    def to_string(bytes):
200        if isinstance(bytes, str):
201            return bytes
202        return bytes.encode('utf-8')
203
204    exitCode = None
205    for i,(out,err) in enumerate(procData):
206        res = procs[i].wait()
207        # Detect Ctrl-C in subprocess.
208        if res == -signal.SIGINT:
209            raise KeyboardInterrupt
210
211        # Ensure the resulting output is always of string type.
212        try:
213            out = to_string(out.decode('utf-8'))
214        except:
215            out = str(out)
216        try:
217            err = to_string(err.decode('utf-8'))
218        except:
219            err = str(err)
220
221        results.append((cmd.commands[i], out, err, res))
222        if cmd.pipe_err:
223            # Python treats the exit code as a signed char.
224            if exitCode is None:
225                exitCode = res
226            elif res < 0:
227                exitCode = min(exitCode, res)
228            else:
229                exitCode = max(exitCode, res)
230        else:
231            exitCode = res
232
233    # Remove any named temporary files we created.
234    for f in named_temp_files:
235        try:
236            os.remove(f)
237        except OSError:
238            pass
239
240    if cmd.negate:
241        exitCode = not exitCode
242
243    return exitCode
244
245def executeScriptInternal(test, litConfig, tmpBase, commands, cwd):
246    cmds = []
247    for ln in commands:
248        try:
249            cmds.append(ShUtil.ShParser(ln, litConfig.isWindows,
250                                        test.config.pipefail).parse())
251        except:
252            return lit.Test.Result(Test.FAIL, "shell parser error on: %r" % ln)
253
254    cmd = cmds[0]
255    for c in cmds[1:]:
256        cmd = ShUtil.Seq(cmd, '&&', c)
257
258    results = []
259    try:
260        exitCode = executeShCmd(cmd, test.config, cwd, results)
261    except InternalShellError:
262        e = sys.exc_info()[1]
263        exitCode = 127
264        results.append((e.command, '', e.message, exitCode))
265
266    out = err = ''
267    for i,(cmd, cmd_out,cmd_err,res) in enumerate(results):
268        out += 'Command %d: %s\n' % (i, ' '.join('"%s"' % s for s in cmd.args))
269        out += 'Command %d Result: %r\n' % (i, res)
270        out += 'Command %d Output:\n%s\n\n' % (i, cmd_out)
271        out += 'Command %d Stderr:\n%s\n\n' % (i, cmd_err)
272
273    return out, err, exitCode
274
275def executeScript(test, litConfig, tmpBase, commands, cwd):
276    bashPath = litConfig.getBashPath();
277    isWin32CMDEXE = (litConfig.isWindows and not bashPath)
278    script = tmpBase + '.script'
279    if isWin32CMDEXE:
280        script += '.bat'
281
282    # Write script file
283    mode = 'w'
284    if litConfig.isWindows and not isWin32CMDEXE:
285      mode += 'b'  # Avoid CRLFs when writing bash scripts.
286    f = open(script, mode)
287    if isWin32CMDEXE:
288        f.write('\nif %ERRORLEVEL% NEQ 0 EXIT\n'.join(commands))
289    else:
290        if test.config.pipefail:
291            f.write('set -o pipefail;')
292        f.write('{ ' + '; } &&\n{ '.join(commands) + '; }')
293    f.write('\n')
294    f.close()
295
296    if isWin32CMDEXE:
297        command = ['cmd','/c', script]
298    else:
299        if bashPath:
300            command = [bashPath, script]
301        else:
302            command = ['/bin/sh', script]
303        if litConfig.useValgrind:
304            # FIXME: Running valgrind on sh is overkill. We probably could just
305            # run on clang with no real loss.
306            command = litConfig.valgrindArgs + command
307
308    return lit.util.executeCommand(command, cwd=cwd,
309                                   env=test.config.environment)
310
311def parseIntegratedTestScriptCommands(source_path):
312    """
313    parseIntegratedTestScriptCommands(source_path) -> commands
314
315    Parse the commands in an integrated test script file into a list of
316    (line_number, command_type, line).
317    """
318
319    # This code is carefully written to be dual compatible with Python 2.5+ and
320    # Python 3 without requiring input files to always have valid codings. The
321    # trick we use is to open the file in binary mode and use the regular
322    # expression library to find the commands, with it scanning strings in
323    # Python2 and bytes in Python3.
324    #
325    # Once we find a match, we do require each script line to be decodable to
326    # UTF-8, so we convert the outputs to UTF-8 before returning. This way the
327    # remaining code can work with "strings" agnostic of the executing Python
328    # version.
329
330    keywords = ['RUN:', 'XFAIL:', 'REQUIRES:', 'UNSUPPORTED:', 'END.']
331    keywords_re = re.compile(
332        to_bytes("(%s)(.*)\n" % ("|".join(k for k in keywords),)))
333
334    f = open(source_path, 'rb')
335    try:
336        # Read the entire file contents.
337        data = f.read()
338
339        # Ensure the data ends with a newline.
340        if not data.endswith(to_bytes('\n')):
341            data = data + to_bytes('\n')
342
343        # Iterate over the matches.
344        line_number = 1
345        last_match_position = 0
346        for match in keywords_re.finditer(data):
347            # Compute the updated line number by counting the intervening
348            # newlines.
349            match_position = match.start()
350            line_number += data.count(to_bytes('\n'), last_match_position,
351                                      match_position)
352            last_match_position = match_position
353
354            # Convert the keyword and line to UTF-8 strings and yield the
355            # command. Note that we take care to return regular strings in
356            # Python 2, to avoid other code having to differentiate between the
357            # str and unicode types.
358            keyword,ln = match.groups()
359            yield (line_number, to_string(keyword[:-1].decode('utf-8')),
360                   to_string(ln.decode('utf-8')))
361    finally:
362        f.close()
363
364
365def parseIntegratedTestScript(test, normalize_slashes=False,
366                              extra_substitutions=[], require_script=True):
367    """parseIntegratedTestScript - Scan an LLVM/Clang style integrated test
368    script and extract the lines to 'RUN' as well as 'XFAIL' and 'REQUIRES'
369    and 'UNSUPPORTED' information. The RUN lines also will have variable
370    substitution performed. If 'require_script' is False an empty script may be
371    returned. This can be used for test formats where the actual script is
372    optional or ignored.
373    """
374
375    # Get the temporary location, this is always relative to the test suite
376    # root, not test source root.
377    #
378    # FIXME: This should not be here?
379    sourcepath = test.getSourcePath()
380    sourcedir = os.path.dirname(sourcepath)
381    execpath = test.getExecPath()
382    execdir,execbase = os.path.split(execpath)
383    tmpDir = os.path.join(execdir, 'Output')
384    tmpBase = os.path.join(tmpDir, execbase)
385
386    # Normalize slashes, if requested.
387    if normalize_slashes:
388        sourcepath = sourcepath.replace('\\', '/')
389        sourcedir = sourcedir.replace('\\', '/')
390        tmpDir = tmpDir.replace('\\', '/')
391        tmpBase = tmpBase.replace('\\', '/')
392
393    # We use #_MARKER_# to hide %% while we do the other substitutions.
394    substitutions = list(extra_substitutions)
395    substitutions.extend([('%%', '#_MARKER_#')])
396    substitutions.extend(test.config.substitutions)
397    substitutions.extend([('%s', sourcepath),
398                          ('%S', sourcedir),
399                          ('%p', sourcedir),
400                          ('%{pathsep}', os.pathsep),
401                          ('%t', tmpBase + '.tmp'),
402                          ('%T', tmpDir),
403                          ('#_MARKER_#', '%')])
404
405    # "%/[STpst]" should be normalized.
406    substitutions.extend([
407            ('%/s', sourcepath.replace('\\', '/')),
408            ('%/S', sourcedir.replace('\\', '/')),
409            ('%/p', sourcedir.replace('\\', '/')),
410            ('%/t', tmpBase.replace('\\', '/') + '.tmp'),
411            ('%/T', tmpDir.replace('\\', '/')),
412            ])
413
414    # Collect the test lines from the script.
415    script = []
416    requires = []
417    unsupported = []
418    for line_number, command_type, ln in \
419            parseIntegratedTestScriptCommands(sourcepath):
420        if command_type == 'RUN':
421            # Trim trailing whitespace.
422            ln = ln.rstrip()
423
424            # Substitute line number expressions
425            ln = re.sub('%\(line\)', str(line_number), ln)
426            def replace_line_number(match):
427                if match.group(1) == '+':
428                    return str(line_number + int(match.group(2)))
429                if match.group(1) == '-':
430                    return str(line_number - int(match.group(2)))
431            ln = re.sub('%\(line *([\+-]) *(\d+)\)', replace_line_number, ln)
432
433            # Collapse lines with trailing '\\'.
434            if script and script[-1][-1] == '\\':
435                script[-1] = script[-1][:-1] + ln
436            else:
437                script.append(ln)
438        elif command_type == 'XFAIL':
439            test.xfails.extend([s.strip() for s in ln.split(',')])
440        elif command_type == 'REQUIRES':
441            requires.extend([s.strip() for s in ln.split(',')])
442        elif command_type == 'UNSUPPORTED':
443            unsupported.extend([s.strip() for s in ln.split(',')])
444        elif command_type == 'END':
445            # END commands are only honored if the rest of the line is empty.
446            if not ln.strip():
447                break
448        else:
449            raise ValueError("unknown script command type: %r" % (
450                    command_type,))
451
452    # Apply substitutions to the script.  Allow full regular
453    # expression syntax.  Replace each matching occurrence of regular
454    # expression pattern a with substitution b in line ln.
455    def processLine(ln):
456        # Apply substitutions
457        for a,b in substitutions:
458            if kIsWindows:
459                b = b.replace("\\","\\\\")
460            ln = re.sub(a, b, ln)
461
462        # Strip the trailing newline and any extra whitespace.
463        return ln.strip()
464    script = [processLine(ln)
465              for ln in script]
466
467    # Verify the script contains a run line.
468    if require_script and not script:
469        return lit.Test.Result(Test.UNRESOLVED, "Test has no run line!")
470
471    # Check for unterminated run lines.
472    if script and script[-1][-1] == '\\':
473        return lit.Test.Result(Test.UNRESOLVED,
474                               "Test has unterminated run lines (with '\\')")
475
476    # Check that we have the required features:
477    missing_required_features = [f for f in requires
478                                 if f not in test.config.available_features]
479    if missing_required_features:
480        msg = ', '.join(missing_required_features)
481        return lit.Test.Result(Test.UNSUPPORTED,
482                               "Test requires the following features: %s" % msg)
483    unsupported_features = [f for f in unsupported
484                            if f in test.config.available_features]
485    if unsupported_features:
486        msg = ', '.join(unsupported_features)
487        return lit.Test.Result(Test.UNSUPPORTED,
488                    "Test is unsupported with the following features: %s" % msg)
489
490    return script,tmpBase,execdir
491
492def _runShTest(test, litConfig, useExternalSh,
493                   script, tmpBase, execdir):
494    # Create the output directory if it does not already exist.
495    lit.util.mkdir_p(os.path.dirname(tmpBase))
496
497    if useExternalSh:
498        res = executeScript(test, litConfig, tmpBase, script, execdir)
499    else:
500        res = executeScriptInternal(test, litConfig, tmpBase, script, execdir)
501    if isinstance(res, lit.Test.Result):
502        return res
503
504    out,err,exitCode = res
505    if exitCode == 0:
506        status = Test.PASS
507    else:
508        status = Test.FAIL
509
510    # Form the output log.
511    output = """Script:\n--\n%s\n--\nExit Code: %d\n\n""" % (
512        '\n'.join(script), exitCode)
513
514    # Append the outputs, if present.
515    if out:
516        output += """Command Output (stdout):\n--\n%s\n--\n""" % (out,)
517    if err:
518        output += """Command Output (stderr):\n--\n%s\n--\n""" % (err,)
519
520    return lit.Test.Result(status, output)
521
522
523def executeShTest(test, litConfig, useExternalSh,
524                  extra_substitutions=[]):
525    if test.config.unsupported:
526        return (Test.UNSUPPORTED, 'Test is unsupported')
527
528    res = parseIntegratedTestScript(test, useExternalSh, extra_substitutions)
529    if isinstance(res, lit.Test.Result):
530        return res
531    if litConfig.noExecute:
532        return lit.Test.Result(Test.PASS)
533
534    script, tmpBase, execdir = res
535    return _runShTest(test, litConfig, useExternalSh, script, tmpBase, execdir)
536
537