1""" 2 The LLVM Compiler Infrastructure 3 4This file is distributed under the University of Illinois Open Source 5License. See LICENSE.TXT for details. 6 7Provides the LLDBTestResult class, which holds information about progress 8and results of a single test run. 9""" 10 11from __future__ import absolute_import 12from __future__ import print_function 13 14# System modules 15import inspect 16import os 17 18# Third-party modules 19import unittest2 20 21# LLDB Modules 22from . import configuration 23from lldbsuite.test_event.event_builder import EventBuilder 24from lldbsuite.test_event import build_exception 25 26 27class LLDBTestResult(unittest2.TextTestResult): 28 """ 29 Enforce a singleton pattern to allow introspection of test progress. 30 31 Overwrite addError(), addFailure(), and addExpectedFailure() methods 32 to enable each test instance to track its failure/error status. It 33 is used in the LLDB test framework to emit detailed trace messages 34 to a log file for easier human inspection of test failures/errors. 35 """ 36 __singleton__ = None 37 __ignore_singleton__ = False 38 39 @staticmethod 40 def getTerminalSize(): 41 import os 42 env = os.environ 43 44 def ioctl_GWINSZ(fd): 45 try: 46 import fcntl 47 import termios 48 import struct 49 import os 50 cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, 51 '1234')) 52 except: 53 return 54 return cr 55 cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) 56 if not cr: 57 try: 58 fd = os.open(os.ctermid(), os.O_RDONLY) 59 cr = ioctl_GWINSZ(fd) 60 os.close(fd) 61 except: 62 pass 63 if not cr: 64 cr = (env.get('LINES', 25), env.get('COLUMNS', 80)) 65 return int(cr[1]), int(cr[0]) 66 67 def __init__(self, *args): 68 if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__: 69 raise Exception("LLDBTestResult instantiated more than once") 70 super(LLDBTestResult, self).__init__(*args) 71 LLDBTestResult.__singleton__ = self 72 # Now put this singleton into the lldb module namespace. 73 configuration.test_result = self 74 # Computes the format string for displaying the counter. 75 counterWidth = len(str(configuration.suite.countTestCases())) 76 self.fmt = "%" + str(counterWidth) + "d: " 77 self.indentation = ' ' * (counterWidth + 2) 78 # This counts from 1 .. suite.countTestCases(). 79 self.counter = 0 80 (width, height) = LLDBTestResult.getTerminalSize() 81 self.results_formatter = configuration.results_formatter_object 82 83 def _config_string(self, test): 84 compiler = getattr(test, "getCompiler", None) 85 arch = getattr(test, "getArchitecture", None) 86 return "%s-%s" % (compiler() if compiler else "", 87 arch() if arch else "") 88 89 def _exc_info_to_string(self, err, test): 90 """Overrides superclass TestResult's method in order to append 91 our test config info string to the exception info string.""" 92 if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"): 93 return '%sConfig=%s-%s' % (super(LLDBTestResult, 94 self)._exc_info_to_string(err, 95 test), 96 test.getArchitecture(), 97 test.getCompiler()) 98 else: 99 return super(LLDBTestResult, self)._exc_info_to_string(err, test) 100 101 def getDescription(self, test): 102 doc_first_line = test.shortDescription() 103 if self.descriptions and doc_first_line: 104 return '\n'.join((str(test), self.indentation + doc_first_line)) 105 else: 106 return str(test) 107 108 def getCategoriesForTest(self, test): 109 """ 110 Gets all the categories for the currently running test method in test case 111 """ 112 test_categories = [] 113 test_method = getattr(test, test._testMethodName) 114 if test_method is not None and hasattr(test_method, "categories"): 115 test_categories.extend(test_method.categories) 116 117 test_categories.extend(test.getCategories()) 118 119 return test_categories 120 121 def hardMarkAsSkipped(self, test): 122 getattr(test, test._testMethodName).__func__.__unittest_skip__ = True 123 getattr( 124 test, 125 test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run" 126 127 def startTest(self, test): 128 if configuration.shouldSkipBecauseOfCategories( 129 self.getCategoriesForTest(test)): 130 self.hardMarkAsSkipped(test) 131 configuration.setCrashInfoHook( 132 "%s at %s" % 133 (str(test), inspect.getfile( 134 test.__class__))) 135 self.counter += 1 136 # if self.counter == 4: 137 # import crashinfo 138 # crashinfo.testCrashReporterDescription(None) 139 test.test_number = self.counter 140 if self.showAll: 141 self.stream.write(self.fmt % self.counter) 142 super(LLDBTestResult, self).startTest(test) 143 if self.results_formatter: 144 self.results_formatter.handle_event( 145 EventBuilder.event_for_start(test)) 146 147 def addSuccess(self, test): 148 super(LLDBTestResult, self).addSuccess(test) 149 if configuration.parsable: 150 self.stream.write( 151 "PASS: LLDB (%s) :: %s\n" % 152 (self._config_string(test), str(test))) 153 if self.results_formatter: 154 self.results_formatter.handle_event( 155 EventBuilder.event_for_success(test)) 156 157 def _isBuildError(self, err_tuple): 158 exception = err_tuple[1] 159 return isinstance(exception, build_exception.BuildError) 160 161 def _getTestPath(self, test): 162 if test is None: 163 return "" 164 elif hasattr(test, "test_filename"): 165 return test.test_filename 166 else: 167 return inspect.getsourcefile(test.__class__) 168 169 def _saveBuildErrorTuple(self, test, err): 170 # Adjust the error description so it prints the build command and build error 171 # rather than an uninformative Python backtrace. 172 build_error = err[1] 173 error_description = "{}\nTest Directory:\n{}".format( 174 str(build_error), 175 os.path.dirname(self._getTestPath(test))) 176 self.errors.append((test, error_description)) 177 self._mirrorOutput = True 178 179 def addError(self, test, err): 180 configuration.sdir_has_content = True 181 if self._isBuildError(err): 182 self._saveBuildErrorTuple(test, err) 183 else: 184 super(LLDBTestResult, self).addError(test, err) 185 186 method = getattr(test, "markError", None) 187 if method: 188 method() 189 if configuration.parsable: 190 self.stream.write( 191 "FAIL: LLDB (%s) :: %s\n" % 192 (self._config_string(test), str(test))) 193 if self.results_formatter: 194 # Handle build errors as a separate event type 195 if self._isBuildError(err): 196 error_event = EventBuilder.event_for_build_error(test, err) 197 else: 198 error_event = EventBuilder.event_for_error(test, err) 199 self.results_formatter.handle_event(error_event) 200 201 def addCleanupError(self, test, err): 202 configuration.sdir_has_content = True 203 super(LLDBTestResult, self).addCleanupError(test, err) 204 method = getattr(test, "markCleanupError", None) 205 if method: 206 method() 207 if configuration.parsable: 208 self.stream.write( 209 "CLEANUP ERROR: LLDB (%s) :: %s\n" % 210 (self._config_string(test), str(test))) 211 if self.results_formatter: 212 self.results_formatter.handle_event( 213 EventBuilder.event_for_cleanup_error( 214 test, err)) 215 216 def addFailure(self, test, err): 217 configuration.sdir_has_content = True 218 super(LLDBTestResult, self).addFailure(test, err) 219 method = getattr(test, "markFailure", None) 220 if method: 221 method() 222 if configuration.parsable: 223 self.stream.write( 224 "FAIL: LLDB (%s) :: %s\n" % 225 (self._config_string(test), str(test))) 226 if configuration.useCategories: 227 test_categories = self.getCategoriesForTest(test) 228 for category in test_categories: 229 if category in configuration.failuresPerCategory: 230 configuration.failuresPerCategory[ 231 category] = configuration.failuresPerCategory[category] + 1 232 else: 233 configuration.failuresPerCategory[category] = 1 234 if self.results_formatter: 235 self.results_formatter.handle_event( 236 EventBuilder.event_for_failure(test, err)) 237 238 def addExpectedFailure(self, test, err, bugnumber): 239 configuration.sdir_has_content = True 240 super(LLDBTestResult, self).addExpectedFailure(test, err, bugnumber) 241 method = getattr(test, "markExpectedFailure", None) 242 if method: 243 method(err, bugnumber) 244 if configuration.parsable: 245 self.stream.write( 246 "XFAIL: LLDB (%s) :: %s\n" % 247 (self._config_string(test), str(test))) 248 if self.results_formatter: 249 self.results_formatter.handle_event( 250 EventBuilder.event_for_expected_failure( 251 test, err, bugnumber)) 252 253 def addSkip(self, test, reason): 254 configuration.sdir_has_content = True 255 super(LLDBTestResult, self).addSkip(test, reason) 256 method = getattr(test, "markSkippedTest", None) 257 if method: 258 method() 259 if configuration.parsable: 260 self.stream.write( 261 "UNSUPPORTED: LLDB (%s) :: %s (%s) \n" % 262 (self._config_string(test), str(test), reason)) 263 if self.results_formatter: 264 self.results_formatter.handle_event( 265 EventBuilder.event_for_skip(test, reason)) 266 267 def addUnexpectedSuccess(self, test, bugnumber): 268 configuration.sdir_has_content = True 269 super(LLDBTestResult, self).addUnexpectedSuccess(test, bugnumber) 270 method = getattr(test, "markUnexpectedSuccess", None) 271 if method: 272 method(bugnumber) 273 if configuration.parsable: 274 self.stream.write( 275 "XPASS: LLDB (%s) :: %s\n" % 276 (self._config_string(test), str(test))) 277 if self.results_formatter: 278 self.results_formatter.handle_event( 279 EventBuilder.event_for_unexpected_success( 280 test, bugnumber)) 281