1""" 2Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 3See https://llvm.org/LICENSE.txt for license information. 4SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 5 6Provides the LLDBTestResult class, which holds information about progress 7and results of a single test run. 8""" 9 10# System modules 11import os 12 13# Third-party modules 14import unittest2 15 16# LLDB Modules 17from . import configuration 18from lldbsuite.test_event.event_builder import EventBuilder 19from lldbsuite.test_event import build_exception 20 21 22class LLDBTestResult(unittest2.TextTestResult): 23 """ 24 Enforce a singleton pattern to allow introspection of test progress. 25 26 Overwrite addError(), addFailure(), and addExpectedFailure() methods 27 to enable each test instance to track its failure/error status. It 28 is used in the LLDB test framework to emit detailed trace messages 29 to a log file for easier human inspection of test failures/errors. 30 """ 31 __singleton__ = None 32 __ignore_singleton__ = False 33 34 @staticmethod 35 def getTerminalSize(): 36 import os 37 env = os.environ 38 39 def ioctl_GWINSZ(fd): 40 try: 41 import fcntl 42 import termios 43 import struct 44 cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, 45 '1234')) 46 except: 47 return 48 return cr 49 cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) 50 if not cr: 51 try: 52 fd = os.open(os.ctermid(), os.O_RDONLY) 53 cr = ioctl_GWINSZ(fd) 54 os.close(fd) 55 except: 56 pass 57 if not cr: 58 cr = (env.get('LINES', 25), env.get('COLUMNS', 80)) 59 return int(cr[1]), int(cr[0]) 60 61 def __init__(self, *args): 62 if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__: 63 raise Exception("LLDBTestResult instantiated more than once") 64 super(LLDBTestResult, self).__init__(*args) 65 LLDBTestResult.__singleton__ = self 66 # Now put this singleton into the lldb module namespace. 67 configuration.test_result = self 68 # Computes the format string for displaying the counter. 69 counterWidth = len(str(configuration.suite.countTestCases())) 70 self.fmt = "%" + str(counterWidth) + "d: " 71 self.indentation = ' ' * (counterWidth + 2) 72 # This counts from 1 .. suite.countTestCases(). 73 self.counter = 0 74 (width, height) = LLDBTestResult.getTerminalSize() 75 self.results_formatter = configuration.results_formatter_object 76 77 def _config_string(self, test): 78 compiler = getattr(test, "getCompiler", None) 79 arch = getattr(test, "getArchitecture", None) 80 return "%s-%s" % (compiler() if compiler else "", 81 arch() if arch else "") 82 83 def _exc_info_to_string(self, err, test): 84 """Overrides superclass TestResult's method in order to append 85 our test config info string to the exception info string.""" 86 if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"): 87 return '%sConfig=%s-%s' % (super(LLDBTestResult, 88 self)._exc_info_to_string(err, 89 test), 90 test.getArchitecture(), 91 test.getCompiler()) 92 else: 93 return super(LLDBTestResult, self)._exc_info_to_string(err, test) 94 95 def getDescription(self, test): 96 doc_first_line = test.shortDescription() 97 if self.descriptions and doc_first_line: 98 return '\n'.join((str(test), self.indentation + doc_first_line)) 99 else: 100 return str(test) 101 102 def _getTestPath(self, test): 103 # Use test.test_filename if the test was created with 104 # lldbinline.MakeInlineTest(). 105 if test is None: 106 return "" 107 elif hasattr(test, "test_filename"): 108 return test.test_filename 109 else: 110 import inspect 111 return inspect.getsourcefile(test.__class__) 112 113 def _getFileBasedCategories(self, test): 114 """ 115 Returns the list of categories to which this test case belongs by 116 collecting values of ".categories" files. We start at the folder the test is in 117 and traverse the hierarchy upwards until the test-suite root directory. 118 """ 119 start_path = self._getTestPath(test) 120 121 import os.path 122 folder = os.path.dirname(start_path) 123 124 from lldbsuite import lldb_test_root as test_root 125 if test_root != os.path.commonprefix([folder, test_root]): 126 raise Exception("The test file %s is outside the test root directory" % start_path) 127 128 categories = set() 129 while not os.path.samefile(folder, test_root): 130 categories_file_name = os.path.join(folder, ".categories") 131 if os.path.exists(categories_file_name): 132 categories_file = open(categories_file_name, 'r') 133 categories_str = categories_file.readline().strip() 134 categories_file.close() 135 categories.update(categories_str.split(',')) 136 folder = os.path.dirname(folder) 137 138 return list(categories) 139 140 def getCategoriesForTest(self, test): 141 """ 142 Gets all the categories for the currently running test method in test case 143 """ 144 test_categories = [] 145 test_method = getattr(test, test._testMethodName) 146 if test_method is not None and hasattr(test_method, "categories"): 147 test_categories.extend(test_method.categories) 148 149 test_categories.extend(self._getFileBasedCategories(test)) 150 151 return test_categories 152 153 def hardMarkAsSkipped(self, test): 154 getattr(test, test._testMethodName).__func__.__unittest_skip__ = True 155 getattr( 156 test, 157 test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run" 158 159 def checkExclusion(self, exclusion_list, name): 160 if exclusion_list: 161 import re 162 for item in exclusion_list: 163 if re.search(item, name): 164 return True 165 return False 166 167 def checkCategoryExclusion(self, exclusion_list, test): 168 return not set(exclusion_list).isdisjoint( 169 self.getCategoriesForTest(test)) 170 171 def startTest(self, test): 172 if configuration.shouldSkipBecauseOfCategories( 173 self.getCategoriesForTest(test)): 174 self.hardMarkAsSkipped(test) 175 if self.checkExclusion( 176 configuration.skip_tests, test.id()): 177 self.hardMarkAsSkipped(test) 178 179 self.counter += 1 180 test.test_number = self.counter 181 if self.showAll: 182 self.stream.write(self.fmt % self.counter) 183 super(LLDBTestResult, self).startTest(test) 184 if self.results_formatter: 185 self.results_formatter.handle_event( 186 EventBuilder.event_for_start(test)) 187 188 def addSuccess(self, test): 189 if (self.checkExclusion( 190 configuration.xfail_tests, test.id()) or 191 self.checkCategoryExclusion( 192 configuration.xfail_categories, test)): 193 self.addUnexpectedSuccess(test, None) 194 return 195 196 super(LLDBTestResult, self).addSuccess(test) 197 self.stream.write( 198 "PASS: LLDB (%s) :: %s\n" % 199 (self._config_string(test), str(test))) 200 if self.results_formatter: 201 self.results_formatter.handle_event( 202 EventBuilder.event_for_success(test)) 203 204 def _isBuildError(self, err_tuple): 205 exception = err_tuple[1] 206 return isinstance(exception, build_exception.BuildError) 207 208 def _saveBuildErrorTuple(self, test, err): 209 # Adjust the error description so it prints the build command and build error 210 # rather than an uninformative Python backtrace. 211 build_error = err[1] 212 error_description = "{}\nTest Directory:\n{}".format( 213 str(build_error), 214 os.path.dirname(self._getTestPath(test))) 215 self.errors.append((test, error_description)) 216 self._mirrorOutput = True 217 218 def addError(self, test, err): 219 configuration.sdir_has_content = True 220 if self._isBuildError(err): 221 self._saveBuildErrorTuple(test, err) 222 else: 223 super(LLDBTestResult, self).addError(test, err) 224 225 method = getattr(test, "markError", None) 226 if method: 227 method() 228 self.stream.write( 229 "FAIL: LLDB (%s) :: %s\n" % 230 (self._config_string(test), str(test))) 231 if self.results_formatter: 232 # Handle build errors as a separate event type 233 if self._isBuildError(err): 234 error_event = EventBuilder.event_for_build_error(test, err) 235 else: 236 error_event = EventBuilder.event_for_error(test, err) 237 self.results_formatter.handle_event(error_event) 238 239 def addCleanupError(self, test, err): 240 configuration.sdir_has_content = True 241 super(LLDBTestResult, self).addCleanupError(test, err) 242 method = getattr(test, "markCleanupError", None) 243 if method: 244 method() 245 self.stream.write( 246 "CLEANUP ERROR: LLDB (%s) :: %s\n" % 247 (self._config_string(test), str(test))) 248 if self.results_formatter: 249 self.results_formatter.handle_event( 250 EventBuilder.event_for_cleanup_error( 251 test, err)) 252 253 def addFailure(self, test, err): 254 if (self.checkExclusion( 255 configuration.xfail_tests, test.id()) or 256 self.checkCategoryExclusion( 257 configuration.xfail_categories, test)): 258 self.addExpectedFailure(test, err, None) 259 return 260 261 configuration.sdir_has_content = True 262 super(LLDBTestResult, self).addFailure(test, err) 263 method = getattr(test, "markFailure", None) 264 if method: 265 method() 266 self.stream.write( 267 "FAIL: LLDB (%s) :: %s\n" % 268 (self._config_string(test), str(test))) 269 if configuration.use_categories: 270 test_categories = self.getCategoriesForTest(test) 271 for category in test_categories: 272 if category in configuration.failures_per_category: 273 configuration.failures_per_category[ 274 category] = configuration.failures_per_category[category] + 1 275 else: 276 configuration.failures_per_category[category] = 1 277 if self.results_formatter: 278 self.results_formatter.handle_event( 279 EventBuilder.event_for_failure(test, err)) 280 281 def addExpectedFailure(self, test, err, bugnumber): 282 configuration.sdir_has_content = True 283 super(LLDBTestResult, self).addExpectedFailure(test, err, bugnumber) 284 method = getattr(test, "markExpectedFailure", None) 285 if method: 286 method(err, bugnumber) 287 self.stream.write( 288 "XFAIL: LLDB (%s) :: %s\n" % 289 (self._config_string(test), str(test))) 290 if self.results_formatter: 291 self.results_formatter.handle_event( 292 EventBuilder.event_for_expected_failure( 293 test, err, bugnumber)) 294 295 def addSkip(self, test, reason): 296 configuration.sdir_has_content = True 297 super(LLDBTestResult, self).addSkip(test, reason) 298 method = getattr(test, "markSkippedTest", None) 299 if method: 300 method() 301 self.stream.write( 302 "UNSUPPORTED: LLDB (%s) :: %s (%s) \n" % 303 (self._config_string(test), str(test), reason)) 304 if self.results_formatter: 305 self.results_formatter.handle_event( 306 EventBuilder.event_for_skip(test, reason)) 307 308 def addUnexpectedSuccess(self, test, bugnumber): 309 configuration.sdir_has_content = True 310 super(LLDBTestResult, self).addUnexpectedSuccess(test, bugnumber) 311 method = getattr(test, "markUnexpectedSuccess", None) 312 if method: 313 method(bugnumber) 314 self.stream.write( 315 "XPASS: LLDB (%s) :: %s\n" % 316 (self._config_string(test), str(test))) 317 if self.results_formatter: 318 self.results_formatter.handle_event( 319 EventBuilder.event_for_unexpected_success( 320 test, bugnumber)) 321