1""" 2Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 3See https://llvm.org/LICENSE.txt for license information. 4SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 5 6Provides the LLDBTestResult class, which holds information about progress 7and results of a single test run. 8""" 9 10# System modules 11import os 12import traceback 13 14# Third-party modules 15import unittest 16 17# LLDB Modules 18from . import configuration 19from lldbsuite.test_event import build_exception 20 21 22class LLDBTestResult(unittest.TextTestResult): 23 """ 24 Enforce a singleton pattern to allow introspection of test progress. 25 26 Overwrite addError(), addFailure(), and addExpectedFailure() methods 27 to enable each test instance to track its failure/error status. It 28 is used in the LLDB test framework to emit detailed trace messages 29 to a log file for easier human inspection of test failures/errors. 30 """ 31 32 __singleton__ = None 33 __ignore_singleton__ = False 34 35 @staticmethod 36 def getTerminalSize(): 37 import os 38 39 env = os.environ 40 41 def ioctl_GWINSZ(fd): 42 try: 43 import fcntl 44 import termios 45 import struct 46 47 cr = struct.unpack("hh", fcntl.ioctl(fd, termios.TIOCGWINSZ, "1234")) 48 except: 49 return 50 return cr 51 52 cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) 53 if not cr: 54 try: 55 fd = os.open(os.ctermid(), os.O_RDONLY) 56 cr = ioctl_GWINSZ(fd) 57 os.close(fd) 58 except: 59 pass 60 if not cr: 61 cr = (env.get("LINES", 25), env.get("COLUMNS", 80)) 62 return int(cr[1]), int(cr[0]) 63 64 def __init__(self, *args): 65 if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__: 66 raise Exception("LLDBTestResult instantiated more than once") 67 super(LLDBTestResult, self).__init__(*args) 68 LLDBTestResult.__singleton__ = self 69 # Now put this singleton into the lldb module namespace. 70 configuration.test_result = self 71 # Computes the format string for displaying the counter. 72 counterWidth = len(str(configuration.suite.countTestCases())) 73 self.fmt = "%" + str(counterWidth) + "d: " 74 self.indentation = " " * (counterWidth + 2) 75 # This counts from 1 .. suite.countTestCases(). 76 self.counter = 0 77 (width, height) = LLDBTestResult.getTerminalSize() 78 79 def _config_string(self, test): 80 compiler = getattr(test, "getCompiler", None) 81 arch = getattr(test, "getArchitecture", None) 82 return "%s-%s" % (compiler() if compiler else "", arch() if arch else "") 83 84 def _exc_info_to_string(self, err, test): 85 """Overrides superclass TestResult's method in order to append 86 our test config info string to the exception info string.""" 87 if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"): 88 return "%sConfig=%s-%s" % ( 89 super(LLDBTestResult, self)._exc_info_to_string(err, test), 90 test.getArchitecture(), 91 test.getCompiler(), 92 ) 93 else: 94 return super(LLDBTestResult, self)._exc_info_to_string(err, test) 95 96 def getDescription(self, test): 97 doc_first_line = test.shortDescription() 98 if self.descriptions and doc_first_line: 99 return "\n".join((str(test), self.indentation + doc_first_line)) 100 else: 101 return str(test) 102 103 def _getTestPath(self, test): 104 # Use test.test_filename if the test was created with 105 # lldbinline.MakeInlineTest(). 106 if test is None: 107 return "" 108 elif hasattr(test, "test_filename"): 109 return test.test_filename 110 else: 111 import inspect 112 113 return inspect.getsourcefile(test.__class__) 114 115 def _getFileBasedCategories(self, test): 116 """ 117 Returns the list of categories to which this test case belongs by 118 collecting values of "categories" files. We start at the folder the test is in 119 and traverse the hierarchy upwards until the test-suite root directory. 120 """ 121 start_path = self._getTestPath(test) 122 123 import os.path 124 125 folder = os.path.dirname(start_path) 126 127 from lldbsuite import lldb_test_root as test_root 128 129 if test_root != os.path.commonprefix([folder, test_root]): 130 raise Exception( 131 "The test file %s is outside the test root directory" % start_path 132 ) 133 134 categories = set() 135 while not os.path.samefile(folder, test_root): 136 categories_file_name = os.path.join(folder, "categories") 137 if os.path.exists(categories_file_name): 138 categories_file = open(categories_file_name, "r") 139 categories_str = categories_file.readline().strip() 140 categories_file.close() 141 categories.update(categories_str.split(",")) 142 folder = os.path.dirname(folder) 143 144 return list(categories) 145 146 def getCategoriesForTest(self, test): 147 """ 148 Gets all the categories for the currently running test method in test case 149 """ 150 test_categories = [] 151 test_categories.extend(getattr(test, "categories", [])) 152 153 test_method = getattr(test, test._testMethodName) 154 if test_method is not None: 155 test_categories.extend(getattr(test_method, "categories", [])) 156 157 test_categories.extend(self._getFileBasedCategories(test)) 158 159 return test_categories 160 161 def hardMarkAsSkipped(self, test): 162 getattr(test, test._testMethodName).__func__.__unittest_skip__ = True 163 getattr( 164 test, test._testMethodName 165 ).__func__.__unittest_skip_why__ = ( 166 "test case does not fall in any category of interest for this run" 167 ) 168 169 def checkExclusion(self, exclusion_list, name): 170 if exclusion_list: 171 import re 172 173 for item in exclusion_list: 174 if re.search(item, name): 175 return True 176 return False 177 178 def checkCategoryExclusion(self, exclusion_list, test): 179 return not set(exclusion_list).isdisjoint(self.getCategoriesForTest(test)) 180 181 def startTest(self, test): 182 if configuration.shouldSkipBecauseOfCategories(self.getCategoriesForTest(test)): 183 self.hardMarkAsSkipped(test) 184 if self.checkExclusion(configuration.skip_tests, test.id()): 185 self.hardMarkAsSkipped(test) 186 187 self.counter += 1 188 test.test_number = self.counter 189 if self.showAll: 190 self.stream.write(self.fmt % self.counter) 191 super(LLDBTestResult, self).startTest(test) 192 193 def addSuccess(self, test): 194 if self.checkExclusion( 195 configuration.xfail_tests, test.id() 196 ) or self.checkCategoryExclusion(configuration.xfail_categories, test): 197 self.addUnexpectedSuccess(test, None) 198 return 199 200 super(LLDBTestResult, self).addSuccess(test) 201 self.stream.write( 202 "PASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)) 203 ) 204 205 def _isBuildError(self, err_tuple): 206 exception = err_tuple[1] 207 return isinstance(exception, build_exception.BuildError) 208 209 def _saveBuildErrorTuple(self, test, err): 210 # Adjust the error description so it prints the build command and build error 211 # rather than an uninformative Python backtrace. 212 build_error = err[1] 213 error_description = "{}\nTest Directory:\n{}".format( 214 str(build_error), os.path.dirname(self._getTestPath(test)) 215 ) 216 self.errors.append((test, error_description)) 217 self._mirrorOutput = True 218 219 def addError(self, test, err): 220 configuration.sdir_has_content = True 221 if self._isBuildError(err): 222 self._saveBuildErrorTuple(test, err) 223 else: 224 super(LLDBTestResult, self).addError(test, err) 225 226 method = getattr(test, "markError", None) 227 if method: 228 method() 229 self.stream.write( 230 "FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)) 231 ) 232 233 def addCleanupError(self, test, err): 234 configuration.sdir_has_content = True 235 super(LLDBTestResult, self).addCleanupError(test, err) 236 method = getattr(test, "markCleanupError", None) 237 if method: 238 method() 239 self.stream.write( 240 "CLEANUP ERROR: LLDB (%s) :: %s\n%s\n" 241 % (self._config_string(test), str(test), traceback.format_exc()) 242 ) 243 244 def addFailure(self, test, err): 245 if self.checkExclusion( 246 configuration.xfail_tests, test.id() 247 ) or self.checkCategoryExclusion(configuration.xfail_categories, test): 248 self.addExpectedFailure(test, err) 249 return 250 251 configuration.sdir_has_content = True 252 super(LLDBTestResult, self).addFailure(test, err) 253 method = getattr(test, "markFailure", None) 254 if method: 255 method() 256 self.stream.write( 257 "FAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)) 258 ) 259 if configuration.use_categories: 260 test_categories = self.getCategoriesForTest(test) 261 for category in test_categories: 262 if category in configuration.failures_per_category: 263 configuration.failures_per_category[category] = ( 264 configuration.failures_per_category[category] + 1 265 ) 266 else: 267 configuration.failures_per_category[category] = 1 268 269 def addExpectedFailure(self, test, err): 270 configuration.sdir_has_content = True 271 super(LLDBTestResult, self).addExpectedFailure(test, err) 272 method = getattr(test, "markExpectedFailure", None) 273 if method: 274 method(err) 275 self.stream.write( 276 "XFAIL: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)) 277 ) 278 279 def addSkip(self, test, reason): 280 configuration.sdir_has_content = True 281 super(LLDBTestResult, self).addSkip(test, reason) 282 method = getattr(test, "markSkippedTest", None) 283 if method: 284 method() 285 self.stream.write( 286 "UNSUPPORTED: LLDB (%s) :: %s (%s) \n" 287 % (self._config_string(test), str(test), reason) 288 ) 289 290 def addUnexpectedSuccess(self, test): 291 configuration.sdir_has_content = True 292 super(LLDBTestResult, self).addUnexpectedSuccess(test) 293 method = getattr(test, "markUnexpectedSuccess", None) 294 if method: 295 method() 296 self.stream.write( 297 "XPASS: LLDB (%s) :: %s\n" % (self._config_string(test), str(test)) 298 ) 299