1""" 2Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 3See https://llvm.org/LICENSE.txt for license information. 4SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 5 6Provides the LLDBTestResult class, which holds information about progress 7and results of a single test run. 8""" 9 10# System modules 11import os 12import traceback 13 14# Third-party modules 15import unittest2 16 17# LLDB Modules 18from . import configuration 19from lldbsuite.test_event import build_exception 20 21 22class LLDBTestResult(unittest2.TextTestResult): 23 """ 24 Enforce a singleton pattern to allow introspection of test progress. 25 26 Overwrite addError(), addFailure(), and addExpectedFailure() methods 27 to enable each test instance to track its failure/error status. It 28 is used in the LLDB test framework to emit detailed trace messages 29 to a log file for easier human inspection of test failures/errors. 30 """ 31 __singleton__ = None 32 __ignore_singleton__ = False 33 34 @staticmethod 35 def getTerminalSize(): 36 import os 37 env = os.environ 38 39 def ioctl_GWINSZ(fd): 40 try: 41 import fcntl 42 import termios 43 import struct 44 cr = struct.unpack('hh', fcntl.ioctl(fd, termios.TIOCGWINSZ, 45 '1234')) 46 except: 47 return 48 return cr 49 cr = ioctl_GWINSZ(0) or ioctl_GWINSZ(1) or ioctl_GWINSZ(2) 50 if not cr: 51 try: 52 fd = os.open(os.ctermid(), os.O_RDONLY) 53 cr = ioctl_GWINSZ(fd) 54 os.close(fd) 55 except: 56 pass 57 if not cr: 58 cr = (env.get('LINES', 25), env.get('COLUMNS', 80)) 59 return int(cr[1]), int(cr[0]) 60 61 def __init__(self, *args): 62 if not LLDBTestResult.__ignore_singleton__ and LLDBTestResult.__singleton__: 63 raise Exception("LLDBTestResult instantiated more than once") 64 super(LLDBTestResult, self).__init__(*args) 65 LLDBTestResult.__singleton__ = self 66 # Now put this singleton into the lldb module namespace. 67 configuration.test_result = self 68 # Computes the format string for displaying the counter. 69 counterWidth = len(str(configuration.suite.countTestCases())) 70 self.fmt = "%" + str(counterWidth) + "d: " 71 self.indentation = ' ' * (counterWidth + 2) 72 # This counts from 1 .. suite.countTestCases(). 73 self.counter = 0 74 (width, height) = LLDBTestResult.getTerminalSize() 75 76 def _config_string(self, test): 77 compiler = getattr(test, "getCompiler", None) 78 arch = getattr(test, "getArchitecture", None) 79 return "%s-%s" % (compiler() if compiler else "", 80 arch() if arch else "") 81 82 def _exc_info_to_string(self, err, test): 83 """Overrides superclass TestResult's method in order to append 84 our test config info string to the exception info string.""" 85 if hasattr(test, "getArchitecture") and hasattr(test, "getCompiler"): 86 return '%sConfig=%s-%s' % (super(LLDBTestResult, 87 self)._exc_info_to_string(err, 88 test), 89 test.getArchitecture(), 90 test.getCompiler()) 91 else: 92 return super(LLDBTestResult, self)._exc_info_to_string(err, test) 93 94 def getDescription(self, test): 95 doc_first_line = test.shortDescription() 96 if self.descriptions and doc_first_line: 97 return '\n'.join((str(test), self.indentation + doc_first_line)) 98 else: 99 return str(test) 100 101 def _getTestPath(self, test): 102 # Use test.test_filename if the test was created with 103 # lldbinline.MakeInlineTest(). 104 if test is None: 105 return "" 106 elif hasattr(test, "test_filename"): 107 return test.test_filename 108 else: 109 import inspect 110 return inspect.getsourcefile(test.__class__) 111 112 def _getFileBasedCategories(self, test): 113 """ 114 Returns the list of categories to which this test case belongs by 115 collecting values of "categories" files. We start at the folder the test is in 116 and traverse the hierarchy upwards until the test-suite root directory. 117 """ 118 start_path = self._getTestPath(test) 119 120 import os.path 121 folder = os.path.dirname(start_path) 122 123 from lldbsuite import lldb_test_root as test_root 124 if test_root != os.path.commonprefix([folder, test_root]): 125 raise Exception("The test file %s is outside the test root directory" % start_path) 126 127 categories = set() 128 while not os.path.samefile(folder, test_root): 129 categories_file_name = os.path.join(folder, "categories") 130 if os.path.exists(categories_file_name): 131 categories_file = open(categories_file_name, 'r') 132 categories_str = categories_file.readline().strip() 133 categories_file.close() 134 categories.update(categories_str.split(',')) 135 folder = os.path.dirname(folder) 136 137 return list(categories) 138 139 def getCategoriesForTest(self, test): 140 """ 141 Gets all the categories for the currently running test method in test case 142 """ 143 test_categories = [] 144 test_method = getattr(test, test._testMethodName) 145 if test_method is not None and hasattr(test_method, "categories"): 146 test_categories.extend(test_method.categories) 147 148 test_categories.extend(self._getFileBasedCategories(test)) 149 150 return test_categories 151 152 def hardMarkAsSkipped(self, test): 153 getattr(test, test._testMethodName).__func__.__unittest_skip__ = True 154 getattr( 155 test, 156 test._testMethodName).__func__.__unittest_skip_why__ = "test case does not fall in any category of interest for this run" 157 158 def checkExclusion(self, exclusion_list, name): 159 if exclusion_list: 160 import re 161 for item in exclusion_list: 162 if re.search(item, name): 163 return True 164 return False 165 166 def checkCategoryExclusion(self, exclusion_list, test): 167 return not set(exclusion_list).isdisjoint( 168 self.getCategoriesForTest(test)) 169 170 def startTest(self, test): 171 if configuration.shouldSkipBecauseOfCategories( 172 self.getCategoriesForTest(test)): 173 self.hardMarkAsSkipped(test) 174 if self.checkExclusion( 175 configuration.skip_tests, test.id()): 176 self.hardMarkAsSkipped(test) 177 178 self.counter += 1 179 test.test_number = self.counter 180 if self.showAll: 181 self.stream.write(self.fmt % self.counter) 182 super(LLDBTestResult, self).startTest(test) 183 184 def addSuccess(self, test): 185 if (self.checkExclusion( 186 configuration.xfail_tests, test.id()) or 187 self.checkCategoryExclusion( 188 configuration.xfail_categories, test)): 189 self.addUnexpectedSuccess(test, None) 190 return 191 192 super(LLDBTestResult, self).addSuccess(test) 193 self.stream.write( 194 "PASS: LLDB (%s) :: %s\n" % 195 (self._config_string(test), str(test))) 196 197 def _isBuildError(self, err_tuple): 198 exception = err_tuple[1] 199 return isinstance(exception, build_exception.BuildError) 200 201 def _saveBuildErrorTuple(self, test, err): 202 # Adjust the error description so it prints the build command and build error 203 # rather than an uninformative Python backtrace. 204 build_error = err[1] 205 error_description = "{}\nTest Directory:\n{}".format( 206 str(build_error), 207 os.path.dirname(self._getTestPath(test))) 208 self.errors.append((test, error_description)) 209 self._mirrorOutput = True 210 211 def addError(self, test, err): 212 configuration.sdir_has_content = True 213 if self._isBuildError(err): 214 self._saveBuildErrorTuple(test, err) 215 else: 216 super(LLDBTestResult, self).addError(test, err) 217 218 method = getattr(test, "markError", None) 219 if method: 220 method() 221 self.stream.write( 222 "FAIL: LLDB (%s) :: %s\n" % 223 (self._config_string(test), str(test))) 224 225 def addCleanupError(self, test, err): 226 configuration.sdir_has_content = True 227 super(LLDBTestResult, self).addCleanupError(test, err) 228 method = getattr(test, "markCleanupError", None) 229 if method: 230 method() 231 self.stream.write( 232 "CLEANUP ERROR: LLDB (%s) :: %s\n%s\n" % 233 (self._config_string(test), str(test), traceback.format_exc())) 234 235 def addFailure(self, test, err): 236 if (self.checkExclusion( 237 configuration.xfail_tests, test.id()) or 238 self.checkCategoryExclusion( 239 configuration.xfail_categories, test)): 240 self.addExpectedFailure(test, err, None) 241 return 242 243 configuration.sdir_has_content = True 244 super(LLDBTestResult, self).addFailure(test, err) 245 method = getattr(test, "markFailure", None) 246 if method: 247 method() 248 self.stream.write( 249 "FAIL: LLDB (%s) :: %s\n" % 250 (self._config_string(test), str(test))) 251 if configuration.use_categories: 252 test_categories = self.getCategoriesForTest(test) 253 for category in test_categories: 254 if category in configuration.failures_per_category: 255 configuration.failures_per_category[ 256 category] = configuration.failures_per_category[category] + 1 257 else: 258 configuration.failures_per_category[category] = 1 259 260 def addExpectedFailure(self, test, err, bugnumber): 261 configuration.sdir_has_content = True 262 super(LLDBTestResult, self).addExpectedFailure(test, err, bugnumber) 263 method = getattr(test, "markExpectedFailure", None) 264 if method: 265 method(err, bugnumber) 266 self.stream.write( 267 "XFAIL: LLDB (%s) :: %s\n" % 268 (self._config_string(test), str(test))) 269 270 def addSkip(self, test, reason): 271 configuration.sdir_has_content = True 272 super(LLDBTestResult, self).addSkip(test, reason) 273 method = getattr(test, "markSkippedTest", None) 274 if method: 275 method() 276 self.stream.write( 277 "UNSUPPORTED: LLDB (%s) :: %s (%s) \n" % 278 (self._config_string(test), str(test), reason)) 279 280 def addUnexpectedSuccess(self, test, bugnumber): 281 configuration.sdir_has_content = True 282 super(LLDBTestResult, self).addUnexpectedSuccess(test, bugnumber) 283 method = getattr(test, "markUnexpectedSuccess", None) 284 if method: 285 method(bugnumber) 286 self.stream.write( 287 "XPASS: LLDB (%s) :: %s\n" % 288 (self._config_string(test), str(test))) 289