1import os 2from xml.sax.saxutils import escape 3from json import JSONEncoder 4 5# Test result codes. 6 7class ResultCode(object): 8 """Test result codes.""" 9 10 # We override __new__ and __getnewargs__ to ensure that pickling still 11 # provides unique ResultCode objects in any particular instance. 12 _instances = {} 13 def __new__(cls, name, isFailure): 14 res = cls._instances.get(name) 15 if res is None: 16 cls._instances[name] = res = super(ResultCode, cls).__new__(cls) 17 return res 18 def __getnewargs__(self): 19 return (self.name, self.isFailure) 20 21 def __init__(self, name, isFailure): 22 self.name = name 23 self.isFailure = isFailure 24 25 def __repr__(self): 26 return '%s%r' % (self.__class__.__name__, 27 (self.name, self.isFailure)) 28 29PASS = ResultCode('PASS', False) 30XFAIL = ResultCode('XFAIL', False) 31FAIL = ResultCode('FAIL', True) 32XPASS = ResultCode('XPASS', True) 33UNRESOLVED = ResultCode('UNRESOLVED', True) 34UNSUPPORTED = ResultCode('UNSUPPORTED', False) 35 36# Test metric values. 37 38class MetricValue(object): 39 def format(self): 40 """ 41 format() -> str 42 43 Convert this metric to a string suitable for displaying as part of the 44 console output. 45 """ 46 raise RuntimeError("abstract method") 47 48 def todata(self): 49 """ 50 todata() -> json-serializable data 51 52 Convert this metric to content suitable for serializing in the JSON test 53 output. 54 """ 55 raise RuntimeError("abstract method") 56 57class IntMetricValue(MetricValue): 58 def __init__(self, value): 59 self.value = value 60 61 def format(self): 62 return str(self.value) 63 64 def todata(self): 65 return self.value 66 67class RealMetricValue(MetricValue): 68 def __init__(self, value): 69 self.value = value 70 71 def format(self): 72 return '%.4f' % self.value 73 74 def todata(self): 75 return self.value 76 77class JSONMetricValue(MetricValue): 78 """ 79 JSONMetricValue is used for types that are representable in the output 80 but that are otherwise uninterpreted. 81 """ 82 def __init__(self, value): 83 # Ensure the value is a serializable by trying to encode it. 84 # WARNING: The value may change before it is encoded again, and may 85 # not be encodable after the change. 86 try: 87 e = JSONEncoder() 88 e.encode(value) 89 except TypeError: 90 raise 91 self.value = value 92 93 def format(self): 94 return str(self.value) 95 96 def todata(self): 97 return self.value 98 99def toMetricValue(value): 100 if isinstance(value, MetricValue): 101 return value 102 elif isinstance(value, int) or isinstance(value, long): 103 return IntMetricValue(value) 104 elif isinstance(value, float): 105 return RealMetricValue(value) 106 else: 107 # Try to create a JSONMetricValue and let the constructor throw 108 # if value is not a valid type. 109 return JSONMetricValue(value) 110 111 112# Test results. 113 114class Result(object): 115 """Wrapper for the results of executing an individual test.""" 116 117 def __init__(self, code, output='', elapsed=None): 118 # The result code. 119 self.code = code 120 # The test output. 121 self.output = output 122 # The wall timing to execute the test, if timing. 123 self.elapsed = elapsed 124 # The metrics reported by this test. 125 self.metrics = {} 126 127 def addMetric(self, name, value): 128 """ 129 addMetric(name, value) 130 131 Attach a test metric to the test result, with the given name and list of 132 values. It is an error to attempt to attach the metrics with the same 133 name multiple times. 134 135 Each value must be an instance of a MetricValue subclass. 136 """ 137 if name in self.metrics: 138 raise ValueError("result already includes metrics for %r" % ( 139 name,)) 140 if not isinstance(value, MetricValue): 141 raise TypeError("unexpected metric value: %r" % (value,)) 142 self.metrics[name] = value 143 144# Test classes. 145 146class TestSuite: 147 """TestSuite - Information on a group of tests. 148 149 A test suite groups together a set of logically related tests. 150 """ 151 152 def __init__(self, name, source_root, exec_root, config): 153 self.name = name 154 self.source_root = source_root 155 self.exec_root = exec_root 156 # The test suite configuration. 157 self.config = config 158 159 def getSourcePath(self, components): 160 return os.path.join(self.source_root, *components) 161 162 def getExecPath(self, components): 163 return os.path.join(self.exec_root, *components) 164 165class Test: 166 """Test - Information on a single test instance.""" 167 168 def __init__(self, suite, path_in_suite, config, file_path = None): 169 self.suite = suite 170 self.path_in_suite = path_in_suite 171 self.config = config 172 self.file_path = file_path 173 # A list of conditions under which this test is expected to fail. These 174 # can optionally be provided by test format handlers, and will be 175 # honored when the test result is supplied. 176 self.xfails = [] 177 # The test result, once complete. 178 self.result = None 179 180 def setResult(self, result): 181 if self.result is not None: 182 raise ArgumentError("test result already set") 183 if not isinstance(result, Result): 184 raise ArgumentError("unexpected result type") 185 186 self.result = result 187 188 # Apply the XFAIL handling to resolve the result exit code. 189 if self.isExpectedToFail(): 190 if self.result.code == PASS: 191 self.result.code = XPASS 192 elif self.result.code == FAIL: 193 self.result.code = XFAIL 194 195 def getFullName(self): 196 return self.suite.config.name + ' :: ' + '/'.join(self.path_in_suite) 197 198 def getFilePath(self): 199 if self.file_path: 200 return self.file_path 201 return self.getSourcePath() 202 203 def getSourcePath(self): 204 return self.suite.getSourcePath(self.path_in_suite) 205 206 def getExecPath(self): 207 return self.suite.getExecPath(self.path_in_suite) 208 209 def isExpectedToFail(self): 210 """ 211 isExpectedToFail() -> bool 212 213 Check whether this test is expected to fail in the current 214 configuration. This check relies on the test xfails property which by 215 some test formats may not be computed until the test has first been 216 executed. 217 """ 218 219 # Check if any of the xfails match an available feature or the target. 220 for item in self.xfails: 221 # If this is the wildcard, it always fails. 222 if item == '*': 223 return True 224 225 # If this is an exact match for one of the features, it fails. 226 if item in self.config.available_features: 227 return True 228 229 # If this is a part of the target triple, it fails. 230 if item in self.suite.config.target_triple: 231 return True 232 233 return False 234 235 236 def getJUnitXML(self): 237 test_name = self.path_in_suite[-1] 238 test_path = self.path_in_suite[:-1] 239 safe_test_path = [x.replace(".","_") for x in test_path] 240 safe_name = self.suite.name.replace(".","-") 241 242 if safe_test_path: 243 class_name = safe_name + "." + "/".join(safe_test_path) 244 else: 245 class_name = safe_name + "." + safe_name 246 247 xml = "<testcase classname='" + class_name + "' name='" + \ 248 test_name + "'" 249 xml += " time='%.2f'" % (self.result.elapsed,) 250 if self.result.code.isFailure: 251 xml += ">\n\t<failure >\n" + escape(self.result.output) 252 xml += "\n\t</failure>\n</testcase>" 253 else: 254 xml += "/>" 255 return xml