1#!/usr/bin/env python 2 3from __future__ import print_function 4import re, string, sys, os, time, math 5 6DEBUG = 0 7 8(tp, exp) = ("compile", "exec") 9 10 11def parse(file): 12 f = open(file, "r") 13 d = f.read() 14 15 # Cleanup weird stuff 16 d = re.sub(r",\d+:\d", "", d) 17 18 r = re.findall(r"TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n", d) 19 20 test = {} 21 fname = "" 22 for t in r: 23 if DEBUG: 24 print(t) 25 26 if t[0] == "PASS" or t[0] == "FAIL": 27 tmp = t[2].split("llvm-test/") 28 29 if DEBUG: 30 print(tmp) 31 32 if len(tmp) == 2: 33 fname = tmp[1].strip("\r\n") 34 else: 35 fname = tmp[0].strip("\r\n") 36 37 if fname not in test: 38 test[fname] = {} 39 40 test[fname][t[1] + " state"] = t[0] 41 test[fname][t[1] + " time"] = float("nan") 42 else: 43 try: 44 n = t[0].split("RESULT-")[1] 45 46 if DEBUG: 47 print("n == ", n) 48 49 if n == "compile-success": 50 test[fname]["compile time"] = float( 51 t[2].split("program")[1].strip("\r\n") 52 ) 53 54 elif n == "exec-success": 55 test[fname]["exec time"] = float( 56 t[2].split("program")[1].strip("\r\n") 57 ) 58 if DEBUG: 59 print(test[fname][string.replace(n, "-success", "")]) 60 61 else: 62 # print "ERROR!" 63 sys.exit(1) 64 65 except: 66 continue 67 68 return test 69 70 71# Diff results and look for regressions. 72def diffResults(d_old, d_new): 73 regressions = {} 74 passes = {} 75 removed = "" 76 77 for x in ["compile state", "compile time", "exec state", "exec time"]: 78 regressions[x] = "" 79 passes[x] = "" 80 81 for t in sorted(d_old.keys()): 82 if t in d_new: 83 84 # Check if the test passed or failed. 85 for x in ["compile state", "compile time", "exec state", "exec time"]: 86 87 if x not in d_old[t] and x not in d_new[t]: 88 continue 89 90 if x in d_old[t]: 91 if x in d_new[t]: 92 93 if d_old[t][x] == "PASS": 94 if d_new[t][x] != "PASS": 95 regressions[x] += t + "\n" 96 else: 97 if d_new[t][x] == "PASS": 98 passes[x] += t + "\n" 99 100 else: 101 regressions[x] += t + "\n" 102 103 if x == "compile state" or x == "exec state": 104 continue 105 106 # For execution time, if there is no result it's a fail. 107 if x not in d_old[t] and x not in d_new[t]: 108 continue 109 elif x not in d_new[t]: 110 regressions[x] += t + "\n" 111 elif x not in d_old[t]: 112 passes[x] += t + "\n" 113 114 if math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]): 115 continue 116 117 elif math.isnan(d_old[t][x]) and not math.isnan(d_new[t][x]): 118 passes[x] += t + "\n" 119 120 elif not math.isnan(d_old[t][x]) and math.isnan(d_new[t][x]): 121 regressions[x] += t + ": NaN%\n" 122 123 if ( 124 d_new[t][x] > d_old[t][x] 125 and d_old[t][x] > 0.0 126 and (d_new[t][x] - d_old[t][x]) / d_old[t][x] > 0.05 127 ): 128 regressions[x] += ( 129 t 130 + ": " 131 + "{0:.1f}".format( 132 100 * (d_new[t][x] - d_old[t][x]) / d_old[t][x] 133 ) 134 + "%\n" 135 ) 136 137 else: 138 removed += t + "\n" 139 140 if len(regressions["compile state"]) != 0: 141 print("REGRESSION: Compilation Failed") 142 print(regressions["compile state"]) 143 144 if len(regressions["exec state"]) != 0: 145 print("REGRESSION: Execution Failed") 146 print(regressions["exec state"]) 147 148 if len(regressions["compile time"]) != 0: 149 print("REGRESSION: Compilation Time") 150 print(regressions["compile time"]) 151 152 if len(regressions["exec time"]) != 0: 153 print("REGRESSION: Execution Time") 154 print(regressions["exec time"]) 155 156 if len(passes["compile state"]) != 0: 157 print("NEW PASSES: Compilation") 158 print(passes["compile state"]) 159 160 if len(passes["exec state"]) != 0: 161 print("NEW PASSES: Execution") 162 print(passes["exec state"]) 163 164 if len(removed) != 0: 165 print("REMOVED TESTS") 166 print(removed) 167 168 169# Main 170if len(sys.argv) < 3: 171 print("Usage:", sys.argv[0], "<old log> <new log>") 172 sys.exit(-1) 173 174d_old = parse(sys.argv[1]) 175d_new = parse(sys.argv[2]) 176 177diffResults(d_old, d_new) 178