xref: /llvm-project/llvm/utils/release/findRegressions-nightly.py (revision b71edfaa4ec3c998aadb35255ce2f60bba2940b0) 
1#!/usr/bin/env python
2from __future__ import print_function
3
4import re, string, sys, os, time
5
6DEBUG = 0
7testDirName = "llvm-test"
8test = ["compile", "llc", "jit", "cbe"]
9exectime = [
10    "llc-time",
11    "jit-time",
12    "cbe-time",
13]
14comptime = ["llc", "jit-comptime", "compile"]
15
16(tp, exp) = ("compileTime_", "executeTime_")
17
18
19def parse(file):
20    f = open(file, "r")
21    d = f.read()
22
23    # Cleanup weird stuff
24    d = re.sub(r",\d+:\d", "", d)
25
26    r = re.findall(r"TEST-(PASS|FAIL|RESULT.*?):\s+(.*?)\s+(.*?)\r*\n", d)
27
28    test = {}
29    fname = ""
30    for t in r:
31        if DEBUG:
32            print(t)
33        if t[0] == "PASS" or t[0] == "FAIL":
34            tmp = t[2].split(testDirName)
35
36            if DEBUG:
37                print(tmp)
38
39            if len(tmp) == 2:
40                fname = tmp[1].strip("\r\n")
41            else:
42                fname = tmp[0].strip("\r\n")
43
44            if fname not in test:
45                test[fname] = {}
46
47            for k in test:
48                test[fname][k] = "NA"
49                test[fname][t[1]] = t[0]
50                if DEBUG:
51                    print(test[fname][t[1]])
52        else:
53            try:
54                n = t[0].split("RESULT-")[1]
55
56                if DEBUG:
57                    print(n)
58
59                if n == "llc" or n == "jit-comptime" or n == "compile":
60                    test[fname][tp + n] = float(t[2].split(" ")[2])
61                    if DEBUG:
62                        print(test[fname][tp + n])
63
64                elif n.endswith("-time"):
65                    test[fname][exp + n] = float(t[2].strip("\r\n"))
66                    if DEBUG:
67                        print(test[fname][exp + n])
68
69                else:
70                    print("ERROR!")
71                    sys.exit(1)
72
73            except:
74                continue
75
76    return test
77
78
79# Diff results and look for regressions.
80def diffResults(d_old, d_new):
81
82    for t in sorted(d_old.keys()):
83        if DEBUG:
84            print(t)
85
86        if t in d_new:
87
88            # Check if the test passed or failed.
89            for x in test:
90                if x in d_old[t]:
91                    if x in d_new[t]:
92                        if d_old[t][x] == "PASS":
93                            if d_new[t][x] != "PASS":
94                                print(t + " *** REGRESSION (" + x + ")\n")
95                        else:
96                            if d_new[t][x] == "PASS":
97                                print(t + " * NEW PASS (" + x + ")\n")
98
99                    else:
100                        print(t + "*** REGRESSION (" + x + ")\n")
101
102                # For execution time, if there is no result, its a fail.
103                for x in exectime:
104                    if tp + x in d_old[t]:
105                        if tp + x not in d_new[t]:
106                            print(t + " *** REGRESSION (" + tp + x + ")\n")
107
108                    else:
109                        if tp + x in d_new[t]:
110                            print(t + " * NEW PASS (" + tp + x + ")\n")
111
112                for x in comptime:
113                    if exp + x in d_old[t]:
114                        if exp + x not in d_new[t]:
115                            print(t + " *** REGRESSION (" + exp + x + ")\n")
116
117                    else:
118                        if exp + x in d_new[t]:
119                            print(t + " * NEW PASS (" + exp + x + ")\n")
120
121        else:
122            print(t + ": Removed from test-suite.\n")
123
124
125# Main
126if len(sys.argv) < 3:
127    print("Usage:", sys.argv[0], "<old log> <new log>")
128    sys.exit(-1)
129
130d_old = parse(sys.argv[1])
131d_new = parse(sys.argv[2])
132
133
134diffResults(d_old, d_new)
135