1#!/usr/bin/env python3 2import argparse 3import csv 4import re 5import sys 6import os 7from statistics import geometric_mean 8 9TIMING_LOG_RE = re.compile(r"(.*)/(.*).tmp(.*)") 10 11 12def main(): 13 parser = argparse.ArgumentParser( 14 description="BOLT NFC stat parser", 15 formatter_class=argparse.ArgumentDefaultsHelpFormatter, 16 ) 17 parser.add_argument( 18 "input", nargs="+", help="timing.log files produced by llvm-bolt-wrapper" 19 ) 20 parser.add_argument( 21 "--check_longer_than", 22 default=2, 23 type=float, 24 help="Only warn on tests longer than X seconds for at least one side", 25 ) 26 parser.add_argument( 27 "--threshold_single", 28 default=10, 29 type=float, 30 help="Threshold for a single test result swing, abs percent", 31 ), 32 parser.add_argument( 33 "--threshold_agg", 34 default=5, 35 type=float, 36 help="Threshold for geomean test results swing, abs percent", 37 ), 38 parser.add_argument("--verbose", "-v", action="store_true") 39 args = parser.parse_args() 40 41 def fmt_delta(value, exc_threshold, above_bound=True): 42 formatted_value = format(value, "+.2%") 43 if not above_bound: 44 formatted_value += "?" 45 elif exc_threshold and sys.stdout.isatty(): # terminal supports colors 46 return f"\033[1m{formatted_value}\033[0m" 47 return formatted_value 48 49 # Ratios for geomean computation 50 time_ratios = [] 51 mem_ratios = [] 52 # Whether any test exceeds the single test threshold (mem or time) 53 threshold_single = False 54 # Whether geomean exceeds aggregate test threshold (mem or time) 55 threshold_agg = False 56 57 if args.verbose: 58 print(f"# Individual test threshold: +-{args.threshold_single}%") 59 print(f"# Aggregate (geomean) test threshold: +-{args.threshold_agg}%") 60 print( 61 f"# Checking time swings for tests with runtime >" 62 f"{args.check_longer_than}s - otherwise marked as ?" 63 ) 64 print("Test/binary BOLT_wall_time BOLT_max_rss") 65 66 for input_file in args.input: 67 input_dir = os.path.dirname(input_file) 68 with open(input_file) as timing_file: 69 timing_reader = csv.reader(timing_file, delimiter=";") 70 for row in timing_reader: 71 test_name = row[0] 72 m = TIMING_LOG_RE.match(row[0]) 73 if m: 74 test_name = f"{input_dir}/{m.groups()[1]}/{m.groups()[2]}" 75 else: 76 # Prepend input dir to unparsed test name 77 test_name = input_dir + "#" + test_name 78 time_a, time_b = float(row[1]), float(row[3]) 79 mem_a, mem_b = int(row[2]), int(row[4]) 80 # Check if time is above bound for at least one side 81 time_above_bound = any( 82 [x > args.check_longer_than for x in [time_a, time_b]] 83 ) 84 # Compute B/A ratios (for % delta and geomean) 85 time_ratio = time_b / time_a if time_a else float('nan') 86 mem_ratio = mem_b / mem_a if mem_a else float('nan') 87 # Keep ratios for geomean 88 if time_above_bound and time_ratio > 0: # must be >0 for gmean 89 time_ratios += [time_ratio] 90 mem_ratios += [mem_ratio] 91 # Deltas: (B/A)-1 = (B-A)/A 92 time_delta = time_ratio - 1 93 mem_delta = mem_ratio - 1 94 # Check individual test results vs single test threshold 95 time_exc = ( 96 100.0 * abs(time_delta) > args.threshold_single and time_above_bound 97 ) 98 mem_exc = 100.0 * abs(mem_delta) > args.threshold_single 99 if time_exc or mem_exc: 100 threshold_single = True 101 # Print deltas with formatting in verbose mode 102 if args.verbose or time_exc or mem_exc: 103 print( 104 test_name, 105 fmt_delta(time_delta, time_exc, time_above_bound), 106 fmt_delta(mem_delta, mem_exc), 107 ) 108 109 time_gmean_delta = geometric_mean(time_ratios) - 1 110 mem_gmean_delta = geometric_mean(mem_ratios) - 1 111 time_agg_threshold = 100.0 * abs(time_gmean_delta) > args.threshold_agg 112 mem_agg_threshold = 100.0 * abs(mem_gmean_delta) > args.threshold_agg 113 if time_agg_threshold or mem_agg_threshold: 114 threshold_agg = True 115 if time_agg_threshold or mem_agg_threshold or args.verbose: 116 print( 117 "Geomean", 118 fmt_delta(time_gmean_delta, time_agg_threshold), 119 fmt_delta(mem_gmean_delta, mem_agg_threshold), 120 ) 121 exit(threshold_single or threshold_agg) 122 123 124if __name__ == "__main__": 125 main() 126