1*4d6fc14bSjoerg"""util.py - General utilities for running, loading, and processing benchmarks 2*4d6fc14bSjoerg""" 3*4d6fc14bSjoergimport json 4*4d6fc14bSjoergimport os 5*4d6fc14bSjoergimport tempfile 6*4d6fc14bSjoergimport subprocess 7*4d6fc14bSjoergimport sys 8*4d6fc14bSjoerg 9*4d6fc14bSjoerg# Input file type enumeration 10*4d6fc14bSjoergIT_Invalid = 0 11*4d6fc14bSjoergIT_JSON = 1 12*4d6fc14bSjoergIT_Executable = 2 13*4d6fc14bSjoerg 14*4d6fc14bSjoerg_num_magic_bytes = 2 if sys.platform.startswith('win') else 4 15*4d6fc14bSjoerg 16*4d6fc14bSjoerg 17*4d6fc14bSjoergdef is_executable_file(filename): 18*4d6fc14bSjoerg """ 19*4d6fc14bSjoerg Return 'True' if 'filename' names a valid file which is likely 20*4d6fc14bSjoerg an executable. A file is considered an executable if it starts with the 21*4d6fc14bSjoerg magic bytes for a EXE, Mach O, or ELF file. 22*4d6fc14bSjoerg """ 23*4d6fc14bSjoerg if not os.path.isfile(filename): 24*4d6fc14bSjoerg return False 25*4d6fc14bSjoerg with open(filename, mode='rb') as f: 26*4d6fc14bSjoerg magic_bytes = f.read(_num_magic_bytes) 27*4d6fc14bSjoerg if sys.platform == 'darwin': 28*4d6fc14bSjoerg return magic_bytes in [ 29*4d6fc14bSjoerg b'\xfe\xed\xfa\xce', # MH_MAGIC 30*4d6fc14bSjoerg b'\xce\xfa\xed\xfe', # MH_CIGAM 31*4d6fc14bSjoerg b'\xfe\xed\xfa\xcf', # MH_MAGIC_64 32*4d6fc14bSjoerg b'\xcf\xfa\xed\xfe', # MH_CIGAM_64 33*4d6fc14bSjoerg b'\xca\xfe\xba\xbe', # FAT_MAGIC 34*4d6fc14bSjoerg b'\xbe\xba\xfe\xca' # FAT_CIGAM 35*4d6fc14bSjoerg ] 36*4d6fc14bSjoerg elif sys.platform.startswith('win'): 37*4d6fc14bSjoerg return magic_bytes == b'MZ' 38*4d6fc14bSjoerg else: 39*4d6fc14bSjoerg return magic_bytes == b'\x7FELF' 40*4d6fc14bSjoerg 41*4d6fc14bSjoerg 42*4d6fc14bSjoergdef is_json_file(filename): 43*4d6fc14bSjoerg """ 44*4d6fc14bSjoerg Returns 'True' if 'filename' names a valid JSON output file. 45*4d6fc14bSjoerg 'False' otherwise. 46*4d6fc14bSjoerg """ 47*4d6fc14bSjoerg try: 48*4d6fc14bSjoerg with open(filename, 'r') as f: 49*4d6fc14bSjoerg json.load(f) 50*4d6fc14bSjoerg return True 51*4d6fc14bSjoerg except BaseException: 52*4d6fc14bSjoerg pass 53*4d6fc14bSjoerg return False 54*4d6fc14bSjoerg 55*4d6fc14bSjoerg 56*4d6fc14bSjoergdef classify_input_file(filename): 57*4d6fc14bSjoerg """ 58*4d6fc14bSjoerg Return a tuple (type, msg) where 'type' specifies the classified type 59*4d6fc14bSjoerg of 'filename'. If 'type' is 'IT_Invalid' then 'msg' is a human readable 60*4d6fc14bSjoerg string represeting the error. 61*4d6fc14bSjoerg """ 62*4d6fc14bSjoerg ftype = IT_Invalid 63*4d6fc14bSjoerg err_msg = None 64*4d6fc14bSjoerg if not os.path.exists(filename): 65*4d6fc14bSjoerg err_msg = "'%s' does not exist" % filename 66*4d6fc14bSjoerg elif not os.path.isfile(filename): 67*4d6fc14bSjoerg err_msg = "'%s' does not name a file" % filename 68*4d6fc14bSjoerg elif is_executable_file(filename): 69*4d6fc14bSjoerg ftype = IT_Executable 70*4d6fc14bSjoerg elif is_json_file(filename): 71*4d6fc14bSjoerg ftype = IT_JSON 72*4d6fc14bSjoerg else: 73*4d6fc14bSjoerg err_msg = "'%s' does not name a valid benchmark executable or JSON file" % filename 74*4d6fc14bSjoerg return ftype, err_msg 75*4d6fc14bSjoerg 76*4d6fc14bSjoerg 77*4d6fc14bSjoergdef check_input_file(filename): 78*4d6fc14bSjoerg """ 79*4d6fc14bSjoerg Classify the file named by 'filename' and return the classification. 80*4d6fc14bSjoerg If the file is classified as 'IT_Invalid' print an error message and exit 81*4d6fc14bSjoerg the program. 82*4d6fc14bSjoerg """ 83*4d6fc14bSjoerg ftype, msg = classify_input_file(filename) 84*4d6fc14bSjoerg if ftype == IT_Invalid: 85*4d6fc14bSjoerg print("Invalid input file: %s" % msg) 86*4d6fc14bSjoerg sys.exit(1) 87*4d6fc14bSjoerg return ftype 88*4d6fc14bSjoerg 89*4d6fc14bSjoerg 90*4d6fc14bSjoergdef find_benchmark_flag(prefix, benchmark_flags): 91*4d6fc14bSjoerg """ 92*4d6fc14bSjoerg Search the specified list of flags for a flag matching `<prefix><arg>` and 93*4d6fc14bSjoerg if it is found return the arg it specifies. If specified more than once the 94*4d6fc14bSjoerg last value is returned. If the flag is not found None is returned. 95*4d6fc14bSjoerg """ 96*4d6fc14bSjoerg assert prefix.startswith('--') and prefix.endswith('=') 97*4d6fc14bSjoerg result = None 98*4d6fc14bSjoerg for f in benchmark_flags: 99*4d6fc14bSjoerg if f.startswith(prefix): 100*4d6fc14bSjoerg result = f[len(prefix):] 101*4d6fc14bSjoerg return result 102*4d6fc14bSjoerg 103*4d6fc14bSjoerg 104*4d6fc14bSjoergdef remove_benchmark_flags(prefix, benchmark_flags): 105*4d6fc14bSjoerg """ 106*4d6fc14bSjoerg Return a new list containing the specified benchmark_flags except those 107*4d6fc14bSjoerg with the specified prefix. 108*4d6fc14bSjoerg """ 109*4d6fc14bSjoerg assert prefix.startswith('--') and prefix.endswith('=') 110*4d6fc14bSjoerg return [f for f in benchmark_flags if not f.startswith(prefix)] 111*4d6fc14bSjoerg 112*4d6fc14bSjoerg 113*4d6fc14bSjoergdef load_benchmark_results(fname): 114*4d6fc14bSjoerg """ 115*4d6fc14bSjoerg Read benchmark output from a file and return the JSON object. 116*4d6fc14bSjoerg REQUIRES: 'fname' names a file containing JSON benchmark output. 117*4d6fc14bSjoerg """ 118*4d6fc14bSjoerg with open(fname, 'r') as f: 119*4d6fc14bSjoerg return json.load(f) 120*4d6fc14bSjoerg 121*4d6fc14bSjoerg 122*4d6fc14bSjoergdef run_benchmark(exe_name, benchmark_flags): 123*4d6fc14bSjoerg """ 124*4d6fc14bSjoerg Run a benchmark specified by 'exe_name' with the specified 125*4d6fc14bSjoerg 'benchmark_flags'. The benchmark is run directly as a subprocess to preserve 126*4d6fc14bSjoerg real time console output. 127*4d6fc14bSjoerg RETURNS: A JSON object representing the benchmark output 128*4d6fc14bSjoerg """ 129*4d6fc14bSjoerg output_name = find_benchmark_flag('--benchmark_out=', 130*4d6fc14bSjoerg benchmark_flags) 131*4d6fc14bSjoerg is_temp_output = False 132*4d6fc14bSjoerg if output_name is None: 133*4d6fc14bSjoerg is_temp_output = True 134*4d6fc14bSjoerg thandle, output_name = tempfile.mkstemp() 135*4d6fc14bSjoerg os.close(thandle) 136*4d6fc14bSjoerg benchmark_flags = list(benchmark_flags) + \ 137*4d6fc14bSjoerg ['--benchmark_out=%s' % output_name] 138*4d6fc14bSjoerg 139*4d6fc14bSjoerg cmd = [exe_name] + benchmark_flags 140*4d6fc14bSjoerg print("RUNNING: %s" % ' '.join(cmd)) 141*4d6fc14bSjoerg exitCode = subprocess.call(cmd) 142*4d6fc14bSjoerg if exitCode != 0: 143*4d6fc14bSjoerg print('TEST FAILED...') 144*4d6fc14bSjoerg sys.exit(exitCode) 145*4d6fc14bSjoerg json_res = load_benchmark_results(output_name) 146*4d6fc14bSjoerg if is_temp_output: 147*4d6fc14bSjoerg os.unlink(output_name) 148*4d6fc14bSjoerg return json_res 149*4d6fc14bSjoerg 150*4d6fc14bSjoerg 151*4d6fc14bSjoergdef run_or_load_benchmark(filename, benchmark_flags): 152*4d6fc14bSjoerg """ 153*4d6fc14bSjoerg Get the results for a specified benchmark. If 'filename' specifies 154*4d6fc14bSjoerg an executable benchmark then the results are generated by running the 155*4d6fc14bSjoerg benchmark. Otherwise 'filename' must name a valid JSON output file, 156*4d6fc14bSjoerg which is loaded and the result returned. 157*4d6fc14bSjoerg """ 158*4d6fc14bSjoerg ftype = check_input_file(filename) 159*4d6fc14bSjoerg if ftype == IT_JSON: 160*4d6fc14bSjoerg return load_benchmark_results(filename) 161*4d6fc14bSjoerg elif ftype == IT_Executable: 162*4d6fc14bSjoerg return run_benchmark(filename, benchmark_flags) 163*4d6fc14bSjoerg else: 164*4d6fc14bSjoerg assert False # This branch is unreachable 165