xref: /netbsd-src/external/apache2/llvm/dist/libcxx/utils/google-benchmark/tools/gbench/util.py (revision 4d6fc14bc9b0c5bf3e30be318c143ee82cadd108)
1*4d6fc14bSjoerg"""util.py - General utilities for running, loading, and processing benchmarks
2*4d6fc14bSjoerg"""
3*4d6fc14bSjoergimport json
4*4d6fc14bSjoergimport os
5*4d6fc14bSjoergimport tempfile
6*4d6fc14bSjoergimport subprocess
7*4d6fc14bSjoergimport sys
8*4d6fc14bSjoerg
9*4d6fc14bSjoerg# Input file type enumeration
10*4d6fc14bSjoergIT_Invalid = 0
11*4d6fc14bSjoergIT_JSON = 1
12*4d6fc14bSjoergIT_Executable = 2
13*4d6fc14bSjoerg
14*4d6fc14bSjoerg_num_magic_bytes = 2 if sys.platform.startswith('win') else 4
15*4d6fc14bSjoerg
16*4d6fc14bSjoerg
17*4d6fc14bSjoergdef is_executable_file(filename):
18*4d6fc14bSjoerg    """
19*4d6fc14bSjoerg    Return 'True' if 'filename' names a valid file which is likely
20*4d6fc14bSjoerg    an executable. A file is considered an executable if it starts with the
21*4d6fc14bSjoerg    magic bytes for a EXE, Mach O, or ELF file.
22*4d6fc14bSjoerg    """
23*4d6fc14bSjoerg    if not os.path.isfile(filename):
24*4d6fc14bSjoerg        return False
25*4d6fc14bSjoerg    with open(filename, mode='rb') as f:
26*4d6fc14bSjoerg        magic_bytes = f.read(_num_magic_bytes)
27*4d6fc14bSjoerg    if sys.platform == 'darwin':
28*4d6fc14bSjoerg        return magic_bytes in [
29*4d6fc14bSjoerg            b'\xfe\xed\xfa\xce',  # MH_MAGIC
30*4d6fc14bSjoerg            b'\xce\xfa\xed\xfe',  # MH_CIGAM
31*4d6fc14bSjoerg            b'\xfe\xed\xfa\xcf',  # MH_MAGIC_64
32*4d6fc14bSjoerg            b'\xcf\xfa\xed\xfe',  # MH_CIGAM_64
33*4d6fc14bSjoerg            b'\xca\xfe\xba\xbe',  # FAT_MAGIC
34*4d6fc14bSjoerg            b'\xbe\xba\xfe\xca'   # FAT_CIGAM
35*4d6fc14bSjoerg        ]
36*4d6fc14bSjoerg    elif sys.platform.startswith('win'):
37*4d6fc14bSjoerg        return magic_bytes == b'MZ'
38*4d6fc14bSjoerg    else:
39*4d6fc14bSjoerg        return magic_bytes == b'\x7FELF'
40*4d6fc14bSjoerg
41*4d6fc14bSjoerg
42*4d6fc14bSjoergdef is_json_file(filename):
43*4d6fc14bSjoerg    """
44*4d6fc14bSjoerg    Returns 'True' if 'filename' names a valid JSON output file.
45*4d6fc14bSjoerg    'False' otherwise.
46*4d6fc14bSjoerg    """
47*4d6fc14bSjoerg    try:
48*4d6fc14bSjoerg        with open(filename, 'r') as f:
49*4d6fc14bSjoerg            json.load(f)
50*4d6fc14bSjoerg        return True
51*4d6fc14bSjoerg    except BaseException:
52*4d6fc14bSjoerg        pass
53*4d6fc14bSjoerg    return False
54*4d6fc14bSjoerg
55*4d6fc14bSjoerg
56*4d6fc14bSjoergdef classify_input_file(filename):
57*4d6fc14bSjoerg    """
58*4d6fc14bSjoerg    Return a tuple (type, msg) where 'type' specifies the classified type
59*4d6fc14bSjoerg    of 'filename'. If 'type' is 'IT_Invalid' then 'msg' is a human readable
60*4d6fc14bSjoerg    string represeting the error.
61*4d6fc14bSjoerg    """
62*4d6fc14bSjoerg    ftype = IT_Invalid
63*4d6fc14bSjoerg    err_msg = None
64*4d6fc14bSjoerg    if not os.path.exists(filename):
65*4d6fc14bSjoerg        err_msg = "'%s' does not exist" % filename
66*4d6fc14bSjoerg    elif not os.path.isfile(filename):
67*4d6fc14bSjoerg        err_msg = "'%s' does not name a file" % filename
68*4d6fc14bSjoerg    elif is_executable_file(filename):
69*4d6fc14bSjoerg        ftype = IT_Executable
70*4d6fc14bSjoerg    elif is_json_file(filename):
71*4d6fc14bSjoerg        ftype = IT_JSON
72*4d6fc14bSjoerg    else:
73*4d6fc14bSjoerg        err_msg = "'%s' does not name a valid benchmark executable or JSON file" % filename
74*4d6fc14bSjoerg    return ftype, err_msg
75*4d6fc14bSjoerg
76*4d6fc14bSjoerg
77*4d6fc14bSjoergdef check_input_file(filename):
78*4d6fc14bSjoerg    """
79*4d6fc14bSjoerg    Classify the file named by 'filename' and return the classification.
80*4d6fc14bSjoerg    If the file is classified as 'IT_Invalid' print an error message and exit
81*4d6fc14bSjoerg    the program.
82*4d6fc14bSjoerg    """
83*4d6fc14bSjoerg    ftype, msg = classify_input_file(filename)
84*4d6fc14bSjoerg    if ftype == IT_Invalid:
85*4d6fc14bSjoerg        print("Invalid input file: %s" % msg)
86*4d6fc14bSjoerg        sys.exit(1)
87*4d6fc14bSjoerg    return ftype
88*4d6fc14bSjoerg
89*4d6fc14bSjoerg
90*4d6fc14bSjoergdef find_benchmark_flag(prefix, benchmark_flags):
91*4d6fc14bSjoerg    """
92*4d6fc14bSjoerg    Search the specified list of flags for a flag matching `<prefix><arg>` and
93*4d6fc14bSjoerg    if it is found return the arg it specifies. If specified more than once the
94*4d6fc14bSjoerg    last value is returned. If the flag is not found None is returned.
95*4d6fc14bSjoerg    """
96*4d6fc14bSjoerg    assert prefix.startswith('--') and prefix.endswith('=')
97*4d6fc14bSjoerg    result = None
98*4d6fc14bSjoerg    for f in benchmark_flags:
99*4d6fc14bSjoerg        if f.startswith(prefix):
100*4d6fc14bSjoerg            result = f[len(prefix):]
101*4d6fc14bSjoerg    return result
102*4d6fc14bSjoerg
103*4d6fc14bSjoerg
104*4d6fc14bSjoergdef remove_benchmark_flags(prefix, benchmark_flags):
105*4d6fc14bSjoerg    """
106*4d6fc14bSjoerg    Return a new list containing the specified benchmark_flags except those
107*4d6fc14bSjoerg    with the specified prefix.
108*4d6fc14bSjoerg    """
109*4d6fc14bSjoerg    assert prefix.startswith('--') and prefix.endswith('=')
110*4d6fc14bSjoerg    return [f for f in benchmark_flags if not f.startswith(prefix)]
111*4d6fc14bSjoerg
112*4d6fc14bSjoerg
113*4d6fc14bSjoergdef load_benchmark_results(fname):
114*4d6fc14bSjoerg    """
115*4d6fc14bSjoerg    Read benchmark output from a file and return the JSON object.
116*4d6fc14bSjoerg    REQUIRES: 'fname' names a file containing JSON benchmark output.
117*4d6fc14bSjoerg    """
118*4d6fc14bSjoerg    with open(fname, 'r') as f:
119*4d6fc14bSjoerg        return json.load(f)
120*4d6fc14bSjoerg
121*4d6fc14bSjoerg
122*4d6fc14bSjoergdef run_benchmark(exe_name, benchmark_flags):
123*4d6fc14bSjoerg    """
124*4d6fc14bSjoerg    Run a benchmark specified by 'exe_name' with the specified
125*4d6fc14bSjoerg    'benchmark_flags'. The benchmark is run directly as a subprocess to preserve
126*4d6fc14bSjoerg    real time console output.
127*4d6fc14bSjoerg    RETURNS: A JSON object representing the benchmark output
128*4d6fc14bSjoerg    """
129*4d6fc14bSjoerg    output_name = find_benchmark_flag('--benchmark_out=',
130*4d6fc14bSjoerg                                      benchmark_flags)
131*4d6fc14bSjoerg    is_temp_output = False
132*4d6fc14bSjoerg    if output_name is None:
133*4d6fc14bSjoerg        is_temp_output = True
134*4d6fc14bSjoerg        thandle, output_name = tempfile.mkstemp()
135*4d6fc14bSjoerg        os.close(thandle)
136*4d6fc14bSjoerg        benchmark_flags = list(benchmark_flags) + \
137*4d6fc14bSjoerg            ['--benchmark_out=%s' % output_name]
138*4d6fc14bSjoerg
139*4d6fc14bSjoerg    cmd = [exe_name] + benchmark_flags
140*4d6fc14bSjoerg    print("RUNNING: %s" % ' '.join(cmd))
141*4d6fc14bSjoerg    exitCode = subprocess.call(cmd)
142*4d6fc14bSjoerg    if exitCode != 0:
143*4d6fc14bSjoerg        print('TEST FAILED...')
144*4d6fc14bSjoerg        sys.exit(exitCode)
145*4d6fc14bSjoerg    json_res = load_benchmark_results(output_name)
146*4d6fc14bSjoerg    if is_temp_output:
147*4d6fc14bSjoerg        os.unlink(output_name)
148*4d6fc14bSjoerg    return json_res
149*4d6fc14bSjoerg
150*4d6fc14bSjoerg
151*4d6fc14bSjoergdef run_or_load_benchmark(filename, benchmark_flags):
152*4d6fc14bSjoerg    """
153*4d6fc14bSjoerg    Get the results for a specified benchmark. If 'filename' specifies
154*4d6fc14bSjoerg    an executable benchmark then the results are generated by running the
155*4d6fc14bSjoerg    benchmark. Otherwise 'filename' must name a valid JSON output file,
156*4d6fc14bSjoerg    which is loaded and the result returned.
157*4d6fc14bSjoerg    """
158*4d6fc14bSjoerg    ftype = check_input_file(filename)
159*4d6fc14bSjoerg    if ftype == IT_JSON:
160*4d6fc14bSjoerg        return load_benchmark_results(filename)
161*4d6fc14bSjoerg    elif ftype == IT_Executable:
162*4d6fc14bSjoerg        return run_benchmark(filename, benchmark_flags)
163*4d6fc14bSjoerg    else:
164*4d6fc14bSjoerg        assert False  # This branch is unreachable
165