xref: /openbsd-src/gnu/llvm/lld/utils/benchmark.py (revision ece8a53058b183aa0a669e21c57225431117089d)
1*ece8a530Spatrick#!/usr/bin/env python
2*ece8a530Spatrick#
3*ece8a530Spatrick# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions.
4*ece8a530Spatrick# See https://llvm.org/LICENSE.txt for license information.
5*ece8a530Spatrick# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception
6*ece8a530Spatrick#
7*ece8a530Spatrick# ==------------------------------------------------------------------------==#
8*ece8a530Spatrick
9*ece8a530Spatrickimport os
10*ece8a530Spatrickimport glob
11*ece8a530Spatrickimport re
12*ece8a530Spatrickimport subprocess
13*ece8a530Spatrickimport json
14*ece8a530Spatrickimport datetime
15*ece8a530Spatrickimport argparse
16*ece8a530Spatricktry:
17*ece8a530Spatrick    from urllib.parse import urlencode
18*ece8a530Spatrick    from urllib.request import urlopen, Request
19*ece8a530Spatrickexcept ImportError:
20*ece8a530Spatrick    from urllib import urlencode
21*ece8a530Spatrick    from urllib2 import urlopen, Request
22*ece8a530Spatrick
23*ece8a530Spatrick
24*ece8a530Spatrickparser = argparse.ArgumentParser()
25*ece8a530Spatrickparser.add_argument('benchmark_directory')
26*ece8a530Spatrickparser.add_argument('--runs', type=int, default=10)
27*ece8a530Spatrickparser.add_argument('--wrapper', default='')
28*ece8a530Spatrickparser.add_argument('--machine', required=True)
29*ece8a530Spatrickparser.add_argument('--revision', required=True)
30*ece8a530Spatrickparser.add_argument('--threads', action='store_true')
31*ece8a530Spatrickparser.add_argument('--url', help='The lnt server url to send the results to',
32*ece8a530Spatrick                    default='http://localhost:8000/db_default/v4/link/submitRun')
33*ece8a530Spatrickargs = parser.parse_args()
34*ece8a530Spatrick
35*ece8a530Spatrickclass Bench:
36*ece8a530Spatrick    def __init__(self, directory, variant):
37*ece8a530Spatrick        self.directory = directory
38*ece8a530Spatrick        self.variant = variant
39*ece8a530Spatrick    def __str__(self):
40*ece8a530Spatrick        if not self.variant:
41*ece8a530Spatrick            return self.directory
42*ece8a530Spatrick        return '%s-%s' % (self.directory, self.variant)
43*ece8a530Spatrick
44*ece8a530Spatrickdef getBenchmarks():
45*ece8a530Spatrick    ret = []
46*ece8a530Spatrick    for i in glob.glob('*/response*.txt'):
47*ece8a530Spatrick        m = re.match('response-(.*)\.txt', os.path.basename(i))
48*ece8a530Spatrick        variant = m.groups()[0] if m else None
49*ece8a530Spatrick        ret.append(Bench(os.path.dirname(i), variant))
50*ece8a530Spatrick    return ret
51*ece8a530Spatrick
52*ece8a530Spatrickdef parsePerfNum(num):
53*ece8a530Spatrick    num = num.replace(b',',b'')
54*ece8a530Spatrick    try:
55*ece8a530Spatrick        return int(num)
56*ece8a530Spatrick    except ValueError:
57*ece8a530Spatrick        return float(num)
58*ece8a530Spatrick
59*ece8a530Spatrickdef parsePerfLine(line):
60*ece8a530Spatrick    ret = {}
61*ece8a530Spatrick    line = line.split(b'#')[0].strip()
62*ece8a530Spatrick    if len(line) != 0:
63*ece8a530Spatrick        p = line.split()
64*ece8a530Spatrick        ret[p[1].strip().decode('ascii')] = parsePerfNum(p[0])
65*ece8a530Spatrick    return ret
66*ece8a530Spatrick
67*ece8a530Spatrickdef parsePerf(output):
68*ece8a530Spatrick    ret = {}
69*ece8a530Spatrick    lines = [x.strip() for x in output.split(b'\n')]
70*ece8a530Spatrick
71*ece8a530Spatrick    seconds = [x for x in lines if b'seconds time elapsed' in x][0]
72*ece8a530Spatrick    seconds = seconds.strip().split()[0].strip()
73*ece8a530Spatrick    ret['seconds-elapsed'] = parsePerfNum(seconds)
74*ece8a530Spatrick
75*ece8a530Spatrick    measurement_lines = [x for x in lines if b'#' in x]
76*ece8a530Spatrick    for l in measurement_lines:
77*ece8a530Spatrick        ret.update(parsePerfLine(l))
78*ece8a530Spatrick    return ret
79*ece8a530Spatrick
80*ece8a530Spatrickdef run(cmd):
81*ece8a530Spatrick    try:
82*ece8a530Spatrick        return subprocess.check_output(cmd, stderr=subprocess.STDOUT)
83*ece8a530Spatrick    except subprocess.CalledProcessError as e:
84*ece8a530Spatrick        print(e.output)
85*ece8a530Spatrick        raise e
86*ece8a530Spatrick
87*ece8a530Spatrickdef combinePerfRun(acc, d):
88*ece8a530Spatrick    for k,v in d.items():
89*ece8a530Spatrick        a = acc.get(k, [])
90*ece8a530Spatrick        a.append(v)
91*ece8a530Spatrick        acc[k] = a
92*ece8a530Spatrick
93*ece8a530Spatrickdef perf(cmd):
94*ece8a530Spatrick    # Discard the first run to warm up any system cache.
95*ece8a530Spatrick    run(cmd)
96*ece8a530Spatrick
97*ece8a530Spatrick    ret = {}
98*ece8a530Spatrick    wrapper_args = [x for x in args.wrapper.split(',') if x]
99*ece8a530Spatrick    for i in range(args.runs):
100*ece8a530Spatrick        os.unlink('t')
101*ece8a530Spatrick        out = run(wrapper_args + ['perf', 'stat'] + cmd)
102*ece8a530Spatrick        r = parsePerf(out)
103*ece8a530Spatrick        combinePerfRun(ret, r)
104*ece8a530Spatrick    os.unlink('t')
105*ece8a530Spatrick    return ret
106*ece8a530Spatrick
107*ece8a530Spatrickdef runBench(bench):
108*ece8a530Spatrick    thread_arg = [] if args.threads else ['--no-threads']
109*ece8a530Spatrick    os.chdir(bench.directory)
110*ece8a530Spatrick    suffix = '-%s' % bench.variant if bench.variant else ''
111*ece8a530Spatrick    response = 'response' + suffix + '.txt'
112*ece8a530Spatrick    ret = perf(['../ld.lld', '@' + response, '-o', 't'] + thread_arg)
113*ece8a530Spatrick    ret['name'] = str(bench)
114*ece8a530Spatrick    os.chdir('..')
115*ece8a530Spatrick    return ret
116*ece8a530Spatrick
117*ece8a530Spatrickdef buildLntJson(benchmarks):
118*ece8a530Spatrick    start = datetime.datetime.utcnow().isoformat()
119*ece8a530Spatrick    tests = [runBench(b) for b in benchmarks]
120*ece8a530Spatrick    end = datetime.datetime.utcnow().isoformat()
121*ece8a530Spatrick    ret = {
122*ece8a530Spatrick        'format_version' : 2,
123*ece8a530Spatrick        'machine' : { 'name' : args.machine },
124*ece8a530Spatrick        'run' : {
125*ece8a530Spatrick            'end_time' : start,
126*ece8a530Spatrick            'start_time' : end,
127*ece8a530Spatrick            'llvm_project_revision': args.revision
128*ece8a530Spatrick        },
129*ece8a530Spatrick        'tests' : tests
130*ece8a530Spatrick    }
131*ece8a530Spatrick    return json.dumps(ret, sort_keys=True, indent=4)
132*ece8a530Spatrick
133*ece8a530Spatrickdef submitToServer(data):
134*ece8a530Spatrick    data2 = urlencode({ 'input_data' : data }).encode('ascii')
135*ece8a530Spatrick    urlopen(Request(args.url, data2))
136*ece8a530Spatrick
137*ece8a530Spatrickos.chdir(args.benchmark_directory)
138*ece8a530Spatrickdata = buildLntJson(getBenchmarks())
139*ece8a530SpatricksubmitToServer(data)
140