1*ece8a530Spatrick#!/usr/bin/env python 2*ece8a530Spatrick# 3*ece8a530Spatrick# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 4*ece8a530Spatrick# See https://llvm.org/LICENSE.txt for license information. 5*ece8a530Spatrick# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 6*ece8a530Spatrick# 7*ece8a530Spatrick# ==------------------------------------------------------------------------==# 8*ece8a530Spatrick 9*ece8a530Spatrickimport os 10*ece8a530Spatrickimport glob 11*ece8a530Spatrickimport re 12*ece8a530Spatrickimport subprocess 13*ece8a530Spatrickimport json 14*ece8a530Spatrickimport datetime 15*ece8a530Spatrickimport argparse 16*ece8a530Spatricktry: 17*ece8a530Spatrick from urllib.parse import urlencode 18*ece8a530Spatrick from urllib.request import urlopen, Request 19*ece8a530Spatrickexcept ImportError: 20*ece8a530Spatrick from urllib import urlencode 21*ece8a530Spatrick from urllib2 import urlopen, Request 22*ece8a530Spatrick 23*ece8a530Spatrick 24*ece8a530Spatrickparser = argparse.ArgumentParser() 25*ece8a530Spatrickparser.add_argument('benchmark_directory') 26*ece8a530Spatrickparser.add_argument('--runs', type=int, default=10) 27*ece8a530Spatrickparser.add_argument('--wrapper', default='') 28*ece8a530Spatrickparser.add_argument('--machine', required=True) 29*ece8a530Spatrickparser.add_argument('--revision', required=True) 30*ece8a530Spatrickparser.add_argument('--threads', action='store_true') 31*ece8a530Spatrickparser.add_argument('--url', help='The lnt server url to send the results to', 32*ece8a530Spatrick default='http://localhost:8000/db_default/v4/link/submitRun') 33*ece8a530Spatrickargs = parser.parse_args() 34*ece8a530Spatrick 35*ece8a530Spatrickclass Bench: 36*ece8a530Spatrick def __init__(self, directory, variant): 37*ece8a530Spatrick self.directory = directory 38*ece8a530Spatrick self.variant = variant 39*ece8a530Spatrick def __str__(self): 40*ece8a530Spatrick if not self.variant: 41*ece8a530Spatrick return self.directory 42*ece8a530Spatrick return '%s-%s' % (self.directory, self.variant) 43*ece8a530Spatrick 44*ece8a530Spatrickdef getBenchmarks(): 45*ece8a530Spatrick ret = [] 46*ece8a530Spatrick for i in glob.glob('*/response*.txt'): 47*ece8a530Spatrick m = re.match('response-(.*)\.txt', os.path.basename(i)) 48*ece8a530Spatrick variant = m.groups()[0] if m else None 49*ece8a530Spatrick ret.append(Bench(os.path.dirname(i), variant)) 50*ece8a530Spatrick return ret 51*ece8a530Spatrick 52*ece8a530Spatrickdef parsePerfNum(num): 53*ece8a530Spatrick num = num.replace(b',',b'') 54*ece8a530Spatrick try: 55*ece8a530Spatrick return int(num) 56*ece8a530Spatrick except ValueError: 57*ece8a530Spatrick return float(num) 58*ece8a530Spatrick 59*ece8a530Spatrickdef parsePerfLine(line): 60*ece8a530Spatrick ret = {} 61*ece8a530Spatrick line = line.split(b'#')[0].strip() 62*ece8a530Spatrick if len(line) != 0: 63*ece8a530Spatrick p = line.split() 64*ece8a530Spatrick ret[p[1].strip().decode('ascii')] = parsePerfNum(p[0]) 65*ece8a530Spatrick return ret 66*ece8a530Spatrick 67*ece8a530Spatrickdef parsePerf(output): 68*ece8a530Spatrick ret = {} 69*ece8a530Spatrick lines = [x.strip() for x in output.split(b'\n')] 70*ece8a530Spatrick 71*ece8a530Spatrick seconds = [x for x in lines if b'seconds time elapsed' in x][0] 72*ece8a530Spatrick seconds = seconds.strip().split()[0].strip() 73*ece8a530Spatrick ret['seconds-elapsed'] = parsePerfNum(seconds) 74*ece8a530Spatrick 75*ece8a530Spatrick measurement_lines = [x for x in lines if b'#' in x] 76*ece8a530Spatrick for l in measurement_lines: 77*ece8a530Spatrick ret.update(parsePerfLine(l)) 78*ece8a530Spatrick return ret 79*ece8a530Spatrick 80*ece8a530Spatrickdef run(cmd): 81*ece8a530Spatrick try: 82*ece8a530Spatrick return subprocess.check_output(cmd, stderr=subprocess.STDOUT) 83*ece8a530Spatrick except subprocess.CalledProcessError as e: 84*ece8a530Spatrick print(e.output) 85*ece8a530Spatrick raise e 86*ece8a530Spatrick 87*ece8a530Spatrickdef combinePerfRun(acc, d): 88*ece8a530Spatrick for k,v in d.items(): 89*ece8a530Spatrick a = acc.get(k, []) 90*ece8a530Spatrick a.append(v) 91*ece8a530Spatrick acc[k] = a 92*ece8a530Spatrick 93*ece8a530Spatrickdef perf(cmd): 94*ece8a530Spatrick # Discard the first run to warm up any system cache. 95*ece8a530Spatrick run(cmd) 96*ece8a530Spatrick 97*ece8a530Spatrick ret = {} 98*ece8a530Spatrick wrapper_args = [x for x in args.wrapper.split(',') if x] 99*ece8a530Spatrick for i in range(args.runs): 100*ece8a530Spatrick os.unlink('t') 101*ece8a530Spatrick out = run(wrapper_args + ['perf', 'stat'] + cmd) 102*ece8a530Spatrick r = parsePerf(out) 103*ece8a530Spatrick combinePerfRun(ret, r) 104*ece8a530Spatrick os.unlink('t') 105*ece8a530Spatrick return ret 106*ece8a530Spatrick 107*ece8a530Spatrickdef runBench(bench): 108*ece8a530Spatrick thread_arg = [] if args.threads else ['--no-threads'] 109*ece8a530Spatrick os.chdir(bench.directory) 110*ece8a530Spatrick suffix = '-%s' % bench.variant if bench.variant else '' 111*ece8a530Spatrick response = 'response' + suffix + '.txt' 112*ece8a530Spatrick ret = perf(['../ld.lld', '@' + response, '-o', 't'] + thread_arg) 113*ece8a530Spatrick ret['name'] = str(bench) 114*ece8a530Spatrick os.chdir('..') 115*ece8a530Spatrick return ret 116*ece8a530Spatrick 117*ece8a530Spatrickdef buildLntJson(benchmarks): 118*ece8a530Spatrick start = datetime.datetime.utcnow().isoformat() 119*ece8a530Spatrick tests = [runBench(b) for b in benchmarks] 120*ece8a530Spatrick end = datetime.datetime.utcnow().isoformat() 121*ece8a530Spatrick ret = { 122*ece8a530Spatrick 'format_version' : 2, 123*ece8a530Spatrick 'machine' : { 'name' : args.machine }, 124*ece8a530Spatrick 'run' : { 125*ece8a530Spatrick 'end_time' : start, 126*ece8a530Spatrick 'start_time' : end, 127*ece8a530Spatrick 'llvm_project_revision': args.revision 128*ece8a530Spatrick }, 129*ece8a530Spatrick 'tests' : tests 130*ece8a530Spatrick } 131*ece8a530Spatrick return json.dumps(ret, sort_keys=True, indent=4) 132*ece8a530Spatrick 133*ece8a530Spatrickdef submitToServer(data): 134*ece8a530Spatrick data2 = urlencode({ 'input_data' : data }).encode('ascii') 135*ece8a530Spatrick urlopen(Request(args.url, data2)) 136*ece8a530Spatrick 137*ece8a530Spatrickos.chdir(args.benchmark_directory) 138*ece8a530Spatrickdata = buildLntJson(getBenchmarks()) 139*ece8a530SpatricksubmitToServer(data) 140