1# DExTer : Debugging Experience Tester 2# ~~~~~~ ~ ~~ ~ ~~ 3# 4# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 5# See https://llvm.org/LICENSE.txt for license information. 6# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 7"""Clang opt-bisect tool.""" 8 9from collections import defaultdict 10import os 11import csv 12import re 13import pickle 14 15from dex.command.ParseCommand import get_command_infos 16from dex.debugger.Debuggers import run_debugger_subprocess 17from dex.debugger.DebuggerControllers.DefaultController import DefaultController 18from dex.dextIR.DextIR import DextIR 19from dex.heuristic import Heuristic 20from dex.tools import TestToolBase 21from dex.utils.Exceptions import DebuggerException, Error 22from dex.utils.Exceptions import BuildScriptException, HeuristicException 23from dex.utils.PrettyOutputBase import Stream 24from dex.utils.ReturnCode import ReturnCode 25 26 27class BisectPass(object): 28 def __init__(self, no, description, description_no_loc): 29 self.no = no 30 self.description = description 31 self.description_no_loc = description_no_loc 32 33 self.penalty = 0 34 self.differences = [] 35 36 37class Tool(TestToolBase): 38 """Use the LLVM "-opt-bisect-limit=<n>" flag to get information on the 39 contribution of each LLVM pass to the overall DExTer score when using 40 clang. 41 42 Clang is run multiple times, with an increasing value of n, measuring the 43 debugging experience at each value. 44 """ 45 46 _re_running_pass = re.compile(r"^BISECT\: running pass \((\d+)\) (.+?)( \(.+\))?$") 47 48 def __init__(self, *args, **kwargs): 49 super(Tool, self).__init__(*args, **kwargs) 50 self._all_bisect_pass_summary = defaultdict(list) 51 52 @property 53 def name(self): 54 return "DExTer clang opt bisect" 55 56 def _get_bisect_limits(self): 57 options = self.context.options 58 59 max_limit = 999999 60 limits = [max_limit for _ in options.source_files] 61 all_passes = [ 62 l 63 for l in self._clang_opt_bisect_build(limits)[1].splitlines() 64 if l.startswith("BISECT: running pass (") 65 ] 66 67 results = [] 68 for i, pass_ in enumerate(all_passes[1:]): 69 if pass_.startswith("BISECT: running pass (1)"): 70 results.append(all_passes[i]) 71 results.append(all_passes[-1]) 72 73 assert len(results) == len(options.source_files), ( 74 results, 75 options.source_files, 76 ) 77 78 limits = [int(Tool._re_running_pass.match(r).group(1)) for r in results] 79 80 return limits 81 82 def handle_options(self, defaults): 83 options = self.context.options 84 if "clang" not in options.builder.lower(): 85 raise Error( 86 "--builder %s is not supported by the clang-opt-bisect tool - only 'clang' is " 87 "supported " % options.builder 88 ) 89 super(Tool, self).handle_options(defaults) 90 91 def _init_debugger_controller(self): 92 step_collection = DextIR( 93 executable_path=self.context.options.executable, 94 source_paths=self.context.options.source_files, 95 dexter_version=self.context.version, 96 ) 97 98 step_collection.commands, new_source_files = get_command_infos( 99 self.context.options.source_files, self.context.options.source_root_dir 100 ) 101 self.context.options.source_files.extend(list(new_source_files)) 102 103 debugger_controller = DefaultController(self.context, step_collection) 104 return debugger_controller 105 106 def _run_test(self, test_name): # noqa 107 options = self.context.options 108 109 per_pass_score = [] 110 current_bisect_pass_summary = defaultdict(list) 111 112 max_limits = self._get_bisect_limits() 113 overall_limit = sum(max_limits) 114 prev_score = 1.0 115 prev_steps_str = None 116 117 for current_limit in range(overall_limit + 1): 118 # Take the overall limit number and split it across buckets for 119 # each source file. 120 limit_remaining = current_limit 121 file_limits = [0] * len(max_limits) 122 for i, max_limit in enumerate(max_limits): 123 if limit_remaining < max_limit: 124 file_limits[i] += limit_remaining 125 break 126 else: 127 file_limits[i] = max_limit 128 limit_remaining -= file_limits[i] 129 130 f = [l for l in file_limits if l] 131 current_file_index = len(f) - 1 if f else 0 132 133 _, err, builderIR = self._clang_opt_bisect_build(file_limits) 134 err_lines = err.splitlines() 135 # Find the last line that specified a running pass. 136 for l in err_lines[::-1]: 137 match = Tool._re_running_pass.match(l) 138 if match: 139 pass_info = match.groups() 140 break 141 else: 142 pass_info = (0, None, None) 143 144 try: 145 debugger_controller = self._init_debugger_controller() 146 debugger_controller = run_debugger_subprocess( 147 debugger_controller, self.context.working_directory.path 148 ) 149 steps = debugger_controller.step_collection 150 except DebuggerException: 151 steps = DextIR( 152 executable_path=self.context.options.executable, 153 source_paths=self.context.options.source_files, 154 dexter_version=self.context.version, 155 ) 156 157 steps.builder = builderIR 158 159 try: 160 heuristic = Heuristic(self.context, steps) 161 except HeuristicException as e: 162 raise Error(e) 163 164 score_difference = heuristic.score - prev_score 165 prev_score = heuristic.score 166 167 isnan = heuristic.score != heuristic.score 168 if isnan or score_difference < 0: 169 color1 = "r" 170 color2 = "r" 171 elif score_difference > 0: 172 color1 = "g" 173 color2 = "g" 174 else: 175 color1 = "y" 176 color2 = "d" 177 178 summary = '<{}>running pass {}/{} on "{}"'.format( 179 color2, pass_info[0], max_limits[current_file_index], test_name 180 ) 181 if len(options.source_files) > 1: 182 summary += " [{}/{}]".format(current_limit, overall_limit) 183 184 pass_text = "".join(p for p in pass_info[1:] if p) 185 summary += ": {} <{}>{:+.4f}</> <{}>{}</></>\n".format( 186 heuristic.summary_string, color1, score_difference, color2, pass_text 187 ) 188 189 self.context.o.auto(summary) 190 191 heuristic_verbose_output = heuristic.verbose_output 192 193 if options.verbose: 194 self.context.o.auto(heuristic_verbose_output) 195 196 steps_str = str(steps) 197 steps_changed = steps_str != prev_steps_str 198 prev_steps_str = steps_str 199 200 # If a results directory has been specified and this is the first 201 # pass or something has changed, write a text file containing 202 # verbose information on the current status. 203 if options.results_directory and ( 204 current_limit == 0 or score_difference or steps_changed 205 ): 206 file_name = "-".join( 207 str(s) 208 for s in [ 209 "status", 210 test_name, 211 "{{:0>{}}}".format(len(str(overall_limit))).format( 212 current_limit 213 ), 214 "{:.4f}".format(heuristic.score).replace(".", "_"), 215 pass_info[1], 216 ] 217 if s is not None 218 ) 219 220 file_name = ( 221 "".join(c for c in file_name if c.isalnum() or c in "()-_./ ") 222 .strip() 223 .replace(" ", "_") 224 .replace("/", "_") 225 ) 226 227 output_text_path = os.path.join( 228 options.results_directory, "{}.txt".format(file_name) 229 ) 230 with open(output_text_path, "w") as fp: 231 self.context.o.auto(summary + "\n", stream=Stream(fp)) 232 self.context.o.auto(str(steps) + "\n", stream=Stream(fp)) 233 self.context.o.auto( 234 heuristic_verbose_output + "\n", stream=Stream(fp) 235 ) 236 237 output_dextIR_path = os.path.join( 238 options.results_directory, "{}.dextIR".format(file_name) 239 ) 240 with open(output_dextIR_path, "wb") as fp: 241 pickle.dump(steps, fp, protocol=pickle.HIGHEST_PROTOCOL) 242 243 per_pass_score.append((test_name, pass_text, heuristic.score)) 244 245 if pass_info[1]: 246 self._all_bisect_pass_summary[pass_info[1]].append(score_difference) 247 248 current_bisect_pass_summary[pass_info[1]].append(score_difference) 249 250 if options.results_directory: 251 per_pass_score_path = os.path.join( 252 options.results_directory, "{}-per_pass_score.csv".format(test_name) 253 ) 254 255 with open(per_pass_score_path, mode="w", newline="") as fp: 256 writer = csv.writer(fp, delimiter=",") 257 writer.writerow(["Source File", "Pass", "Score"]) 258 259 for path, pass_, score in per_pass_score: 260 writer.writerow([path, pass_, score]) 261 self.context.o.blue('wrote "{}"\n'.format(per_pass_score_path)) 262 263 pass_summary_path = os.path.join( 264 options.results_directory, "{}-pass-summary.csv".format(test_name) 265 ) 266 267 self._write_pass_summary(pass_summary_path, current_bisect_pass_summary) 268 269 def _handle_results(self) -> ReturnCode: 270 options = self.context.options 271 if options.results_directory: 272 pass_summary_path = os.path.join( 273 options.results_directory, "overall-pass-summary.csv" 274 ) 275 276 self._write_pass_summary(pass_summary_path, self._all_bisect_pass_summary) 277 return ReturnCode.OK 278 279 def _clang_opt_bisect_build(self, opt_bisect_limits): 280 options = self.context.options 281 compiler_options = [ 282 "{} -mllvm -opt-bisect-limit={}".format(options.cflags, opt_bisect_limit) 283 for opt_bisect_limit in opt_bisect_limits 284 ] 285 linker_options = options.ldflags 286 287 try: 288 return run_external_build_script( 289 self.context, 290 source_files=options.source_files, 291 compiler_options=compiler_options, 292 linker_options=linker_options, 293 script_path=self.build_script, 294 executable_file=options.executable, 295 ) 296 except BuildScriptException as e: 297 raise Error(e) 298 299 def _write_pass_summary(self, path, pass_summary): 300 # Get a list of tuples. 301 pass_summary_list = list(pass_summary.items()) 302 303 for i, item in enumerate(pass_summary_list): 304 # Add elems for the sum, min, and max of the values, as well as 305 # 'interestingness' which is whether any of these values are 306 # non-zero. 307 pass_summary_list[i] += ( 308 sum(item[1]), 309 min(item[1]), 310 max(item[1]), 311 any(item[1]), 312 ) 313 314 # Split the pass name into the basic name and kind. 315 pass_summary_list[i] += tuple(item[0].rsplit(" on ", 1)) 316 317 # Sort the list by the following columns in order of precedence: 318 # - Is interesting (True first) 319 # - Sum (smallest first) 320 # - Number of times pass ran (largest first) 321 # - Kind (alphabetically) 322 # - Name (alphabetically) 323 pass_summary_list.sort( 324 key=lambda tup: (not tup[5], tup[2], -len(tup[1]), tup[7], tup[6]) 325 ) 326 327 with open(path, mode="w", newline="") as fp: 328 writer = csv.writer(fp, delimiter=",") 329 writer.writerow(["Pass", "Kind", "Sum", "Min", "Max", "Interesting"]) 330 331 for ( 332 _, 333 vals, 334 sum_, 335 min_, 336 max_, 337 interesting, 338 name, 339 kind, 340 ) in pass_summary_list: 341 writer.writerow([name, kind, sum_, min_, max_, interesting] + vals) 342 343 self.context.o.blue('wrote "{}"\n'.format(path)) 344