1d9cf8291SDaniel Hwang# -*- coding: utf-8 -*- 2d9cf8291SDaniel Hwang# Part of the LLVM Project, under the Apache License v2.0 with LLVM Exceptions. 3d9cf8291SDaniel Hwang# See https://llvm.org/LICENSE.txt for license information. 4d9cf8291SDaniel Hwang# SPDX-License-Identifier: Apache-2.0 WITH LLVM-exception 5d9cf8291SDaniel Hwang""" This module is responsible to generate 'index.html' for the report. 6d9cf8291SDaniel Hwang 7d9cf8291SDaniel HwangThe input for this step is the output directory, where individual reports 8d9cf8291SDaniel Hwangcould be found. It parses those reports and generates 'index.html'. """ 9d9cf8291SDaniel Hwang 10d9cf8291SDaniel Hwangimport re 11d9cf8291SDaniel Hwangimport os 12d9cf8291SDaniel Hwangimport os.path 13d9cf8291SDaniel Hwangimport sys 14d9cf8291SDaniel Hwangimport shutil 15d9cf8291SDaniel Hwangimport plistlib 16d9cf8291SDaniel Hwangimport glob 17d9cf8291SDaniel Hwangimport json 18d9cf8291SDaniel Hwangimport logging 19d9cf8291SDaniel Hwangimport datetime 20d9cf8291SDaniel Hwangfrom libscanbuild import duplicate_check 21d9cf8291SDaniel Hwangfrom libscanbuild.clang import get_version 22d9cf8291SDaniel Hwang 23*dd3c26a0STobias Hieta__all__ = ["document"] 24d9cf8291SDaniel Hwang 25d9cf8291SDaniel Hwang 26d9cf8291SDaniel Hwangdef document(args): 27d9cf8291SDaniel Hwang """Generates cover report and returns the number of bugs/crashes.""" 28d9cf8291SDaniel Hwang 29*dd3c26a0STobias Hieta html_reports_available = args.output_format in {"html", "plist-html", "sarif-html"} 30*dd3c26a0STobias Hieta sarif_reports_available = args.output_format in {"sarif", "sarif-html"} 31d9cf8291SDaniel Hwang 32*dd3c26a0STobias Hieta logging.debug("count crashes and bugs") 33d9cf8291SDaniel Hwang crash_count = sum(1 for _ in read_crashes(args.output)) 34d9cf8291SDaniel Hwang bug_counter = create_counters() 35d9cf8291SDaniel Hwang for bug in read_bugs(args.output, html_reports_available): 36d9cf8291SDaniel Hwang bug_counter(bug) 37d9cf8291SDaniel Hwang result = crash_count + bug_counter.total 38d9cf8291SDaniel Hwang 39d9cf8291SDaniel Hwang if html_reports_available and result: 40d9cf8291SDaniel Hwang use_cdb = os.path.exists(args.cdb) 41d9cf8291SDaniel Hwang 42*dd3c26a0STobias Hieta logging.debug("generate index.html file") 43d9cf8291SDaniel Hwang # common prefix for source files to have sorter path 44d9cf8291SDaniel Hwang prefix = commonprefix_from(args.cdb) if use_cdb else os.getcwd() 45d9cf8291SDaniel Hwang # assemble the cover from multiple fragments 46d9cf8291SDaniel Hwang fragments = [] 47d9cf8291SDaniel Hwang try: 48d9cf8291SDaniel Hwang if bug_counter.total: 49d9cf8291SDaniel Hwang fragments.append(bug_summary(args.output, bug_counter)) 50d9cf8291SDaniel Hwang fragments.append(bug_report(args.output, prefix)) 51d9cf8291SDaniel Hwang if crash_count: 52d9cf8291SDaniel Hwang fragments.append(crash_report(args.output, prefix)) 53d9cf8291SDaniel Hwang assemble_cover(args, prefix, fragments) 54d9cf8291SDaniel Hwang # copy additional files to the report 55d9cf8291SDaniel Hwang copy_resource_files(args.output) 56d9cf8291SDaniel Hwang if use_cdb: 57d9cf8291SDaniel Hwang shutil.copy(args.cdb, args.output) 58d9cf8291SDaniel Hwang finally: 59d9cf8291SDaniel Hwang for fragment in fragments: 60d9cf8291SDaniel Hwang os.remove(fragment) 61d9cf8291SDaniel Hwang 62d9cf8291SDaniel Hwang if sarif_reports_available: 63*dd3c26a0STobias Hieta logging.debug("merging sarif files") 64d9cf8291SDaniel Hwang merge_sarif_files(args.output) 65d9cf8291SDaniel Hwang 66d9cf8291SDaniel Hwang return result 67d9cf8291SDaniel Hwang 68d9cf8291SDaniel Hwang 69d9cf8291SDaniel Hwangdef assemble_cover(args, prefix, fragments): 70d9cf8291SDaniel Hwang """Put together the fragments into a final report.""" 71d9cf8291SDaniel Hwang 72d9cf8291SDaniel Hwang import getpass 73d9cf8291SDaniel Hwang import socket 74d9cf8291SDaniel Hwang 75d9cf8291SDaniel Hwang if args.html_title is None: 76*dd3c26a0STobias Hieta args.html_title = os.path.basename(prefix) + " - analyzer results" 77d9cf8291SDaniel Hwang 78*dd3c26a0STobias Hieta with open(os.path.join(args.output, "index.html"), "w") as handle: 79d9cf8291SDaniel Hwang indent = 0 80*dd3c26a0STobias Hieta handle.write( 81*dd3c26a0STobias Hieta reindent( 82*dd3c26a0STobias Hieta """ 83d9cf8291SDaniel Hwang |<!DOCTYPE html> 84d9cf8291SDaniel Hwang |<html> 85d9cf8291SDaniel Hwang | <head> 86d9cf8291SDaniel Hwang | <title>{html_title}</title> 87d9cf8291SDaniel Hwang | <link type="text/css" rel="stylesheet" href="scanview.css"/> 88d9cf8291SDaniel Hwang | <script type='text/javascript' src="sorttable.js"></script> 89d9cf8291SDaniel Hwang | <script type='text/javascript' src='selectable.js'></script> 90*dd3c26a0STobias Hieta | </head>""", 91*dd3c26a0STobias Hieta indent, 92*dd3c26a0STobias Hieta ).format(html_title=args.html_title) 93*dd3c26a0STobias Hieta ) 94*dd3c26a0STobias Hieta handle.write(comment("SUMMARYENDHEAD")) 95*dd3c26a0STobias Hieta handle.write( 96*dd3c26a0STobias Hieta reindent( 97*dd3c26a0STobias Hieta """ 98d9cf8291SDaniel Hwang | <body> 99d9cf8291SDaniel Hwang | <h1>{html_title}</h1> 100d9cf8291SDaniel Hwang | <table> 101d9cf8291SDaniel Hwang | <tr><th>User:</th><td>{user_name}@{host_name}</td></tr> 102d9cf8291SDaniel Hwang | <tr><th>Working Directory:</th><td>{current_dir}</td></tr> 103d9cf8291SDaniel Hwang | <tr><th>Command Line:</th><td>{cmd_args}</td></tr> 104d9cf8291SDaniel Hwang | <tr><th>Clang Version:</th><td>{clang_version}</td></tr> 105d9cf8291SDaniel Hwang | <tr><th>Date:</th><td>{date}</td></tr> 106*dd3c26a0STobias Hieta | </table>""", 107*dd3c26a0STobias Hieta indent, 108*dd3c26a0STobias Hieta ).format( 109*dd3c26a0STobias Hieta html_title=args.html_title, 110d9cf8291SDaniel Hwang user_name=getpass.getuser(), 111d9cf8291SDaniel Hwang host_name=socket.gethostname(), 112d9cf8291SDaniel Hwang current_dir=prefix, 113*dd3c26a0STobias Hieta cmd_args=" ".join(sys.argv), 114d9cf8291SDaniel Hwang clang_version=get_version(args.clang), 115*dd3c26a0STobias Hieta date=datetime.datetime.today().strftime("%c"), 116*dd3c26a0STobias Hieta ) 117*dd3c26a0STobias Hieta ) 118d9cf8291SDaniel Hwang for fragment in fragments: 119d9cf8291SDaniel Hwang # copy the content of fragments 120*dd3c26a0STobias Hieta with open(fragment, "r") as input_handle: 121d9cf8291SDaniel Hwang shutil.copyfileobj(input_handle, handle) 122*dd3c26a0STobias Hieta handle.write( 123*dd3c26a0STobias Hieta reindent( 124*dd3c26a0STobias Hieta """ 125d9cf8291SDaniel Hwang | </body> 126*dd3c26a0STobias Hieta |</html>""", 127*dd3c26a0STobias Hieta indent, 128*dd3c26a0STobias Hieta ) 129*dd3c26a0STobias Hieta ) 130d9cf8291SDaniel Hwang 131d9cf8291SDaniel Hwang 132d9cf8291SDaniel Hwangdef bug_summary(output_dir, bug_counter): 133d9cf8291SDaniel Hwang """Bug summary is a HTML table to give a better overview of the bugs.""" 134d9cf8291SDaniel Hwang 135*dd3c26a0STobias Hieta name = os.path.join(output_dir, "summary.html.fragment") 136*dd3c26a0STobias Hieta with open(name, "w") as handle: 137d9cf8291SDaniel Hwang indent = 4 138*dd3c26a0STobias Hieta handle.write( 139*dd3c26a0STobias Hieta reindent( 140*dd3c26a0STobias Hieta """ 141d9cf8291SDaniel Hwang |<h2>Bug Summary</h2> 142d9cf8291SDaniel Hwang |<table> 143d9cf8291SDaniel Hwang | <thead> 144d9cf8291SDaniel Hwang | <tr> 145d9cf8291SDaniel Hwang | <td>Bug Type</td> 146d9cf8291SDaniel Hwang | <td>Quantity</td> 147d9cf8291SDaniel Hwang | <td class="sorttable_nosort">Display?</td> 148d9cf8291SDaniel Hwang | </tr> 149d9cf8291SDaniel Hwang | </thead> 150*dd3c26a0STobias Hieta | <tbody>""", 151*dd3c26a0STobias Hieta indent, 152*dd3c26a0STobias Hieta ) 153*dd3c26a0STobias Hieta ) 154*dd3c26a0STobias Hieta handle.write( 155*dd3c26a0STobias Hieta reindent( 156*dd3c26a0STobias Hieta """ 157d9cf8291SDaniel Hwang | <tr style="font-weight:bold"> 158d9cf8291SDaniel Hwang | <td class="SUMM_DESC">All Bugs</td> 159d9cf8291SDaniel Hwang | <td class="Q">{0}</td> 160d9cf8291SDaniel Hwang | <td> 161d9cf8291SDaniel Hwang | <center> 162d9cf8291SDaniel Hwang | <input checked type="checkbox" id="AllBugsCheck" 163d9cf8291SDaniel Hwang | onClick="CopyCheckedStateToCheckButtons(this);"/> 164d9cf8291SDaniel Hwang | </center> 165d9cf8291SDaniel Hwang | </td> 166*dd3c26a0STobias Hieta | </tr>""", 167*dd3c26a0STobias Hieta indent, 168*dd3c26a0STobias Hieta ).format(bug_counter.total) 169*dd3c26a0STobias Hieta ) 170d9cf8291SDaniel Hwang for category, types in bug_counter.categories.items(): 171*dd3c26a0STobias Hieta handle.write( 172*dd3c26a0STobias Hieta reindent( 173*dd3c26a0STobias Hieta """ 174d9cf8291SDaniel Hwang | <tr> 175d9cf8291SDaniel Hwang | <th>{0}</th><th colspan=2></th> 176*dd3c26a0STobias Hieta | </tr>""", 177*dd3c26a0STobias Hieta indent, 178*dd3c26a0STobias Hieta ).format(category) 179*dd3c26a0STobias Hieta ) 180d9cf8291SDaniel Hwang for bug_type in types.values(): 181*dd3c26a0STobias Hieta handle.write( 182*dd3c26a0STobias Hieta reindent( 183*dd3c26a0STobias Hieta """ 184d9cf8291SDaniel Hwang | <tr> 185d9cf8291SDaniel Hwang | <td class="SUMM_DESC">{bug_type}</td> 186d9cf8291SDaniel Hwang | <td class="Q">{bug_count}</td> 187d9cf8291SDaniel Hwang | <td> 188d9cf8291SDaniel Hwang | <center> 189d9cf8291SDaniel Hwang | <input checked type="checkbox" 190d9cf8291SDaniel Hwang | onClick="ToggleDisplay(this,'{bug_type_class}');"/> 191d9cf8291SDaniel Hwang | </center> 192d9cf8291SDaniel Hwang | </td> 193*dd3c26a0STobias Hieta | </tr>""", 194*dd3c26a0STobias Hieta indent, 195*dd3c26a0STobias Hieta ).format(**bug_type) 196*dd3c26a0STobias Hieta ) 197*dd3c26a0STobias Hieta handle.write( 198*dd3c26a0STobias Hieta reindent( 199*dd3c26a0STobias Hieta """ 200d9cf8291SDaniel Hwang | </tbody> 201*dd3c26a0STobias Hieta |</table>""", 202*dd3c26a0STobias Hieta indent, 203*dd3c26a0STobias Hieta ) 204*dd3c26a0STobias Hieta ) 205*dd3c26a0STobias Hieta handle.write(comment("SUMMARYBUGEND")) 206d9cf8291SDaniel Hwang return name 207d9cf8291SDaniel Hwang 208d9cf8291SDaniel Hwang 209d9cf8291SDaniel Hwangdef bug_report(output_dir, prefix): 210d9cf8291SDaniel Hwang """Creates a fragment from the analyzer reports.""" 211d9cf8291SDaniel Hwang 212d9cf8291SDaniel Hwang pretty = prettify_bug(prefix, output_dir) 213d9cf8291SDaniel Hwang bugs = (pretty(bug) for bug in read_bugs(output_dir, True)) 214d9cf8291SDaniel Hwang 215*dd3c26a0STobias Hieta name = os.path.join(output_dir, "bugs.html.fragment") 216*dd3c26a0STobias Hieta with open(name, "w") as handle: 217d9cf8291SDaniel Hwang indent = 4 218*dd3c26a0STobias Hieta handle.write( 219*dd3c26a0STobias Hieta reindent( 220*dd3c26a0STobias Hieta """ 221d9cf8291SDaniel Hwang |<h2>Reports</h2> 222d9cf8291SDaniel Hwang |<table class="sortable" style="table-layout:automatic"> 223d9cf8291SDaniel Hwang | <thead> 224d9cf8291SDaniel Hwang | <tr> 225d9cf8291SDaniel Hwang | <td>Bug Group</td> 226d9cf8291SDaniel Hwang | <td class="sorttable_sorted"> 227d9cf8291SDaniel Hwang | Bug Type 228d9cf8291SDaniel Hwang | <span id="sorttable_sortfwdind"> ▾</span> 229d9cf8291SDaniel Hwang | </td> 230d9cf8291SDaniel Hwang | <td>File</td> 231d9cf8291SDaniel Hwang | <td>Function/Method</td> 232d9cf8291SDaniel Hwang | <td class="Q">Line</td> 233d9cf8291SDaniel Hwang | <td class="Q">Path Length</td> 234d9cf8291SDaniel Hwang | <td class="sorttable_nosort"></td> 235d9cf8291SDaniel Hwang | </tr> 236d9cf8291SDaniel Hwang | </thead> 237*dd3c26a0STobias Hieta | <tbody>""", 238*dd3c26a0STobias Hieta indent, 239*dd3c26a0STobias Hieta ) 240*dd3c26a0STobias Hieta ) 241*dd3c26a0STobias Hieta handle.write(comment("REPORTBUGCOL")) 242d9cf8291SDaniel Hwang for current in bugs: 243*dd3c26a0STobias Hieta handle.write( 244*dd3c26a0STobias Hieta reindent( 245*dd3c26a0STobias Hieta """ 246d9cf8291SDaniel Hwang | <tr class="{bug_type_class}"> 247d9cf8291SDaniel Hwang | <td class="DESC">{bug_category}</td> 248d9cf8291SDaniel Hwang | <td class="DESC">{bug_type}</td> 249d9cf8291SDaniel Hwang | <td>{bug_file}</td> 250d9cf8291SDaniel Hwang | <td class="DESC">{bug_function}</td> 251d9cf8291SDaniel Hwang | <td class="Q">{bug_line}</td> 252d9cf8291SDaniel Hwang | <td class="Q">{bug_path_length}</td> 253d9cf8291SDaniel Hwang | <td><a href="{report_file}#EndPath">View Report</a></td> 254*dd3c26a0STobias Hieta | </tr>""", 255*dd3c26a0STobias Hieta indent, 256*dd3c26a0STobias Hieta ).format(**current) 257*dd3c26a0STobias Hieta ) 258*dd3c26a0STobias Hieta handle.write(comment("REPORTBUG", {"id": current["report_file"]})) 259*dd3c26a0STobias Hieta handle.write( 260*dd3c26a0STobias Hieta reindent( 261*dd3c26a0STobias Hieta """ 262d9cf8291SDaniel Hwang | </tbody> 263*dd3c26a0STobias Hieta |</table>""", 264*dd3c26a0STobias Hieta indent, 265*dd3c26a0STobias Hieta ) 266*dd3c26a0STobias Hieta ) 267*dd3c26a0STobias Hieta handle.write(comment("REPORTBUGEND")) 268d9cf8291SDaniel Hwang return name 269d9cf8291SDaniel Hwang 270d9cf8291SDaniel Hwang 271d9cf8291SDaniel Hwangdef crash_report(output_dir, prefix): 272d9cf8291SDaniel Hwang """Creates a fragment from the compiler crashes.""" 273d9cf8291SDaniel Hwang 274d9cf8291SDaniel Hwang pretty = prettify_crash(prefix, output_dir) 275d9cf8291SDaniel Hwang crashes = (pretty(crash) for crash in read_crashes(output_dir)) 276d9cf8291SDaniel Hwang 277*dd3c26a0STobias Hieta name = os.path.join(output_dir, "crashes.html.fragment") 278*dd3c26a0STobias Hieta with open(name, "w") as handle: 279d9cf8291SDaniel Hwang indent = 4 280*dd3c26a0STobias Hieta handle.write( 281*dd3c26a0STobias Hieta reindent( 282*dd3c26a0STobias Hieta """ 283d9cf8291SDaniel Hwang |<h2>Analyzer Failures</h2> 284d9cf8291SDaniel Hwang |<p>The analyzer had problems processing the following files:</p> 285d9cf8291SDaniel Hwang |<table> 286d9cf8291SDaniel Hwang | <thead> 287d9cf8291SDaniel Hwang | <tr> 288d9cf8291SDaniel Hwang | <td>Problem</td> 289d9cf8291SDaniel Hwang | <td>Source File</td> 290d9cf8291SDaniel Hwang | <td>Preprocessed File</td> 291d9cf8291SDaniel Hwang | <td>STDERR Output</td> 292d9cf8291SDaniel Hwang | </tr> 293d9cf8291SDaniel Hwang | </thead> 294*dd3c26a0STobias Hieta | <tbody>""", 295*dd3c26a0STobias Hieta indent, 296*dd3c26a0STobias Hieta ) 297*dd3c26a0STobias Hieta ) 298d9cf8291SDaniel Hwang for current in crashes: 299*dd3c26a0STobias Hieta handle.write( 300*dd3c26a0STobias Hieta reindent( 301*dd3c26a0STobias Hieta """ 302d9cf8291SDaniel Hwang | <tr> 303d9cf8291SDaniel Hwang | <td>{problem}</td> 304d9cf8291SDaniel Hwang | <td>{source}</td> 305d9cf8291SDaniel Hwang | <td><a href="{file}">preprocessor output</a></td> 306d9cf8291SDaniel Hwang | <td><a href="{stderr}">analyzer std err</a></td> 307*dd3c26a0STobias Hieta | </tr>""", 308*dd3c26a0STobias Hieta indent, 309*dd3c26a0STobias Hieta ).format(**current) 310*dd3c26a0STobias Hieta ) 311*dd3c26a0STobias Hieta handle.write(comment("REPORTPROBLEM", current)) 312*dd3c26a0STobias Hieta handle.write( 313*dd3c26a0STobias Hieta reindent( 314*dd3c26a0STobias Hieta """ 315d9cf8291SDaniel Hwang | </tbody> 316*dd3c26a0STobias Hieta |</table>""", 317*dd3c26a0STobias Hieta indent, 318*dd3c26a0STobias Hieta ) 319*dd3c26a0STobias Hieta ) 320*dd3c26a0STobias Hieta handle.write(comment("REPORTCRASHES")) 321d9cf8291SDaniel Hwang return name 322d9cf8291SDaniel Hwang 323d9cf8291SDaniel Hwang 324d9cf8291SDaniel Hwangdef read_crashes(output_dir): 325d9cf8291SDaniel Hwang """Generate a unique sequence of crashes from given output directory.""" 326d9cf8291SDaniel Hwang 327*dd3c26a0STobias Hieta return ( 328*dd3c26a0STobias Hieta parse_crash(filename) 329*dd3c26a0STobias Hieta for filename in glob.iglob(os.path.join(output_dir, "failures", "*.info.txt")) 330*dd3c26a0STobias Hieta ) 331d9cf8291SDaniel Hwang 332d9cf8291SDaniel Hwang 333d9cf8291SDaniel Hwangdef read_bugs(output_dir, html): 334d9cf8291SDaniel Hwang # type: (str, bool) -> Generator[Dict[str, Any], None, None] 335d9cf8291SDaniel Hwang """Generate a unique sequence of bugs from given output directory. 336d9cf8291SDaniel Hwang 337d9cf8291SDaniel Hwang Duplicates can be in a project if the same module was compiled multiple 338d9cf8291SDaniel Hwang times with different compiler options. These would be better to show in 339d9cf8291SDaniel Hwang the final report (cover) only once.""" 340d9cf8291SDaniel Hwang 341d9cf8291SDaniel Hwang def empty(file_name): 342d9cf8291SDaniel Hwang return os.stat(file_name).st_size == 0 343d9cf8291SDaniel Hwang 344d9cf8291SDaniel Hwang duplicate = duplicate_check( 345*dd3c26a0STobias Hieta lambda bug: "{bug_line}.{bug_path_length}:{bug_file}".format(**bug) 346*dd3c26a0STobias Hieta ) 347d9cf8291SDaniel Hwang 348d9cf8291SDaniel Hwang # get the right parser for the job. 349d9cf8291SDaniel Hwang parser = parse_bug_html if html else parse_bug_plist 350d9cf8291SDaniel Hwang # get the input files, which are not empty. 351*dd3c26a0STobias Hieta pattern = os.path.join(output_dir, "*.html" if html else "*.plist") 352d9cf8291SDaniel Hwang bug_files = (file for file in glob.iglob(pattern) if not empty(file)) 353d9cf8291SDaniel Hwang 354d9cf8291SDaniel Hwang for bug_file in bug_files: 355d9cf8291SDaniel Hwang for bug in parser(bug_file): 356d9cf8291SDaniel Hwang if not duplicate(bug): 357d9cf8291SDaniel Hwang yield bug 358d9cf8291SDaniel Hwang 359*dd3c26a0STobias Hieta 360d9cf8291SDaniel Hwangdef merge_sarif_files(output_dir, sort_files=False): 361d9cf8291SDaniel Hwang """Reads and merges all .sarif files in the given output directory. 362d9cf8291SDaniel Hwang 363d9cf8291SDaniel Hwang Each sarif file in the output directory is understood as a single run 364d9cf8291SDaniel Hwang and thus appear separate in the top level runs array. This requires 365d9cf8291SDaniel Hwang modifying the run index of any embedded links in messages. 366d9cf8291SDaniel Hwang """ 367d9cf8291SDaniel Hwang 368d9cf8291SDaniel Hwang def empty(file_name): 369d9cf8291SDaniel Hwang return os.stat(file_name).st_size == 0 370d9cf8291SDaniel Hwang 371d9cf8291SDaniel Hwang def update_sarif_object(sarif_object, runs_count_offset): 372d9cf8291SDaniel Hwang """ 373d9cf8291SDaniel Hwang Given a SARIF object, checks its dictionary entries for a 'message' property. 374d9cf8291SDaniel Hwang If it exists, updates the message index of embedded links in the run index. 375d9cf8291SDaniel Hwang 376d9cf8291SDaniel Hwang Recursively looks through entries in the dictionary. 377d9cf8291SDaniel Hwang """ 378d9cf8291SDaniel Hwang if not isinstance(sarif_object, dict): 379d9cf8291SDaniel Hwang return sarif_object 380d9cf8291SDaniel Hwang 381*dd3c26a0STobias Hieta if "message" in sarif_object: 382*dd3c26a0STobias Hieta sarif_object["message"] = match_and_update_run( 383*dd3c26a0STobias Hieta sarif_object["message"], runs_count_offset 384*dd3c26a0STobias Hieta ) 385d9cf8291SDaniel Hwang 386d9cf8291SDaniel Hwang for key in sarif_object: 387d9cf8291SDaniel Hwang if isinstance(sarif_object[key], list): 388d9cf8291SDaniel Hwang # iterate through subobjects and update it. 389*dd3c26a0STobias Hieta arr = [ 390*dd3c26a0STobias Hieta update_sarif_object(entry, runs_count_offset) 391*dd3c26a0STobias Hieta for entry in sarif_object[key] 392*dd3c26a0STobias Hieta ] 393d9cf8291SDaniel Hwang sarif_object[key] = arr 394d9cf8291SDaniel Hwang elif isinstance(sarif_object[key], dict): 395*dd3c26a0STobias Hieta sarif_object[key] = update_sarif_object( 396*dd3c26a0STobias Hieta sarif_object[key], runs_count_offset 397*dd3c26a0STobias Hieta ) 398d9cf8291SDaniel Hwang else: 399d9cf8291SDaniel Hwang # do nothing 400d9cf8291SDaniel Hwang pass 401d9cf8291SDaniel Hwang 402d9cf8291SDaniel Hwang return sarif_object 403d9cf8291SDaniel Hwang 404d9cf8291SDaniel Hwang def match_and_update_run(message, runs_count_offset): 405d9cf8291SDaniel Hwang """ 406d9cf8291SDaniel Hwang Given a SARIF message object, checks if the text property contains an embedded link and 407d9cf8291SDaniel Hwang updates the run index if necessary. 408d9cf8291SDaniel Hwang """ 409*dd3c26a0STobias Hieta if "text" not in message: 410d9cf8291SDaniel Hwang return message 411d9cf8291SDaniel Hwang 412d9cf8291SDaniel Hwang # we only merge runs, so we only need to update the run index 413*dd3c26a0STobias Hieta pattern = re.compile(r"sarif:/runs/(\d+)") 414d9cf8291SDaniel Hwang 415*dd3c26a0STobias Hieta text = message["text"] 416d9cf8291SDaniel Hwang matches = re.finditer(pattern, text) 417d9cf8291SDaniel Hwang matches_list = list(matches) 418d9cf8291SDaniel Hwang 419d9cf8291SDaniel Hwang # update matches from right to left to make increasing character length (9->10) smoother 420d9cf8291SDaniel Hwang for idx in range(len(matches_list) - 1, -1, -1): 421d9cf8291SDaniel Hwang match = matches_list[idx] 422d9cf8291SDaniel Hwang new_run_count = str(runs_count_offset + int(match.group(1))) 423d9cf8291SDaniel Hwang text = text[0 : match.start(1)] + new_run_count + text[match.end(1) :] 424d9cf8291SDaniel Hwang 425*dd3c26a0STobias Hieta message["text"] = text 426d9cf8291SDaniel Hwang return message 427d9cf8291SDaniel Hwang 428*dd3c26a0STobias Hieta sarif_files = ( 429*dd3c26a0STobias Hieta file 430*dd3c26a0STobias Hieta for file in glob.iglob(os.path.join(output_dir, "*.sarif")) 431*dd3c26a0STobias Hieta if not empty(file) 432*dd3c26a0STobias Hieta ) 433d9cf8291SDaniel Hwang # exposed for testing since the order of files returned by glob is not guaranteed to be sorted 434d9cf8291SDaniel Hwang if sort_files: 435d9cf8291SDaniel Hwang sarif_files = list(sarif_files) 436d9cf8291SDaniel Hwang sarif_files.sort() 437d9cf8291SDaniel Hwang 438d9cf8291SDaniel Hwang runs_count = 0 439d9cf8291SDaniel Hwang merged = {} 440d9cf8291SDaniel Hwang for sarif_file in sarif_files: 441d9cf8291SDaniel Hwang with open(sarif_file) as fp: 442d9cf8291SDaniel Hwang sarif = json.load(fp) 443*dd3c26a0STobias Hieta if "runs" not in sarif: 444d9cf8291SDaniel Hwang continue 445d9cf8291SDaniel Hwang 446d9cf8291SDaniel Hwang # start with the first file 447d9cf8291SDaniel Hwang if not merged: 448d9cf8291SDaniel Hwang merged = sarif 449d9cf8291SDaniel Hwang else: 450d9cf8291SDaniel Hwang # extract the run and append it to the merged output 451*dd3c26a0STobias Hieta for run in sarif["runs"]: 452d9cf8291SDaniel Hwang new_run = update_sarif_object(run, runs_count) 453*dd3c26a0STobias Hieta merged["runs"].append(new_run) 454d9cf8291SDaniel Hwang 455*dd3c26a0STobias Hieta runs_count += len(sarif["runs"]) 456d9cf8291SDaniel Hwang 457*dd3c26a0STobias Hieta with open(os.path.join(output_dir, "results-merged.sarif"), "w") as out: 458d9cf8291SDaniel Hwang json.dump(merged, out, indent=4, sort_keys=True) 459d9cf8291SDaniel Hwang 460d9cf8291SDaniel Hwang 461d9cf8291SDaniel Hwangdef parse_bug_plist(filename): 462d9cf8291SDaniel Hwang """Returns the generator of bugs from a single .plist file.""" 463d9cf8291SDaniel Hwang 464*dd3c26a0STobias Hieta with open(filename, "rb") as fp: 465d9cf8291SDaniel Hwang content = plistlib.load(fp) 466*dd3c26a0STobias Hieta files = content.get("files") 467*dd3c26a0STobias Hieta for bug in content.get("diagnostics", []): 468*dd3c26a0STobias Hieta if len(files) <= int(bug["location"]["file"]): 469d9cf8291SDaniel Hwang logging.warning('Parsing bug from "%s" failed', filename) 470d9cf8291SDaniel Hwang continue 471d9cf8291SDaniel Hwang 472d9cf8291SDaniel Hwang yield { 473*dd3c26a0STobias Hieta "result": filename, 474*dd3c26a0STobias Hieta "bug_type": bug["type"], 475*dd3c26a0STobias Hieta "bug_category": bug["category"], 476*dd3c26a0STobias Hieta "bug_line": int(bug["location"]["line"]), 477*dd3c26a0STobias Hieta "bug_path_length": int(bug["location"]["col"]), 478*dd3c26a0STobias Hieta "bug_file": files[int(bug["location"]["file"])], 479d9cf8291SDaniel Hwang } 480d9cf8291SDaniel Hwang 481d9cf8291SDaniel Hwang 482d9cf8291SDaniel Hwangdef parse_bug_html(filename): 483d9cf8291SDaniel Hwang """Parse out the bug information from HTML output.""" 484d9cf8291SDaniel Hwang 485*dd3c26a0STobias Hieta patterns = [ 486*dd3c26a0STobias Hieta re.compile(r"<!-- BUGTYPE (?P<bug_type>.*) -->$"), 487*dd3c26a0STobias Hieta re.compile(r"<!-- BUGFILE (?P<bug_file>.*) -->$"), 488*dd3c26a0STobias Hieta re.compile(r"<!-- BUGPATHLENGTH (?P<bug_path_length>.*) -->$"), 489*dd3c26a0STobias Hieta re.compile(r"<!-- BUGLINE (?P<bug_line>.*) -->$"), 490*dd3c26a0STobias Hieta re.compile(r"<!-- BUGCATEGORY (?P<bug_category>.*) -->$"), 491*dd3c26a0STobias Hieta re.compile(r"<!-- BUGDESC (?P<bug_description>.*) -->$"), 492*dd3c26a0STobias Hieta re.compile(r"<!-- FUNCTIONNAME (?P<bug_function>.*) -->$"), 493*dd3c26a0STobias Hieta ] 494*dd3c26a0STobias Hieta endsign = re.compile(r"<!-- BUGMETAEND -->") 495d9cf8291SDaniel Hwang 496d9cf8291SDaniel Hwang bug = { 497*dd3c26a0STobias Hieta "report_file": filename, 498*dd3c26a0STobias Hieta "bug_function": "n/a", # compatibility with < clang-3.5 499*dd3c26a0STobias Hieta "bug_category": "Other", 500*dd3c26a0STobias Hieta "bug_line": 0, 501*dd3c26a0STobias Hieta "bug_path_length": 1, 502d9cf8291SDaniel Hwang } 503d9cf8291SDaniel Hwang 504*dd3c26a0STobias Hieta with open(filename, encoding="utf-8") as handler: 505d9cf8291SDaniel Hwang for line in handler.readlines(): 506d9cf8291SDaniel Hwang # do not read the file further 507d9cf8291SDaniel Hwang if endsign.match(line): 508d9cf8291SDaniel Hwang break 509d9cf8291SDaniel Hwang # search for the right lines 510d9cf8291SDaniel Hwang for regex in patterns: 511d9cf8291SDaniel Hwang match = regex.match(line.strip()) 512d9cf8291SDaniel Hwang if match: 513d9cf8291SDaniel Hwang bug.update(match.groupdict()) 514d9cf8291SDaniel Hwang break 515d9cf8291SDaniel Hwang 516*dd3c26a0STobias Hieta encode_value(bug, "bug_line", int) 517*dd3c26a0STobias Hieta encode_value(bug, "bug_path_length", int) 518d9cf8291SDaniel Hwang 519d9cf8291SDaniel Hwang yield bug 520d9cf8291SDaniel Hwang 521d9cf8291SDaniel Hwang 522d9cf8291SDaniel Hwangdef parse_crash(filename): 523d9cf8291SDaniel Hwang """Parse out the crash information from the report file.""" 524d9cf8291SDaniel Hwang 525*dd3c26a0STobias Hieta match = re.match(r"(.*)\.info\.txt", filename) 526d9cf8291SDaniel Hwang name = match.group(1) if match else None 527*dd3c26a0STobias Hieta with open(filename, mode="rb") as handler: 528d9cf8291SDaniel Hwang # this is a workaround to fix windows read '\r\n' as new lines. 529d9cf8291SDaniel Hwang lines = [line.decode().rstrip() for line in handler.readlines()] 530d9cf8291SDaniel Hwang return { 531*dd3c26a0STobias Hieta "source": lines[0], 532*dd3c26a0STobias Hieta "problem": lines[1], 533*dd3c26a0STobias Hieta "file": name, 534*dd3c26a0STobias Hieta "info": name + ".info.txt", 535*dd3c26a0STobias Hieta "stderr": name + ".stderr.txt", 536d9cf8291SDaniel Hwang } 537d9cf8291SDaniel Hwang 538d9cf8291SDaniel Hwang 539d9cf8291SDaniel Hwangdef category_type_name(bug): 540d9cf8291SDaniel Hwang """Create a new bug attribute from bug by category and type. 541d9cf8291SDaniel Hwang 542d9cf8291SDaniel Hwang The result will be used as CSS class selector in the final report.""" 543d9cf8291SDaniel Hwang 544d9cf8291SDaniel Hwang def smash(key): 545d9cf8291SDaniel Hwang """Make value ready to be HTML attribute value.""" 546d9cf8291SDaniel Hwang 547*dd3c26a0STobias Hieta return bug.get(key, "").lower().replace(" ", "_").replace("'", "") 548d9cf8291SDaniel Hwang 549*dd3c26a0STobias Hieta return escape("bt_" + smash("bug_category") + "_" + smash("bug_type")) 550d9cf8291SDaniel Hwang 551d9cf8291SDaniel Hwang 552d9cf8291SDaniel Hwangdef create_counters(): 553d9cf8291SDaniel Hwang """Create counters for bug statistics. 554d9cf8291SDaniel Hwang 555d9cf8291SDaniel Hwang Two entries are maintained: 'total' is an integer, represents the 556d9cf8291SDaniel Hwang number of bugs. The 'categories' is a two level categorisation of bug 557d9cf8291SDaniel Hwang counters. The first level is 'bug category' the second is 'bug type'. 558d9cf8291SDaniel Hwang Each entry in this classification is a dictionary of 'count', 'type' 559d9cf8291SDaniel Hwang and 'label'.""" 560d9cf8291SDaniel Hwang 561d9cf8291SDaniel Hwang def predicate(bug): 562*dd3c26a0STobias Hieta bug_category = bug["bug_category"] 563*dd3c26a0STobias Hieta bug_type = bug["bug_type"] 564d9cf8291SDaniel Hwang current_category = predicate.categories.get(bug_category, dict()) 565*dd3c26a0STobias Hieta current_type = current_category.get( 566*dd3c26a0STobias Hieta bug_type, 567*dd3c26a0STobias Hieta { 568*dd3c26a0STobias Hieta "bug_type": bug_type, 569*dd3c26a0STobias Hieta "bug_type_class": category_type_name(bug), 570*dd3c26a0STobias Hieta "bug_count": 0, 571*dd3c26a0STobias Hieta }, 572*dd3c26a0STobias Hieta ) 573*dd3c26a0STobias Hieta current_type.update({"bug_count": current_type["bug_count"] + 1}) 574d9cf8291SDaniel Hwang current_category.update({bug_type: current_type}) 575d9cf8291SDaniel Hwang predicate.categories.update({bug_category: current_category}) 576d9cf8291SDaniel Hwang predicate.total += 1 577d9cf8291SDaniel Hwang 578d9cf8291SDaniel Hwang predicate.total = 0 579d9cf8291SDaniel Hwang predicate.categories = dict() 580d9cf8291SDaniel Hwang return predicate 581d9cf8291SDaniel Hwang 582d9cf8291SDaniel Hwang 583d9cf8291SDaniel Hwangdef prettify_bug(prefix, output_dir): 584d9cf8291SDaniel Hwang def predicate(bug): 585d9cf8291SDaniel Hwang """Make safe this values to embed into HTML.""" 586d9cf8291SDaniel Hwang 587*dd3c26a0STobias Hieta bug["bug_type_class"] = category_type_name(bug) 588d9cf8291SDaniel Hwang 589*dd3c26a0STobias Hieta encode_value(bug, "bug_file", lambda x: escape(chop(prefix, x))) 590*dd3c26a0STobias Hieta encode_value(bug, "bug_category", escape) 591*dd3c26a0STobias Hieta encode_value(bug, "bug_type", escape) 592*dd3c26a0STobias Hieta encode_value(bug, "report_file", lambda x: escape(chop(output_dir, x))) 593d9cf8291SDaniel Hwang return bug 594d9cf8291SDaniel Hwang 595d9cf8291SDaniel Hwang return predicate 596d9cf8291SDaniel Hwang 597d9cf8291SDaniel Hwang 598d9cf8291SDaniel Hwangdef prettify_crash(prefix, output_dir): 599d9cf8291SDaniel Hwang def predicate(crash): 600d9cf8291SDaniel Hwang """Make safe this values to embed into HTML.""" 601d9cf8291SDaniel Hwang 602*dd3c26a0STobias Hieta encode_value(crash, "source", lambda x: escape(chop(prefix, x))) 603*dd3c26a0STobias Hieta encode_value(crash, "problem", escape) 604*dd3c26a0STobias Hieta encode_value(crash, "file", lambda x: escape(chop(output_dir, x))) 605*dd3c26a0STobias Hieta encode_value(crash, "info", lambda x: escape(chop(output_dir, x))) 606*dd3c26a0STobias Hieta encode_value(crash, "stderr", lambda x: escape(chop(output_dir, x))) 607d9cf8291SDaniel Hwang return crash 608d9cf8291SDaniel Hwang 609d9cf8291SDaniel Hwang return predicate 610d9cf8291SDaniel Hwang 611d9cf8291SDaniel Hwang 612d9cf8291SDaniel Hwangdef copy_resource_files(output_dir): 613d9cf8291SDaniel Hwang """Copy the javascript and css files to the report directory.""" 614d9cf8291SDaniel Hwang 615d9cf8291SDaniel Hwang this_dir = os.path.dirname(os.path.realpath(__file__)) 616*dd3c26a0STobias Hieta for resource in os.listdir(os.path.join(this_dir, "resources")): 617*dd3c26a0STobias Hieta shutil.copy(os.path.join(this_dir, "resources", resource), output_dir) 618d9cf8291SDaniel Hwang 619d9cf8291SDaniel Hwang 620d9cf8291SDaniel Hwangdef encode_value(container, key, encode): 621d9cf8291SDaniel Hwang """Run 'encode' on 'container[key]' value and update it.""" 622d9cf8291SDaniel Hwang 623d9cf8291SDaniel Hwang if key in container: 624d9cf8291SDaniel Hwang value = encode(container[key]) 625d9cf8291SDaniel Hwang container.update({key: value}) 626d9cf8291SDaniel Hwang 627d9cf8291SDaniel Hwang 628d9cf8291SDaniel Hwangdef chop(prefix, filename): 629d9cf8291SDaniel Hwang """Create 'filename' from '/prefix/filename'""" 630d9cf8291SDaniel Hwang 631d9cf8291SDaniel Hwang return filename if not len(prefix) else os.path.relpath(filename, prefix) 632d9cf8291SDaniel Hwang 633d9cf8291SDaniel Hwang 634d9cf8291SDaniel Hwangdef escape(text): 635d9cf8291SDaniel Hwang """Paranoid HTML escape method. (Python version independent)""" 636d9cf8291SDaniel Hwang 637d9cf8291SDaniel Hwang escape_table = { 638*dd3c26a0STobias Hieta "&": "&", 639*dd3c26a0STobias Hieta '"': """, 640*dd3c26a0STobias Hieta "'": "'", 641*dd3c26a0STobias Hieta ">": ">", 642*dd3c26a0STobias Hieta "<": "<", 643d9cf8291SDaniel Hwang } 644*dd3c26a0STobias Hieta return "".join(escape_table.get(c, c) for c in text) 645d9cf8291SDaniel Hwang 646d9cf8291SDaniel Hwang 647d9cf8291SDaniel Hwangdef reindent(text, indent): 648d9cf8291SDaniel Hwang """Utility function to format html output and keep indentation.""" 649d9cf8291SDaniel Hwang 650*dd3c26a0STobias Hieta result = "" 651d9cf8291SDaniel Hwang for line in text.splitlines(): 652d9cf8291SDaniel Hwang if len(line.strip()): 653*dd3c26a0STobias Hieta result += " " * indent + line.split("|")[1] + os.linesep 654d9cf8291SDaniel Hwang return result 655d9cf8291SDaniel Hwang 656d9cf8291SDaniel Hwang 657d9cf8291SDaniel Hwangdef comment(name, opts=dict()): 658d9cf8291SDaniel Hwang """Utility function to format meta information as comment.""" 659d9cf8291SDaniel Hwang 660*dd3c26a0STobias Hieta attributes = "" 661d9cf8291SDaniel Hwang for key, value in opts.items(): 662d9cf8291SDaniel Hwang attributes += ' {0}="{1}"'.format(key, value) 663d9cf8291SDaniel Hwang 664*dd3c26a0STobias Hieta return "<!-- {0}{1} -->{2}".format(name, attributes, os.linesep) 665d9cf8291SDaniel Hwang 666d9cf8291SDaniel Hwang 667d9cf8291SDaniel Hwangdef commonprefix_from(filename): 668d9cf8291SDaniel Hwang """Create file prefix from a compilation database entries.""" 669d9cf8291SDaniel Hwang 670*dd3c26a0STobias Hieta with open(filename, "r") as handle: 671*dd3c26a0STobias Hieta return commonprefix(item["file"] for item in json.load(handle)) 672d9cf8291SDaniel Hwang 673d9cf8291SDaniel Hwang 674d9cf8291SDaniel Hwangdef commonprefix(files): 675d9cf8291SDaniel Hwang """Fixed version of os.path.commonprefix. 676d9cf8291SDaniel Hwang 677d9cf8291SDaniel Hwang :param files: list of file names. 678d9cf8291SDaniel Hwang :return: the longest path prefix that is a prefix of all files.""" 679d9cf8291SDaniel Hwang result = None 680d9cf8291SDaniel Hwang for current in files: 681d9cf8291SDaniel Hwang if result is not None: 682d9cf8291SDaniel Hwang result = os.path.commonprefix([result, current]) 683d9cf8291SDaniel Hwang else: 684d9cf8291SDaniel Hwang result = current 685d9cf8291SDaniel Hwang 686d9cf8291SDaniel Hwang if result is None: 687*dd3c26a0STobias Hieta return "" 688d9cf8291SDaniel Hwang elif not os.path.isdir(result): 689d9cf8291SDaniel Hwang return os.path.dirname(result) 690d9cf8291SDaniel Hwang else: 691d9cf8291SDaniel Hwang return os.path.abspath(result) 692