1#!/usr/bin/env python3 2# -*- coding: UTF-8 -*- 3 4# Polly/LLVM update_check.py 5# Update lit FileCheck files by replacing the 'CHECK:' lines by the actual output of the 'RUN:' command. 6 7import argparse 8import os 9import subprocess 10import shlex 11import re 12 13 14polly_src_dir = """@POLLY_SOURCE_DIR@""" 15polly_lib_dir = """@POLLY_LIB_DIR@""" 16shlibext = """@LLVM_SHLIBEXT@""" 17llvm_tools_dir = """@LLVM_TOOLS_DIR@""" 18llvm_polly_link_into_tools = not """@LLVM_POLLY_LINK_INTO_TOOLS@""".lower() in { 19 "", 20 "0", 21 "n", 22 "no", 23 "off", 24 "false", 25 "notfound", 26 "llvm_polly_link_into_tools-notfound", 27} 28 29runre = re.compile(r"\s*\;\s*RUN\s*\:(?P<tool>.*)") 30filecheckre = re.compile(r"\s*(?P<tool>.*)\|\s*(?P<filecheck>FileCheck\s[^|]*)") 31emptyline = re.compile(r"\s*(\;\s*)?") 32commentline = re.compile(r"\s*(\;.*)?") 33 34 35def ltrim_emptylines(lines, meta=None): 36 while len(lines) and emptyline.fullmatch(lines[0]): 37 del lines[0] 38 if meta is not None: 39 del meta[0] 40 41 42def rtrim_emptylines(lines): 43 while len(lines) and emptyline.fullmatch(lines[-1]): 44 del lines[-1] 45 46 47def trim_emptylines(lines): 48 ltrim_emptylines(lines) 49 rtrim_emptylines(lines) 50 51 52def complete_exename(path, filename): 53 complpath = os.path.join(path, filename) 54 if os.path.isfile(complpath): 55 return complpath 56 elif os.path.isfile(complpath + ".exe"): 57 return complpath + ".exe" 58 return filename 59 60 61def indention(line): 62 for i, c in enumerate(line): 63 if c != " " and c != "\t": 64 return i 65 return None 66 67 68def common_indent(lines): 69 indentions = (indention(line) for line in lines) 70 indentions = (indent for indent in indentions if indent is not None) 71 return min(indentions, default=0) 72 73 74funcre = re.compile(r"^ Function: \S*$") 75regionre = re.compile(r"^ Region: \S*$") 76depthre = re.compile(r"^ Max Loop Depth: .*") 77paramre = re.compile(r" [0-9a-z-A-Z_]+\: .*") 78 79 80def classyfier1(lines): 81 i = iter(lines) 82 line = i.__next__() 83 while True: 84 if line.startswith( 85 "Printing analysis 'Polly - Calculate dependences' for region: " 86 ): 87 yield {"PrintingDependenceInfo"} 88 elif line.startswith("remark: "): 89 yield {"Remark"} 90 elif funcre.fullmatch(line): 91 yield {"Function"} 92 elif regionre.fullmatch(line): 93 yield {"Region"} 94 elif depthre.fullmatch(line): 95 yield {"MaxLoopDepth"} 96 elif line == " Invariant Accesses: {": 97 while True: 98 yield {"InvariantAccesses"} 99 if line == " }": 100 break 101 line = i.__next__() 102 elif line == " Context:": 103 yield {"Context"} 104 line = i.__next__() 105 yield {"Context"} 106 elif line == " Assumed Context:": 107 yield {"AssumedContext"} 108 line = i.__next__() 109 yield {"AssumedContext"} 110 elif line == " Invalid Context:": 111 yield {"InvalidContext"} 112 line = i.__next__() 113 yield {"InvalidContext"} 114 elif line == " Boundary Context:": 115 yield {"BoundaryContext"} 116 line = i.__next__() 117 yield {"BoundaryContext"} 118 line = i.__next__() 119 while paramre.fullmatch(line): 120 yield {"Param"} 121 line = i.__next__() 122 continue 123 elif line == " Arrays {": 124 while True: 125 yield {"Arrays"} 126 if line == " }": 127 break 128 line = i.__next__() 129 elif line == " Arrays (Bounds as pw_affs) {": 130 while True: 131 yield {"PwAffArrays"} 132 if line == " }": 133 break 134 line = i.__next__() 135 elif line.startswith(" Alias Groups ("): 136 while True: 137 yield {"AliasGroups"} 138 line = i.__next__() 139 if not line.startswith(" "): 140 break 141 continue 142 elif line == " Statements {": 143 while True: 144 yield {"Statements"} 145 if line == " }": 146 break 147 line = i.__next__() 148 elif line == " RAW dependences:": 149 yield {"RAWDep", "BasicDep", "Dep", "DepInfo"} 150 line = i.__next__() 151 while line.startswith(" "): 152 yield {"RAWDep", "BasicDep", "Dep", "DepInfo"} 153 line = i.__next__() 154 continue 155 elif line == " WAR dependences:": 156 yield {"WARDep", "BasicDep", "Dep", "DepInfo"} 157 line = i.__next__() 158 while line.startswith(" "): 159 yield {"WARDep", "BasicDep", "Dep", "DepInfo"} 160 line = i.__next__() 161 continue 162 elif line == " WAW dependences:": 163 yield {"WAWDep", "BasicDep", "Dep", "DepInfo"} 164 line = i.__next__() 165 while line.startswith(" "): 166 yield {"WAWDep", "BasicDep", "Dep", "DepInfo"} 167 line = i.__next__() 168 continue 169 elif line == " Reduction dependences:": 170 yield {"RedDep", "Dep", "DepInfo"} 171 line = i.__next__() 172 while line.startswith(" "): 173 yield {"RedDep", "Dep", "DepInfo"} 174 line = i.__next__() 175 continue 176 elif line == " Transitive closure of reduction dependences:": 177 yield {"TransitiveClosureDep", "DepInfo"} 178 line = i.__next__() 179 while line.startswith(" "): 180 yield {"TransitiveClosureDep", "DepInfo"} 181 line = i.__next__() 182 continue 183 elif line.startswith("New access function '"): 184 yield {"NewAccessFunction"} 185 elif line == "Schedule before flattening {": 186 while True: 187 yield {"ScheduleBeforeFlattening"} 188 if line == "}": 189 break 190 line = i.__next__() 191 elif line == "Schedule after flattening {": 192 while True: 193 yield {"ScheduleAfterFlattening"} 194 if line == "}": 195 break 196 line = i.__next__() 197 else: 198 yield set() 199 line = i.__next__() 200 201 202def classyfier2(lines): 203 i = iter(lines) 204 line = i.__next__() 205 while True: 206 if funcre.fullmatch(line): 207 while line.startswith(" "): 208 yield {"FunctionDetail"} 209 line = i.__next__() 210 continue 211 elif line.startswith( 212 "Printing analysis 'Polly - Generate an AST from the SCoP (isl)' for region: " 213 ): 214 yield {"PrintingIslAst"} 215 line = i.__next__() 216 while not line.startswith("Printing analysis"): 217 yield {"AstDetail"} 218 line = i.__next__() 219 continue 220 else: 221 yield set() 222 line = i.__next__() 223 224 225replrepl = { 226 "{{": "{{[{][{]}}", 227 "}}": "{{[}][}]}}", 228 "[[": r"{{\[\[}}", 229 "]]": r"{{\]\]}}", 230} 231replre = re.compile("|".join(re.escape(k) for k in replrepl.keys())) 232 233 234def main(): 235 parser = argparse.ArgumentParser(description="Update CHECK lines") 236 parser.add_argument( 237 "testfile", help="File to update (absolute or relative to --testdir)" 238 ) 239 parser.add_argument( 240 "--check-style", 241 choices=["CHECK", "CHECK-NEXT"], 242 default="CHECK-NEXT", 243 help="What kind of checks lines to generate", 244 ) 245 parser.add_argument( 246 "--check-position", 247 choices=["end", "before-content", "autodetect"], 248 default="autodetect", 249 help="Where to add the CHECK lines into the file; 'autodetect' searches for the first 'CHECK' line ind inserts it there", 250 ) 251 parser.add_argument( 252 "--check-include", 253 action="append", 254 default=[], 255 help="What parts of the output lines to check; use syntax 'CHECK=include' to apply to one CHECK-prefix only (by default, everything)", 256 ) 257 parser.add_argument( 258 "--check-label-include", 259 action="append", 260 default=[], 261 help="Use CHECK-LABEL for these includes", 262 ) 263 parser.add_argument( 264 "--check-part-newline", 265 action="store_true", 266 help="Add empty line between different check parts", 267 ) 268 parser.add_argument( 269 "--prefix-only", 270 action="append", 271 default=None, 272 help="Update only these prefixes (default: all)", 273 ) 274 parser.add_argument("--bindir", help="Location of the opt program") 275 parser.add_argument("--testdir", help="Root dir for unit tests") 276 parser.add_argument( 277 "--inplace", "-i", action="store_true", help="Replace input file" 278 ) 279 parser.add_argument("--output", "-o", help="Write changed input to this file") 280 known = parser.parse_args() 281 282 if not known.inplace and known.output is None: 283 print("Must specify what to do with output (--output or --inplace)") 284 exit(1) 285 if known.inplace and known.output is not None: 286 print("--inplace and --output are mutually exclusive") 287 exit(1) 288 289 outfile = known.output 290 291 filecheckparser = argparse.ArgumentParser(add_help=False) 292 filecheckparser.add_argument("-check-prefix", "--check-prefix", default="CHECK") 293 294 filename = known.testfile 295 for dir in [".", known.testdir, os.path.join(polly_src_dir, "test"), polly_src_dir]: 296 if not dir: 297 continue 298 testfilename = os.path.join(dir, filename) 299 if os.path.isfile(testfilename): 300 filename = testfilename 301 break 302 303 if known.inplace: 304 outfile = filename 305 306 allchecklines = [] 307 checkprefixes = [] 308 309 with open(filename, "r") as file: 310 oldlines = [line.rstrip("\r\n") for line in file.readlines()] 311 312 runlines = [] 313 for line in oldlines: 314 m = runre.match(line) 315 if m: 316 runlines.append(m.group("tool")) 317 318 continuation = "" 319 newrunlines = [] 320 for line in runlines: 321 if line.endswith("\\"): 322 continuation += line[:-2] + " " 323 else: 324 newrunlines.append(continuation + line) 325 continuation = "" 326 if continuation: 327 newrunlines.append(continuation) 328 329 for line in newrunlines: 330 m = filecheckre.match(line) 331 if not m: 332 continue 333 334 tool, filecheck = m.group("tool", "filecheck") 335 filecheck = shlex.split(filecheck) 336 tool = shlex.split(tool) 337 if known.bindir is not None: 338 tool[0] = complete_exename(known.bindir, tool[0]) 339 if os.path.isdir(llvm_tools_dir): 340 tool[0] = complete_exename(llvm_tools_dir, tool[0]) 341 check_prefix = filecheckparser.parse_known_args(filecheck)[0].check_prefix 342 if known.prefix_only is not None and not check_prefix in known.prefix_only: 343 continue 344 if check_prefix in checkprefixes: 345 continue 346 checkprefixes.append(check_prefix) 347 348 newtool = [] 349 optstderr = None 350 for toolarg in tool: 351 toolarg = toolarg.replace("%s", filename) 352 toolarg = toolarg.replace("%S", os.path.dirname(filename)) 353 if toolarg == "%loadPolly": 354 if not llvm_polly_link_into_tools: 355 newtool += [ 356 "-load", 357 os.path.join(polly_lib_dir, "LLVMPolly" + shlibext), 358 ] 359 newtool.append("-polly-process-unprofitable") 360 newtool.append("-polly-remarks-minimal") 361 elif toolarg == "2>&1": 362 optstderr = subprocess.STDOUT 363 else: 364 newtool.append(toolarg) 365 tool = newtool 366 367 inpfile = None 368 i = 1 369 while i < len(tool): 370 if tool[i] == "<": 371 inpfile = tool[i + 1] 372 del tool[i : i + 2] 373 continue 374 i += 1 375 if inpfile: 376 with open(inpfile) as inp: 377 retlines = subprocess.check_output( 378 tool, universal_newlines=True, stdin=inp, stderr=optstderr 379 ) 380 else: 381 retlines = subprocess.check_output( 382 tool, universal_newlines=True, stderr=optstderr 383 ) 384 retlines = [line.replace("\t", " ") for line in retlines.splitlines()] 385 check_include = [] 386 for checkme in known.check_include + known.check_label_include: 387 parts = checkme.split("=") 388 if len(parts) == 2: 389 if parts[0] == check_prefix: 390 check_include.append(parts[1]) 391 else: 392 check_include.append(checkme) 393 394 if check_include: 395 filtered_retlines = [] 396 classified_retlines = [] 397 lastmatch = None 398 for line, kind in ( 399 (line, class1.union(class2)) 400 for line, class1, class2 in zip( 401 retlines, classyfier1(retlines), classyfier2(retlines) 402 ) 403 ): 404 match = kind.intersection(check_include) 405 if match: 406 if lastmatch != match: 407 filtered_retlines.append("") 408 classified_retlines.append({"Separator"}) 409 filtered_retlines.append(line) 410 classified_retlines.append(kind) 411 lastmatch = match 412 413 retlines = filtered_retlines 414 else: 415 classified_retlines = (set() for line in retlines) 416 417 rtrim_emptylines(retlines) 418 ltrim_emptylines(retlines, classified_retlines) 419 retlines = [ 420 replre.sub(lambda m: replrepl[m.group(0)], line) for line in retlines 421 ] 422 indent = common_indent(retlines) 423 retlines = [line[indent:] for line in retlines] 424 checklines = [] 425 previous_was_empty = True 426 for line, kind in zip(retlines, classified_retlines): 427 if line: 428 if known.check_style == "CHECK" and known.check_label_include: 429 if not kind.isdisjoint(known.check_label_include): 430 checklines.append("; " + check_prefix + "-LABEL: " + line) 431 else: 432 checklines.append("; " + check_prefix + ": " + line) 433 elif known.check_style == "CHECK": 434 checklines.append("; " + check_prefix + ": " + line) 435 elif known.check_label_include and known.check_label_include: 436 if not kind.isdisjoint(known.check_label_include): 437 checklines.append("; " + check_prefix + "-LABEL: " + line) 438 elif previous_was_empty: 439 checklines.append("; " + check_prefix + ": " + line) 440 else: 441 checklines.append("; " + check_prefix + "-NEXT: " + line) 442 else: 443 if previous_was_empty: 444 checklines.append("; " + check_prefix + ": " + line) 445 else: 446 checklines.append("; " + check_prefix + "-NEXT: " + line) 447 previous_was_empty = False 448 else: 449 if not "Separator" in kind or known.check_part_newline: 450 checklines.append(";") 451 previous_was_empty = True 452 allchecklines.append(checklines) 453 454 if not checkprefixes: 455 return 456 457 checkre = re.compile( 458 r"^\s*\;\s*(" 459 + "|".join([re.escape(s) for s in checkprefixes]) 460 + r")(\-NEXT|\-DAG|\-NOT|\-LABEL|\-SAME)?\s*\:" 461 ) 462 firstcheckline = None 463 firstnoncommentline = None 464 headerlines = [] 465 newlines = [] 466 uptonowlines = [] 467 emptylines = [] 468 lastwascheck = False 469 for line in oldlines: 470 if checkre.match(line): 471 if firstcheckline is None: 472 firstcheckline = len(newlines) + len(emptylines) 473 if not lastwascheck: 474 uptonowlines += emptylines 475 emptylines = [] 476 lastwascheck = True 477 elif emptyline.fullmatch(line): 478 emptylines.append(line) 479 else: 480 newlines += uptonowlines 481 newlines += emptylines 482 newlines.append(line) 483 emptylines = [] 484 uptonowlines = [] 485 lastwascheck = False 486 487 for i, line in enumerate(newlines): 488 if not commentline.fullmatch(line): 489 firstnoncommentline = i 490 break 491 492 with open(outfile, "w", newline="") as file: 493 494 def writelines(lines): 495 for line in lines: 496 file.write(line) 497 file.write("\n") 498 499 if firstcheckline is not None and known.check_position == "autodetect": 500 writelines(newlines[:firstcheckline]) 501 writelines(uptonowlines) 502 for i, checklines in enumerate(allchecklines): 503 if i != 0: 504 file.write("\n") 505 writelines(checklines) 506 writelines(newlines[firstcheckline:]) 507 writelines(emptylines) 508 elif ( 509 firstnoncommentline is not None and known.check_position == "before-content" 510 ): 511 headerlines = newlines[:firstnoncommentline] 512 rtrim_emptylines(headerlines) 513 contentlines = newlines[firstnoncommentline:] 514 ltrim_emptylines(contentlines) 515 516 writelines(headerlines) 517 for checklines in allchecklines: 518 file.write("\n") 519 writelines(checklines) 520 file.write("\n") 521 writelines(contentlines) 522 writelines(uptonowlines) 523 writelines(emptylines) 524 else: 525 writelines(newlines) 526 rtrim_emptylines(newlines) 527 for checklines in allchecklines: 528 file.write("\n\n") 529 writelines(checklines) 530 531 532if __name__ == "__main__": 533 main() 534