1from __future__ import print_function 2 3import argparse 4import copy 5import glob 6import itertools 7import os 8import re 9import subprocess 10import sys 11import shlex 12 13from typing import List 14 15##### Common utilities for update_*test_checks.py 16 17 18_verbose = False 19_prefix_filecheck_ir_name = '' 20 21""" 22Version changelog: 23 241: Initial version, used by tests that don't specify --version explicitly. 252: --function-signature now also checks return type/attributes. 26""" 27DEFAULT_VERSION = 1 28 29class Regex(object): 30 """Wrap a compiled regular expression object to allow deep copy of a regexp. 31 This is required for the deep copy done in do_scrub. 32 33 """ 34 def __init__(self, regex): 35 self.regex = regex 36 37 def __deepcopy__(self, memo): 38 result = copy.copy(self) 39 result.regex = self.regex 40 return result 41 42 def search(self, line): 43 return self.regex.search(line) 44 45 def sub(self, repl, line): 46 return self.regex.sub(repl, line) 47 48 def pattern(self): 49 return self.regex.pattern 50 51 def flags(self): 52 return self.regex.flags 53 54class Filter(Regex): 55 """Augment a Regex object with a flag indicating whether a match should be 56 added (!is_filter_out) or removed (is_filter_out) from the generated checks. 57 58 """ 59 def __init__(self, regex, is_filter_out): 60 super(Filter, self).__init__(regex) 61 self.is_filter_out = is_filter_out 62 63 def __deepcopy__(self, memo): 64 result = copy.deepcopy(super(Filter, self), memo) 65 result.is_filter_out = copy.deepcopy(self.is_filter_out, memo) 66 return result 67 68def parse_commandline_args(parser): 69 class RegexAction(argparse.Action): 70 """Add a regular expression option value to a list of regular expressions. 71 This compiles the expression, wraps it in a Regex and adds it to the option 72 value list.""" 73 def __init__(self, option_strings, dest, nargs=None, **kwargs): 74 if nargs is not None: 75 raise ValueError('nargs not allowed') 76 super(RegexAction, self).__init__(option_strings, dest, **kwargs) 77 78 def do_call(self, namespace, values, flags): 79 value_list = getattr(namespace, self.dest) 80 if value_list is None: 81 value_list = [] 82 83 try: 84 value_list.append(Regex(re.compile(values, flags))) 85 except re.error as error: 86 raise ValueError('{}: Invalid regular expression \'{}\' ({})'.format( 87 option_string, error.pattern, error.msg)) 88 89 setattr(namespace, self.dest, value_list) 90 91 def __call__(self, parser, namespace, values, option_string=None): 92 self.do_call(namespace, values, 0) 93 94 class FilterAction(RegexAction): 95 """Add a filter to a list of filter option values.""" 96 def __init__(self, option_strings, dest, nargs=None, **kwargs): 97 super(FilterAction, self).__init__(option_strings, dest, nargs, **kwargs) 98 99 def __call__(self, parser, namespace, values, option_string=None): 100 super(FilterAction, self).__call__(parser, namespace, values, option_string) 101 102 value_list = getattr(namespace, self.dest) 103 104 is_filter_out = ( option_string == '--filter-out' ) 105 106 value_list[-1] = Filter(value_list[-1].regex, is_filter_out) 107 108 setattr(namespace, self.dest, value_list) 109 110 filter_group = parser.add_argument_group( 111 'filtering', 112 """Filters are applied to each output line according to the order given. The 113 first matching filter terminates filter processing for that current line.""") 114 115 filter_group.add_argument('--filter', action=FilterAction, dest='filters', 116 metavar='REGEX', 117 help='Only include lines matching REGEX (may be specified multiple times)') 118 filter_group.add_argument('--filter-out', action=FilterAction, dest='filters', 119 metavar='REGEX', 120 help='Exclude lines matching REGEX') 121 122 parser.add_argument('--include-generated-funcs', action='store_true', 123 help='Output checks for functions not in source') 124 parser.add_argument('-v', '--verbose', action='store_true', 125 help='Show verbose output') 126 parser.add_argument('-u', '--update-only', action='store_true', 127 help='Only update test if it was already autogened') 128 parser.add_argument('--force-update', action='store_true', 129 help='Update test even if it was autogened by a different script') 130 parser.add_argument('--enable', action='store_true', dest='enabled', default=True, 131 help='Activate CHECK line generation from this point forward') 132 parser.add_argument('--disable', action='store_false', dest='enabled', 133 help='Deactivate CHECK line generation from this point forward') 134 parser.add_argument('--replace-value-regex', nargs='+', default=[], 135 help='List of regular expressions to replace matching value names') 136 parser.add_argument('--prefix-filecheck-ir-name', default='', 137 help='Add a prefix to FileCheck IR value names to avoid conflicts with scripted names') 138 parser.add_argument('--global-value-regex', nargs='+', default=[], 139 help='List of regular expressions that a global value declaration must match to generate a check (has no effect if checking globals is not enabled)') 140 parser.add_argument('--global-hex-value-regex', nargs='+', default=[], 141 help='List of regular expressions such that, for matching global value declarations, literal integer values should be encoded in hex in the associated FileCheck directives') 142 # FIXME: in 3.9, we can use argparse.BooleanOptionalAction. At that point, 143 # we need to rename the flag to just -generate-body-for-unused-prefixes. 144 parser.add_argument('--no-generate-body-for-unused-prefixes', 145 action='store_false', 146 dest='gen_unused_prefix_body', 147 default=True, 148 help='Generate a function body that always matches for unused prefixes. This is useful when unused prefixes are desired, and it avoids needing to annotate each FileCheck as allowing them.') 149 # This is the default when regenerating existing tests. The default when 150 # generating new tests is determined by DEFAULT_VERSION. 151 parser.add_argument('--version', type=int, default=1, 152 help='The version of output format') 153 args = parser.parse_args() 154 global _verbose, _global_value_regex, _global_hex_value_regex 155 _verbose = args.verbose 156 _global_value_regex = args.global_value_regex 157 _global_hex_value_regex = args.global_hex_value_regex 158 return args 159 160 161class InputLineInfo(object): 162 def __init__(self, line, line_number, args, argv): 163 self.line = line 164 self.line_number = line_number 165 self.args = args 166 self.argv = argv 167 168 169class TestInfo(object): 170 def __init__(self, test, parser, script_name, input_lines, args, argv, 171 comment_prefix, argparse_callback): 172 self.parser = parser 173 self.argparse_callback = argparse_callback 174 self.path = test 175 self.args = args 176 if args.prefix_filecheck_ir_name: 177 global _prefix_filecheck_ir_name 178 _prefix_filecheck_ir_name = args.prefix_filecheck_ir_name 179 self.argv = argv 180 self.input_lines = input_lines 181 self.run_lines = find_run_lines(test, self.input_lines) 182 self.comment_prefix = comment_prefix 183 if self.comment_prefix is None: 184 if self.path.endswith('.mir'): 185 self.comment_prefix = '#' 186 else: 187 self.comment_prefix = ';' 188 self.autogenerated_note_prefix = self.comment_prefix + ' ' + UTC_ADVERT 189 self.test_autogenerated_note = self.autogenerated_note_prefix + script_name 190 self.test_autogenerated_note += get_autogennote_suffix(parser, self.args) 191 self.test_unused_note = self.comment_prefix + self.comment_prefix + ' ' + UNUSED_NOTE 192 193 def ro_iterlines(self): 194 for line_num, input_line in enumerate(self.input_lines): 195 args, argv = check_for_command(input_line, self.parser, 196 self.args, self.argv, self.argparse_callback) 197 yield InputLineInfo(input_line, line_num, args, argv) 198 199 def iterlines(self, output_lines): 200 output_lines.append(self.test_autogenerated_note) 201 for line_info in self.ro_iterlines(): 202 input_line = line_info.line 203 # Discard any previous script advertising. 204 if input_line.startswith(self.autogenerated_note_prefix): 205 continue 206 self.args = line_info.args 207 self.argv = line_info.argv 208 if not self.args.enabled: 209 output_lines.append(input_line) 210 continue 211 yield line_info 212 213 def get_checks_for_unused_prefixes(self, run_list, used_prefixes: List[str]) -> List[str]: 214 run_list = [element for element in run_list if element[0] is not None] 215 unused_prefixes = set([ 216 prefix for sublist in run_list for prefix in sublist[0] 217 ]).difference(set(used_prefixes)) 218 219 ret = [] 220 if not unused_prefixes: 221 return ret 222 ret.append(self.test_unused_note) 223 for unused in sorted(unused_prefixes): 224 ret.append('{comment} {prefix}: {match_everything}'.format( 225 comment=self.comment_prefix, 226 prefix=unused, 227 match_everything=r"""{{.*}}""" 228 )) 229 return ret 230 231def itertests(test_patterns, parser, script_name, comment_prefix=None, argparse_callback=None): 232 for pattern in test_patterns: 233 # On Windows we must expand the patterns ourselves. 234 tests_list = glob.glob(pattern) 235 if not tests_list: 236 warn("Test file pattern '%s' was not found. Ignoring it." % (pattern,)) 237 continue 238 for test in tests_list: 239 with open(test) as f: 240 input_lines = [l.rstrip() for l in f] 241 first_line = input_lines[0] if input_lines else "" 242 is_regenerate = UTC_ADVERT in first_line 243 244 # If we're generating a new test, set the default version to the latest. 245 argv = sys.argv[:] 246 if not is_regenerate: 247 argv.insert(1, '--version=' + str(DEFAULT_VERSION)) 248 249 args = parser.parse_args(argv[1:]) 250 if argparse_callback is not None: 251 argparse_callback(args) 252 if is_regenerate: 253 if script_name not in first_line and not args.force_update: 254 warn("Skipping test which wasn't autogenerated by " + script_name, test) 255 continue 256 args, argv = check_for_command(first_line, parser, args, argv, argparse_callback) 257 elif args.update_only: 258 assert UTC_ADVERT not in first_line 259 warn("Skipping test which isn't autogenerated: " + test) 260 continue 261 final_input_lines = [] 262 for l in input_lines: 263 if UNUSED_NOTE in l: 264 break 265 final_input_lines.append(l) 266 yield TestInfo(test, parser, script_name, final_input_lines, args, argv, 267 comment_prefix, argparse_callback) 268 269 270def should_add_line_to_output(input_line, prefix_set, skip_global_checks = False, comment_marker = ';'): 271 # Skip any blank comment lines in the IR. 272 if not skip_global_checks and input_line.strip() == comment_marker: 273 return False 274 # Skip a special double comment line we use as a separator. 275 if input_line.strip() == comment_marker + SEPARATOR: 276 return False 277 # Skip any blank lines in the IR. 278 #if input_line.strip() == '': 279 # return False 280 # And skip any CHECK lines. We're building our own. 281 m = CHECK_RE.match(input_line) 282 if m and m.group(1) in prefix_set: 283 if skip_global_checks: 284 global_ir_value_re = re.compile(r'\[\[', flags=(re.M)) 285 return not global_ir_value_re.search(input_line) 286 return False 287 288 return True 289 290# Perform lit-like substitutions 291def getSubstitutions(sourcepath): 292 sourcedir = os.path.dirname(sourcepath) 293 return [('%s', sourcepath), 294 ('%S', sourcedir), 295 ('%p', sourcedir), 296 ('%{pathsep}', os.pathsep)] 297 298def applySubstitutions(s, substitutions): 299 for a,b in substitutions: 300 s = s.replace(a, b) 301 return s 302 303# Invoke the tool that is being tested. 304def invoke_tool(exe, cmd_args, ir, preprocess_cmd=None, verbose=False): 305 with open(ir) as ir_file: 306 substitutions = getSubstitutions(ir) 307 308 # TODO Remove the str form which is used by update_test_checks.py and 309 # update_llc_test_checks.py 310 # The safer list form is used by update_cc_test_checks.py 311 if preprocess_cmd: 312 # Allow pre-processing the IR file (e.g. using sed): 313 assert isinstance(preprocess_cmd, str) # TODO: use a list instead of using shell 314 preprocess_cmd = applySubstitutions(preprocess_cmd, substitutions).strip() 315 if verbose: 316 print('Pre-processing input file: ', ir, " with command '", 317 preprocess_cmd, "'", sep="", file=sys.stderr) 318 # Python 2.7 doesn't have subprocess.DEVNULL: 319 with open(os.devnull, 'w') as devnull: 320 pp = subprocess.Popen(preprocess_cmd, shell=True, stdin=devnull, 321 stdout=subprocess.PIPE) 322 ir_file = pp.stdout 323 324 if isinstance(cmd_args, list): 325 args = [applySubstitutions(a, substitutions) for a in cmd_args] 326 stdout = subprocess.check_output([exe] + args, stdin=ir_file) 327 else: 328 stdout = subprocess.check_output(exe + ' ' + applySubstitutions(cmd_args, substitutions), 329 shell=True, stdin=ir_file) 330 if sys.version_info[0] > 2: 331 # FYI, if you crashed here with a decode error, your run line probably 332 # results in bitcode or other binary format being written to the pipe. 333 # For an opt test, you probably want to add -S or -disable-output. 334 stdout = stdout.decode() 335 # Fix line endings to unix CR style. 336 return stdout.replace('\r\n', '\n') 337 338##### LLVM IR parser 339RUN_LINE_RE = re.compile(r'^\s*(?://|[;#])\s*RUN:\s*(.*)$') 340CHECK_PREFIX_RE = re.compile(r'--?check-prefix(?:es)?[= ](\S+)') 341PREFIX_RE = re.compile('^[a-zA-Z0-9_-]+$') 342CHECK_RE = re.compile(r'^\s*(?://|[;#])\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL|-SAME|-EMPTY)?:') 343 344UTC_ARGS_KEY = 'UTC_ARGS:' 345UTC_ARGS_CMD = re.compile(r'.*' + UTC_ARGS_KEY + '\s*(?P<cmd>.*)\s*$') 346UTC_ADVERT = 'NOTE: Assertions have been autogenerated by ' 347UNUSED_NOTE = 'NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:' 348 349OPT_FUNCTION_RE = re.compile( 350 r'^(\s*;\s*Function\sAttrs:\s(?P<attrs>[\w\s():,]+?))?\s*define\s+(?P<funcdef_attrs_and_ret>[^@]*)@(?P<func>[\w.$-]+?)\s*' 351 r'(?P<args_and_sig>\((\)|(.*?[\w.-]+?)\))[^{]*\{)\n(?P<body>.*?)^\}$', 352 flags=(re.M | re.S)) 353 354ANALYZE_FUNCTION_RE = re.compile( 355 r'^\s*\'(?P<analysis>[\w\s-]+?)\'\s+for\s+function\s+\'(?P<func>[\w.$-]+?)\':' 356 r'\s*\n(?P<body>.*)$', 357 flags=(re.X | re.S)) 358 359LV_DEBUG_RE = re.compile( 360 r'^\s*\'(?P<func>[\w.$-]+?)\'[^\n]*' 361 r'\s*\n(?P<body>.*)$', 362 flags=(re.X | re.S)) 363 364IR_FUNCTION_RE = re.compile(r'^\s*define\s+(?:internal\s+)?[^@]*@"?([\w.$-]+)"?\s*\(') 365TRIPLE_IR_RE = re.compile(r'^\s*target\s+triple\s*=\s*"([^"]+)"$') 366TRIPLE_ARG_RE = re.compile(r'-mtriple[= ]([^ ]+)') 367MARCH_ARG_RE = re.compile(r'-march[= ]([^ ]+)') 368DEBUG_ONLY_ARG_RE = re.compile(r'-debug-only[= ]([^ ]+)') 369 370SCRUB_LEADING_WHITESPACE_RE = re.compile(r'^(\s+)') 371SCRUB_WHITESPACE_RE = re.compile(r'(?!^(| \w))[ \t]+', flags=re.M) 372SCRUB_TRAILING_WHITESPACE_RE = re.compile(r'[ \t]+$', flags=re.M) 373SCRUB_TRAILING_WHITESPACE_TEST_RE = SCRUB_TRAILING_WHITESPACE_RE 374SCRUB_TRAILING_WHITESPACE_AND_ATTRIBUTES_RE = re.compile(r'([ \t]|(#[0-9]+))+$', flags=re.M) 375SCRUB_KILL_COMMENT_RE = re.compile(r'^ *#+ +kill:.*\n') 376SCRUB_LOOP_COMMENT_RE = re.compile( 377 r'# =>This Inner Loop Header:.*|# in Loop:.*', flags=re.M) 378SCRUB_TAILING_COMMENT_TOKEN_RE = re.compile(r'(?<=\S)+[ \t]*#$', flags=re.M) 379 380SEPARATOR = '.' 381 382def error(msg, test_file=None): 383 if test_file: 384 msg = '{}: {}'.format(msg, test_file) 385 print('ERROR: {}'.format(msg), file=sys.stderr) 386 387def warn(msg, test_file=None): 388 if test_file: 389 msg = '{}: {}'.format(msg, test_file) 390 print('WARNING: {}'.format(msg), file=sys.stderr) 391 392def debug(*args, **kwargs): 393 # Python2 does not allow def debug(*args, file=sys.stderr, **kwargs): 394 if 'file' not in kwargs: 395 kwargs['file'] = sys.stderr 396 if _verbose: 397 print(*args, **kwargs) 398 399def find_run_lines(test, lines): 400 debug('Scanning for RUN lines in test file:', test) 401 raw_lines = [m.group(1) 402 for m in [RUN_LINE_RE.match(l) for l in lines] if m] 403 run_lines = [raw_lines[0]] if len(raw_lines) > 0 else [] 404 for l in raw_lines[1:]: 405 if run_lines[-1].endswith('\\'): 406 run_lines[-1] = run_lines[-1].rstrip('\\') + ' ' + l 407 else: 408 run_lines.append(l) 409 debug('Found {} RUN lines in {}:'.format(len(run_lines), test)) 410 for l in run_lines: 411 debug(' RUN: {}'.format(l)) 412 return run_lines 413 414def get_triple_from_march(march): 415 triples = { 416 'amdgcn': 'amdgcn', 417 'r600': 'r600', 418 'mips': 'mips', 419 'sparc': 'sparc', 420 'hexagon': 'hexagon', 421 've': 've', 422 } 423 for prefix, triple in triples.items(): 424 if march.startswith(prefix): 425 return triple 426 print("Cannot find a triple. Assume 'x86'", file=sys.stderr) 427 return 'x86' 428 429def apply_filters(line, filters): 430 has_filter = False 431 for f in filters: 432 if not f.is_filter_out: 433 has_filter = True 434 if f.search(line): 435 return False if f.is_filter_out else True 436 # If we only used filter-out, keep the line, otherwise discard it since no 437 # filter matched. 438 return False if has_filter else True 439 440def do_filter(body, filters): 441 return body if not filters else '\n'.join(filter( 442 lambda line: apply_filters(line, filters), body.splitlines())) 443 444def scrub_body(body): 445 # Scrub runs of whitespace out of the assembly, but leave the leading 446 # whitespace in place. 447 body = SCRUB_WHITESPACE_RE.sub(r' ', body) 448 # Expand the tabs used for indentation. 449 body = str.expandtabs(body, 2) 450 # Strip trailing whitespace. 451 body = SCRUB_TRAILING_WHITESPACE_TEST_RE.sub(r'', body) 452 return body 453 454def do_scrub(body, scrubber, scrubber_args, extra): 455 if scrubber_args: 456 local_args = copy.deepcopy(scrubber_args) 457 local_args[0].extra_scrub = extra 458 return scrubber(body, *local_args) 459 return scrubber(body, *scrubber_args) 460 461# Build up a dictionary of all the function bodies. 462class function_body(object): 463 def __init__(self, string, extra, funcdef_attrs_and_ret, args_and_sig, attrs, func_name_separator): 464 self.scrub = string 465 self.extrascrub = extra 466 self.funcdef_attrs_and_ret = funcdef_attrs_and_ret 467 self.args_and_sig = args_and_sig 468 self.attrs = attrs 469 self.func_name_separator = func_name_separator 470 def is_same_except_arg_names(self, extrascrub, funcdef_attrs_and_ret, args_and_sig, attrs, is_backend): 471 arg_names = set() 472 def drop_arg_names(match): 473 arg_names.add(match.group(variable_group_in_ir_value_match)) 474 if match.group(attribute_group_in_ir_value_match): 475 attr = match.group(attribute_group_in_ir_value_match) 476 else: 477 attr = '' 478 return match.group(1) + attr + match.group(match.lastindex) 479 def repl_arg_names(match): 480 if match.group(variable_group_in_ir_value_match) is not None and match.group(variable_group_in_ir_value_match) in arg_names: 481 return match.group(1) + match.group(match.lastindex) 482 return match.group(1) + match.group(2) + match.group(match.lastindex) 483 if self.funcdef_attrs_and_ret != funcdef_attrs_and_ret: 484 return False 485 if self.attrs != attrs: 486 return False 487 ans0 = IR_VALUE_RE.sub(drop_arg_names, self.args_and_sig) 488 ans1 = IR_VALUE_RE.sub(drop_arg_names, args_and_sig) 489 if ans0 != ans1: 490 return False 491 if is_backend: 492 # Check without replacements, the replacements are not applied to the 493 # body for backend checks. 494 return self.extrascrub == extrascrub 495 496 es0 = IR_VALUE_RE.sub(repl_arg_names, self.extrascrub) 497 es1 = IR_VALUE_RE.sub(repl_arg_names, extrascrub) 498 es0 = SCRUB_IR_COMMENT_RE.sub(r'', es0) 499 es1 = SCRUB_IR_COMMENT_RE.sub(r'', es1) 500 return es0 == es1 501 502 def __str__(self): 503 return self.scrub 504 505class FunctionTestBuilder: 506 def __init__(self, run_list, flags, scrubber_args, path): 507 self._verbose = flags.verbose 508 self._record_args = flags.function_signature 509 self._check_attributes = flags.check_attributes 510 # Strip double-quotes if input was read by UTC_ARGS 511 self._filters = list(map(lambda f: Filter(re.compile(f.pattern().strip('"'), 512 f.flags()), 513 f.is_filter_out), 514 flags.filters)) if flags.filters else [] 515 self._scrubber_args = scrubber_args 516 self._path = path 517 # Strip double-quotes if input was read by UTC_ARGS 518 self._replace_value_regex = list(map(lambda x: x.strip('"'), flags.replace_value_regex)) 519 self._func_dict = {} 520 self._func_order = {} 521 self._global_var_dict = {} 522 self._processed_prefixes = set() 523 for tuple in run_list: 524 for prefix in tuple[0]: 525 self._func_dict.update({prefix: dict()}) 526 self._func_order.update({prefix: []}) 527 self._global_var_dict.update({prefix: dict()}) 528 529 def finish_and_get_func_dict(self): 530 for prefix in self.get_failed_prefixes(): 531 warn('Prefix %s had conflicting output from different RUN lines for all functions in test %s' % (prefix,self._path,)) 532 return self._func_dict 533 534 def func_order(self): 535 return self._func_order 536 537 def global_var_dict(self): 538 return self._global_var_dict 539 540 def is_filtered(self): 541 return bool(self._filters) 542 543 def process_run_line(self, function_re, scrubber, raw_tool_output, prefixes, is_backend): 544 build_global_values_dictionary(self._global_var_dict, raw_tool_output, prefixes) 545 for m in function_re.finditer(raw_tool_output): 546 if not m: 547 continue 548 func = m.group('func') 549 body = m.group('body') 550 # func_name_separator is the string that is placed right after function name at the 551 # beginning of assembly function definition. In most assemblies, that is just a 552 # colon: `foo:`. But, for example, in nvptx it is a brace: `foo(`. If is_backend is 553 # False, just assume that separator is an empty string. 554 if is_backend: 555 # Use ':' as default separator. 556 func_name_separator = m.group('func_name_separator') if 'func_name_separator' in m.groupdict() else ':' 557 else: 558 func_name_separator = '' 559 attrs = m.group('attrs') if self._check_attributes else '' 560 funcdef_attrs_and_ret = m.group('funcdef_attrs_and_ret') if self._record_args else '' 561 # Determine if we print arguments, the opening brace, or nothing after the 562 # function name 563 if self._record_args and 'args_and_sig' in m.groupdict(): 564 args_and_sig = scrub_body(m.group('args_and_sig').strip()) 565 elif 'args_and_sig' in m.groupdict(): 566 args_and_sig = '(' 567 else: 568 args_and_sig = '' 569 filtered_body = do_filter(body, self._filters) 570 scrubbed_body = do_scrub(filtered_body, scrubber, self._scrubber_args, 571 extra=False) 572 scrubbed_extra = do_scrub(filtered_body, scrubber, self._scrubber_args, 573 extra=True) 574 if 'analysis' in m.groupdict(): 575 analysis = m.group('analysis') 576 if analysis.lower() != 'cost model analysis': 577 warn('Unsupported analysis mode: %r!' % (analysis,)) 578 if func.startswith('stress'): 579 # We only use the last line of the function body for stress tests. 580 scrubbed_body = '\n'.join(scrubbed_body.splitlines()[-1:]) 581 if self._verbose: 582 print('Processing function: ' + func, file=sys.stderr) 583 for l in scrubbed_body.splitlines(): 584 print(' ' + l, file=sys.stderr) 585 for prefix in prefixes: 586 # Replace function names matching the regex. 587 for regex in self._replace_value_regex: 588 # Pattern that matches capture groups in the regex in leftmost order. 589 group_regex = re.compile(r'\(.*?\)') 590 # Replace function name with regex. 591 match = re.match(regex, func) 592 if match: 593 func_repl = regex 594 # Replace any capture groups with their matched strings. 595 for g in match.groups(): 596 func_repl = group_regex.sub(re.escape(g), func_repl, count=1) 597 func = re.sub(func_repl, '{{' + func_repl + '}}', func) 598 599 # Replace all calls to regex matching functions. 600 matches = re.finditer(regex, scrubbed_body) 601 for match in matches: 602 func_repl = regex 603 # Replace any capture groups with their matched strings. 604 for g in match.groups(): 605 func_repl = group_regex.sub(re.escape(g), func_repl, count=1) 606 # Substitute function call names that match the regex with the same 607 # capture groups set. 608 scrubbed_body = re.sub(func_repl, '{{' + func_repl + '}}', 609 scrubbed_body) 610 611 if func in self._func_dict[prefix]: 612 if (self._func_dict[prefix][func] is not None and 613 (str(self._func_dict[prefix][func]) != scrubbed_body or 614 self._func_dict[prefix][func].args_and_sig != args_and_sig or 615 self._func_dict[prefix][func].attrs != attrs or 616 self._func_dict[prefix][func].funcdef_attrs_and_ret != funcdef_attrs_and_ret)): 617 if self._func_dict[prefix][func].is_same_except_arg_names( 618 scrubbed_extra, 619 funcdef_attrs_and_ret, 620 args_and_sig, 621 attrs, 622 is_backend): 623 self._func_dict[prefix][func].scrub = scrubbed_extra 624 self._func_dict[prefix][func].args_and_sig = args_and_sig 625 else: 626 # This means a previous RUN line produced a body for this function 627 # that is different from the one produced by this current RUN line, 628 # so the body can't be common across RUN lines. We use None to 629 # indicate that. 630 self._func_dict[prefix][func] = None 631 else: 632 if prefix not in self._processed_prefixes: 633 self._func_dict[prefix][func] = function_body( 634 scrubbed_body, scrubbed_extra, funcdef_attrs_and_ret, 635 args_and_sig, attrs, func_name_separator) 636 self._func_order[prefix].append(func) 637 else: 638 # An earlier RUN line used this check prefixes but didn't produce 639 # a body for this function. This happens in Clang tests that use 640 # preprocesser directives to exclude individual functions from some 641 # RUN lines. 642 self._func_dict[prefix][func] = None 643 644 def processed_prefixes(self, prefixes): 645 """ 646 Mark a set of prefixes as having had at least one applicable RUN line fully 647 processed. This is used to filter out function bodies that don't have 648 outputs for all RUN lines. 649 """ 650 self._processed_prefixes.update(prefixes) 651 652 def get_failed_prefixes(self): 653 # This returns the list of those prefixes that failed to match any function, 654 # because there were conflicting bodies produced by different RUN lines, in 655 # all instances of the prefix. 656 for prefix in self._func_dict: 657 if (self._func_dict[prefix] and 658 (not [fct for fct in self._func_dict[prefix] 659 if self._func_dict[prefix][fct] is not None])): 660 yield prefix 661 662 663##### Generator of LLVM IR CHECK lines 664 665SCRUB_IR_COMMENT_RE = re.compile(r'\s*;.*') 666 667# TODO: We should also derive check lines for global, debug, loop declarations, etc.. 668 669class NamelessValue: 670 def __init__(self, check_prefix, check_key, ir_prefix, ir_regexp, 671 global_ir_rhs_regexp, *, is_before_functions=False, is_number=False, 672 replace_number_with_counter=False): 673 self.check_prefix = check_prefix 674 self.check_key = check_key 675 self.ir_prefix = ir_prefix 676 self.ir_regexp = ir_regexp 677 self.global_ir_rhs_regexp = global_ir_rhs_regexp 678 self.is_before_functions = is_before_functions 679 self.is_number = is_number 680 # Some variable numbers (e.g. MCINST1234) will change based on unrelated 681 # modifications to LLVM, replace those with an incrementing counter. 682 self.replace_number_with_counter = replace_number_with_counter 683 self.variable_mapping = {} 684 685 # Return true if this kind of IR value is "local", basically if it matches '%{{.*}}'. 686 def is_local_def_ir_value_match(self, match): 687 return self.ir_prefix == '%' 688 689 # Return true if this kind of IR value is "global", basically if it matches '#{{.*}}'. 690 def is_global_scope_ir_value_match(self, match): 691 return self.global_ir_rhs_regexp is not None 692 693 # Return the IR prefix and check prefix we use for this kind or IR value, 694 # e.g., (%, TMP) for locals. 695 def get_ir_prefix_from_ir_value_match(self, match): 696 return self.ir_prefix, self.check_prefix 697 698 # Return the IR regexp we use for this kind or IR value, e.g., [\w.-]+? for locals 699 def get_ir_regex_from_ir_value_re_match(self, match): 700 # for backwards compatibility we check locals with '.*' 701 if self.is_local_def_ir_value_match(match): 702 return '.*' 703 return self.ir_regexp 704 705 # Create a FileCheck variable name based on an IR name. 706 def get_value_name(self, var: str, check_prefix: str): 707 var = var.replace('!', '') 708 if self.replace_number_with_counter: 709 assert var.isdigit(), var 710 replacement = self.variable_mapping.get(var, None) 711 if replacement is None: 712 # Replace variable with an incrementing counter 713 replacement = str(len(self.variable_mapping) + 1) 714 self.variable_mapping[var] = replacement 715 var = replacement 716 # This is a nameless value, prepend check_prefix. 717 if var.isdigit(): 718 var = check_prefix + var 719 else: 720 # This is a named value that clashes with the check_prefix, prepend with 721 # _prefix_filecheck_ir_name, if it has been defined. 722 if may_clash_with_default_check_prefix_name(check_prefix, var) and _prefix_filecheck_ir_name: 723 var = _prefix_filecheck_ir_name + var 724 var = var.replace('.', '_') 725 var = var.replace('-', '_') 726 return var.upper() 727 728 # Create a FileCheck variable from regex. 729 def get_value_definition(self, var, match): 730 # for backwards compatibility we check locals with '.*' 731 varname = self.get_value_name(var, self.check_prefix) 732 prefix = self.get_ir_prefix_from_ir_value_match(match)[0] 733 if self.is_number: 734 regex = '' # always capture a number in the default format 735 capture_start = '[[#' 736 else: 737 regex = self.get_ir_regex_from_ir_value_re_match(match) 738 capture_start = '[[' 739 if self.is_local_def_ir_value_match(match): 740 return capture_start + varname + ':' + prefix + regex + ']]' 741 return prefix + capture_start + varname + ':' + regex + ']]' 742 743 # Use a FileCheck variable. 744 def get_value_use(self, var, match, var_prefix=None): 745 if var_prefix is None: 746 var_prefix = self.check_prefix 747 capture_start = '[[#' if self.is_number else '[[' 748 if self.is_local_def_ir_value_match(match): 749 return capture_start + self.get_value_name(var, var_prefix) + ']]' 750 prefix = self.get_ir_prefix_from_ir_value_match(match)[0] 751 return prefix + capture_start + self.get_value_name(var, var_prefix) + ']]' 752 753# Description of the different "unnamed" values we match in the IR, e.g., 754# (local) ssa values, (debug) metadata, etc. 755ir_nameless_values = [ 756 # check_prefix check_key ir_prefix ir_regexp global_ir_rhs_regexp 757 NamelessValue(r'TMP' , '%' , r'%' , r'[\w$.-]+?' , None ) , 758 NamelessValue(r'ATTR' , '#' , r'#' , r'[0-9]+' , None ) , 759 NamelessValue(r'ATTR' , '#' , r'attributes #' , r'[0-9]+' , r'{[^}]*}' ) , 760 NamelessValue(r'GLOB' , '@' , r'@' , r'[0-9]+' , None ) , 761 NamelessValue(r'GLOB' , '@' , r'@' , r'[a-zA-Z0-9_$"\\.-]+' , r'.+' , is_before_functions=True) , 762 NamelessValue(r'DBG' , '!' , r'!dbg ' , r'![0-9]+' , None ) , 763 NamelessValue(r'DIASSIGNID' , '!' , r'!DIAssignID ' , r'![0-9]+' , None ) , 764 NamelessValue(r'PROF' , '!' , r'!prof ' , r'![0-9]+' , None ) , 765 NamelessValue(r'TBAA' , '!' , r'!tbaa ' , r'![0-9]+' , None ) , 766 NamelessValue(r'TBAA_STRUCT', '!' , r'!tbaa.struct ' , r'![0-9]+' , None ) , 767 NamelessValue(r'RNG' , '!' , r'!range ' , r'![0-9]+' , None ) , 768 NamelessValue(r'LOOP' , '!' , r'!llvm.loop ' , r'![0-9]+' , None ) , 769 NamelessValue(r'META' , '!' , r'metadata ' , r'![0-9]+' , None ) , 770 NamelessValue(r'META' , '!' , r'' , r'![0-9]+' , r'(?:distinct |)!.*' ) , 771 NamelessValue(r'ACC_GRP' , '!' , r'!llvm.access.group ' , r'![0-9]+' , None ) , 772] 773 774asm_nameless_values = [ 775 NamelessValue(r'MCINST' , 'Inst#' , '<MCInst #' , r'\d+' , r'.+', is_number=True, replace_number_with_counter=True), 776 NamelessValue(r'MCREG' , 'Reg:' , '<MCOperand Reg:' , r'\d+' , r'.+', is_number=True, replace_number_with_counter=True), 777] 778 779def createOrRegexp(old, new): 780 if not old: 781 return new 782 if not new: 783 return old 784 return old + '|' + new 785 786def createPrefixMatch(prefix_str, prefix_re): 787 return '(?:' + prefix_str + '(' + prefix_re + '))' 788 789# Build the regexp that matches an "IR value". This can be a local variable, 790# argument, global, or metadata, anything that is "named". It is important that 791# the PREFIX and SUFFIX below only contain a single group, if that changes 792# other locations will need adjustment as well. 793IR_VALUE_REGEXP_PREFIX = r'(\s*)' 794IR_VALUE_REGEXP_STRING = r'' 795for nameless_value in ir_nameless_values: 796 match = createPrefixMatch(nameless_value.ir_prefix, nameless_value.ir_regexp) 797 if nameless_value.global_ir_rhs_regexp is not None: 798 match = '^' + match 799 IR_VALUE_REGEXP_STRING = createOrRegexp(IR_VALUE_REGEXP_STRING, match) 800IR_VALUE_REGEXP_SUFFIX = r'([,\s\(\)]|\Z)' 801IR_VALUE_RE = re.compile(IR_VALUE_REGEXP_PREFIX + r'(' + IR_VALUE_REGEXP_STRING + r')' + IR_VALUE_REGEXP_SUFFIX) 802 803# Build the regexp that matches an "ASM value" (currently only for --asm-show-inst comments). 804ASM_VALUE_REGEXP_STRING = '' 805for nameless_value in asm_nameless_values: 806 match = createPrefixMatch(nameless_value.ir_prefix, nameless_value.ir_regexp) 807 ASM_VALUE_REGEXP_STRING = createOrRegexp(ASM_VALUE_REGEXP_STRING, match) 808ASM_VALUE_REGEXP_SUFFIX = r'([>\s]|\Z)' 809ASM_VALUE_RE = re.compile(r'((?:#|//)\s*)' + '(' + ASM_VALUE_REGEXP_STRING + ')' + ASM_VALUE_REGEXP_SUFFIX) 810 811# The entire match is group 0, the prefix has one group (=1), the entire 812# IR_VALUE_REGEXP_STRING is one group (=2), and then the nameless values start. 813first_nameless_group_in_ir_value_match = 3 814 815# constants for the group id of special matches 816variable_group_in_ir_value_match = 3 817attribute_group_in_ir_value_match = 4 818 819# Check a match for IR_VALUE_RE and inspect it to determine if it was a local 820# value, %..., global @..., debug number !dbg !..., etc. See the PREFIXES above. 821def get_idx_from_ir_value_match(match): 822 for i in range(first_nameless_group_in_ir_value_match, match.lastindex): 823 if match.group(i) is not None: 824 return i - first_nameless_group_in_ir_value_match 825 error("Unable to identify the kind of IR value from the match!") 826 return 0 827 828# See get_idx_from_ir_value_match 829def get_name_from_ir_value_match(match): 830 return match.group(get_idx_from_ir_value_match(match) + first_nameless_group_in_ir_value_match) 831 832def get_nameless_value_from_match(match, nameless_values) -> NamelessValue: 833 return nameless_values[get_idx_from_ir_value_match(match)] 834 835# Return true if var clashes with the scripted FileCheck check_prefix. 836def may_clash_with_default_check_prefix_name(check_prefix, var): 837 return check_prefix and re.match(r'^' + check_prefix + r'[0-9]+?$', var, re.IGNORECASE) 838 839def generalize_check_lines_common(lines, is_analyze, vars_seen, 840 global_vars_seen, nameless_values, 841 nameless_value_regex, is_asm): 842 # This gets called for each match that occurs in 843 # a line. We transform variables we haven't seen 844 # into defs, and variables we have seen into uses. 845 def transform_line_vars(match): 846 var = get_name_from_ir_value_match(match) 847 nameless_value = get_nameless_value_from_match(match, nameless_values) 848 if may_clash_with_default_check_prefix_name(nameless_value.check_prefix, var): 849 warn("Change IR value name '%s' or use --prefix-filecheck-ir-name to prevent possible conflict" 850 " with scripted FileCheck name." % (var,)) 851 key = (var, nameless_value.check_key) 852 is_local_def = nameless_value.is_local_def_ir_value_match(match) 853 if is_local_def and key in vars_seen: 854 rv = nameless_value.get_value_use(var, match) 855 elif not is_local_def and key in global_vars_seen: 856 # We could have seen a different prefix for the global variables first, 857 # ensure we use that one instead of the prefix for the current match. 858 rv = nameless_value.get_value_use(var, match, global_vars_seen[key]) 859 else: 860 if is_local_def: 861 vars_seen.add(key) 862 else: 863 global_vars_seen[key] = nameless_value.check_prefix 864 rv = nameless_value.get_value_definition(var, match) 865 # re.sub replaces the entire regex match 866 # with whatever you return, so we have 867 # to make sure to hand it back everything 868 # including the commas and spaces. 869 return match.group(1) + rv + match.group(match.lastindex) 870 871 lines_with_def = [] 872 873 for i, line in enumerate(lines): 874 if not is_asm: 875 # An IR variable named '%.' matches the FileCheck regex string. 876 line = line.replace('%.', '%dot') 877 for regex in _global_hex_value_regex: 878 if re.match('^@' + regex + ' = ', line): 879 line = re.sub(r'\bi([0-9]+) ([0-9]+)', 880 lambda m : 'i' + m.group(1) + ' [[#' + hex(int(m.group(2))) + ']]', 881 line) 882 break 883 # Ignore any comments, since the check lines will too. 884 scrubbed_line = SCRUB_IR_COMMENT_RE.sub(r'', line) 885 lines[i] = scrubbed_line 886 if is_asm or not is_analyze: 887 # It can happen that two matches are back-to-back and for some reason sub 888 # will not replace both of them. For now we work around this by 889 # substituting until there is no more match. 890 changed = True 891 while changed: 892 (lines[i], changed) = nameless_value_regex.subn(transform_line_vars, 893 lines[i], count=1) 894 return lines 895 896# Replace IR value defs and uses with FileCheck variables. 897def generalize_check_lines(lines, is_analyze, vars_seen, global_vars_seen): 898 return generalize_check_lines_common(lines, is_analyze, vars_seen, 899 global_vars_seen, ir_nameless_values, 900 IR_VALUE_RE, False) 901 902def generalize_asm_check_lines(lines, vars_seen, global_vars_seen): 903 return generalize_check_lines_common(lines, False, vars_seen, 904 global_vars_seen, asm_nameless_values, 905 ASM_VALUE_RE, True) 906 907def add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name, check_label_format, is_backend, is_analyze, version, global_vars_seen_dict, is_filtered): 908 # prefix_exclusions are prefixes we cannot use to print the function because it doesn't exist in run lines that use these prefixes as well. 909 prefix_exclusions = set() 910 printed_prefixes = [] 911 for p in prefix_list: 912 checkprefixes = p[0] 913 # If not all checkprefixes of this run line produced the function we cannot check for it as it does not 914 # exist for this run line. A subset of the check prefixes might know about the function but only because 915 # other run lines created it. 916 if any(map(lambda checkprefix: func_name not in func_dict[checkprefix], checkprefixes)): 917 prefix_exclusions |= set(checkprefixes) 918 continue 919 920 # prefix_exclusions is constructed, we can now emit the output 921 for p in prefix_list: 922 global_vars_seen = {} 923 checkprefixes = p[0] 924 for checkprefix in checkprefixes: 925 if checkprefix in global_vars_seen_dict: 926 global_vars_seen.update(global_vars_seen_dict[checkprefix]) 927 else: 928 global_vars_seen_dict[checkprefix] = {} 929 if checkprefix in printed_prefixes: 930 break 931 932 # Check if the prefix is excluded. 933 if checkprefix in prefix_exclusions: 934 continue 935 936 # If we do not have output for this prefix we skip it. 937 if not func_dict[checkprefix][func_name]: 938 continue 939 940 # Add some space between different check prefixes, but not after the last 941 # check line (before the test code). 942 if is_backend: 943 if len(printed_prefixes) != 0: 944 output_lines.append(comment_marker) 945 946 if checkprefix not in global_vars_seen_dict: 947 global_vars_seen_dict[checkprefix] = {} 948 949 global_vars_seen_before = [key for key in global_vars_seen.keys()] 950 951 vars_seen = set() 952 printed_prefixes.append(checkprefix) 953 attrs = str(func_dict[checkprefix][func_name].attrs) 954 attrs = '' if attrs == 'None' else attrs 955 if version > 1: 956 funcdef_attrs_and_ret = func_dict[checkprefix][func_name].funcdef_attrs_and_ret 957 else: 958 funcdef_attrs_and_ret = '' 959 960 if attrs: 961 output_lines.append('%s %s: Function Attrs: %s' % (comment_marker, checkprefix, attrs)) 962 args_and_sig = str(func_dict[checkprefix][func_name].args_and_sig) 963 if args_and_sig: 964 args_and_sig = generalize_check_lines([args_and_sig], is_analyze, vars_seen, global_vars_seen)[0] 965 func_name_separator = func_dict[checkprefix][func_name].func_name_separator 966 if '[[' in args_and_sig: 967 output_lines.append(check_label_format % (checkprefix, funcdef_attrs_and_ret, func_name, '', func_name_separator)) 968 output_lines.append('%s %s-SAME: %s' % (comment_marker, checkprefix, args_and_sig)) 969 else: 970 output_lines.append(check_label_format % (checkprefix, funcdef_attrs_and_ret, func_name, args_and_sig, func_name_separator)) 971 func_body = str(func_dict[checkprefix][func_name]).splitlines() 972 if not func_body: 973 # We have filtered everything. 974 continue 975 976 # For ASM output, just emit the check lines. 977 if is_backend: 978 body_start = 1 979 if is_filtered: 980 # For filtered output we don't add "-NEXT" so don't add extra spaces 981 # before the first line. 982 body_start = 0 983 else: 984 output_lines.append('%s %s: %s' % (comment_marker, checkprefix, func_body[0])) 985 func_lines = generalize_asm_check_lines(func_body[body_start:], 986 vars_seen, global_vars_seen) 987 for func_line in func_lines: 988 if func_line.strip() == '': 989 output_lines.append('%s %s-EMPTY:' % (comment_marker, checkprefix)) 990 else: 991 check_suffix = '-NEXT' if not is_filtered else '' 992 output_lines.append('%s %s%s: %s' % (comment_marker, checkprefix, 993 check_suffix, func_line)) 994 # Remember new global variables we have not seen before 995 for key in global_vars_seen: 996 if key not in global_vars_seen_before: 997 global_vars_seen_dict[checkprefix][key] = global_vars_seen[key] 998 break 999 1000 # For IR output, change all defs to FileCheck variables, so we're immune 1001 # to variable naming fashions. 1002 func_body = generalize_check_lines(func_body, is_analyze, vars_seen, global_vars_seen) 1003 1004 # This could be selectively enabled with an optional invocation argument. 1005 # Disabled for now: better to check everything. Be safe rather than sorry. 1006 1007 # Handle the first line of the function body as a special case because 1008 # it's often just noise (a useless asm comment or entry label). 1009 #if func_body[0].startswith("#") or func_body[0].startswith("entry:"): 1010 # is_blank_line = True 1011 #else: 1012 # output_lines.append('%s %s: %s' % (comment_marker, checkprefix, func_body[0])) 1013 # is_blank_line = False 1014 1015 is_blank_line = False 1016 1017 for func_line in func_body: 1018 if func_line.strip() == '': 1019 is_blank_line = True 1020 continue 1021 # Do not waste time checking IR comments. 1022 func_line = SCRUB_IR_COMMENT_RE.sub(r'', func_line) 1023 1024 # Skip blank lines instead of checking them. 1025 if is_blank_line: 1026 output_lines.append('{} {}: {}'.format( 1027 comment_marker, checkprefix, func_line)) 1028 else: 1029 check_suffix = '-NEXT' if not is_filtered else '' 1030 output_lines.append('{} {}{}: {}'.format( 1031 comment_marker, checkprefix, check_suffix, func_line)) 1032 is_blank_line = False 1033 1034 # Add space between different check prefixes and also before the first 1035 # line of code in the test function. 1036 output_lines.append(comment_marker) 1037 1038 # Remember new global variables we have not seen before 1039 for key in global_vars_seen: 1040 if key not in global_vars_seen_before: 1041 global_vars_seen_dict[checkprefix][key] = global_vars_seen[key] 1042 break 1043 return printed_prefixes 1044 1045def add_ir_checks(output_lines, comment_marker, prefix_list, func_dict, 1046 func_name, preserve_names, function_sig, version, 1047 global_vars_seen_dict, is_filtered): 1048 # Label format is based on IR string. 1049 if function_sig and version > 1: 1050 function_def_regex = 'define %s' 1051 elif function_sig: 1052 function_def_regex = 'define {{[^@]+}}%s' 1053 else: 1054 function_def_regex = '%s' 1055 check_label_format = '{} %s-LABEL: {}@%s%s%s'.format(comment_marker, function_def_regex) 1056 return add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name, 1057 check_label_format, False, preserve_names, version, 1058 global_vars_seen_dict, 1059 is_filtered) 1060 1061def add_analyze_checks(output_lines, comment_marker, prefix_list, func_dict, func_name, is_filtered): 1062 check_label_format = '{} %s-LABEL: \'%s%s%s%s\''.format(comment_marker) 1063 global_vars_seen_dict = {} 1064 return add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name, 1065 check_label_format, False, True, 1, global_vars_seen_dict, 1066 is_filtered) 1067 1068def build_global_values_dictionary(glob_val_dict, raw_tool_output, prefixes): 1069 for nameless_value in itertools.chain(ir_nameless_values, asm_nameless_values): 1070 if nameless_value.global_ir_rhs_regexp is None: 1071 continue 1072 1073 lhs_re_str = nameless_value.ir_prefix + nameless_value.ir_regexp 1074 rhs_re_str = nameless_value.global_ir_rhs_regexp 1075 1076 global_ir_value_re_str = r'^' + lhs_re_str + r'\s=\s' + rhs_re_str + r'$' 1077 global_ir_value_re = re.compile(global_ir_value_re_str, flags=(re.M)) 1078 lines = [] 1079 for m in global_ir_value_re.finditer(raw_tool_output): 1080 lines.append(m.group(0)) 1081 1082 for prefix in prefixes: 1083 if glob_val_dict[prefix] is None: 1084 continue 1085 if nameless_value.check_prefix in glob_val_dict[prefix]: 1086 if lines == glob_val_dict[prefix][nameless_value.check_prefix]: 1087 continue 1088 if prefix == prefixes[-1]: 1089 warn('Found conflicting asm under the same prefix: %r!' % (prefix,)) 1090 else: 1091 glob_val_dict[prefix][nameless_value.check_prefix] = None 1092 continue 1093 glob_val_dict[prefix][nameless_value.check_prefix] = lines 1094 1095def add_global_checks(glob_val_dict, comment_marker, prefix_list, output_lines, global_vars_seen_dict, is_analyze, is_before_functions): 1096 printed_prefixes = set() 1097 for nameless_value in ir_nameless_values: 1098 if nameless_value.global_ir_rhs_regexp is None: 1099 continue 1100 if nameless_value.is_before_functions != is_before_functions: 1101 continue 1102 for p in prefix_list: 1103 global_vars_seen = {} 1104 checkprefixes = p[0] 1105 if checkprefixes is None: 1106 continue 1107 for checkprefix in checkprefixes: 1108 if checkprefix in global_vars_seen_dict: 1109 global_vars_seen.update(global_vars_seen_dict[checkprefix]) 1110 else: 1111 global_vars_seen_dict[checkprefix] = {} 1112 if (checkprefix, nameless_value.check_prefix) in printed_prefixes: 1113 break 1114 if not glob_val_dict[checkprefix]: 1115 continue 1116 if nameless_value.check_prefix not in glob_val_dict[checkprefix]: 1117 continue 1118 if not glob_val_dict[checkprefix][nameless_value.check_prefix]: 1119 continue 1120 1121 check_lines = [] 1122 global_vars_seen_before = [key for key in global_vars_seen.keys()] 1123 for line in glob_val_dict[checkprefix][nameless_value.check_prefix]: 1124 if _global_value_regex: 1125 matched = False 1126 for regex in _global_value_regex: 1127 if re.match('^@' + regex + ' = ', line): 1128 matched = True 1129 break 1130 if not matched: 1131 continue 1132 tmp = generalize_check_lines([line], is_analyze, set(), global_vars_seen) 1133 check_line = '%s %s: %s' % (comment_marker, checkprefix, tmp[0]) 1134 check_lines.append(check_line) 1135 if not check_lines: 1136 continue 1137 1138 output_lines.append(comment_marker + SEPARATOR) 1139 for check_line in check_lines: 1140 output_lines.append(check_line) 1141 1142 printed_prefixes.add((checkprefix, nameless_value.check_prefix)) 1143 1144 # Remembe new global variables we have not seen before 1145 for key in global_vars_seen: 1146 if key not in global_vars_seen_before: 1147 global_vars_seen_dict[checkprefix][key] = global_vars_seen[key] 1148 break 1149 1150 if printed_prefixes: 1151 output_lines.append(comment_marker + SEPARATOR) 1152 return printed_prefixes 1153 1154 1155def check_prefix(prefix): 1156 if not PREFIX_RE.match(prefix): 1157 hint = "" 1158 if ',' in prefix: 1159 hint = " Did you mean '--check-prefixes=" + prefix + "'?" 1160 warn(("Supplied prefix '%s' is invalid. Prefix must contain only alphanumeric characters, hyphens and underscores." + hint) % 1161 (prefix)) 1162 1163def get_check_prefixes(filecheck_cmd): 1164 check_prefixes = [item for m in CHECK_PREFIX_RE.finditer(filecheck_cmd) 1165 for item in m.group(1).split(',')] 1166 if not check_prefixes: 1167 check_prefixes = ['CHECK'] 1168 return check_prefixes 1169 1170def verify_filecheck_prefixes(fc_cmd): 1171 fc_cmd_parts = fc_cmd.split() 1172 for part in fc_cmd_parts: 1173 if "check-prefix=" in part: 1174 prefix = part.split('=', 1)[1] 1175 check_prefix(prefix) 1176 elif "check-prefixes=" in part: 1177 prefixes = part.split('=', 1)[1].split(',') 1178 for prefix in prefixes: 1179 check_prefix(prefix) 1180 if prefixes.count(prefix) > 1: 1181 warn("Supplied prefix '%s' is not unique in the prefix list." % (prefix,)) 1182 1183 1184def get_autogennote_suffix(parser, args): 1185 autogenerated_note_args = '' 1186 for action in parser._actions: 1187 if not hasattr(args, action.dest): 1188 continue # Ignore options such as --help that aren't included in args 1189 # Ignore parameters such as paths to the binary or the list of tests 1190 if action.dest in ('tests', 'update_only', 'tool_binary', 'opt_binary', 1191 'llc_binary', 'clang', 'opt', 'llvm_bin', 'verbose', 1192 'force_update'): 1193 continue 1194 value = getattr(args, action.dest) 1195 if action.const is not None: # action stores a constant (usually True/False) 1196 # Skip actions with different constant values (this happens with boolean 1197 # --foo/--no-foo options) 1198 if value != action.const: 1199 continue 1200 if parser.get_default(action.dest) == value: 1201 continue # Don't add default values 1202 if action.dest == 'filters': 1203 # Create a separate option for each filter element. The value is a list 1204 # of Filter objects. 1205 for elem in value: 1206 opt_name = 'filter-out' if elem.is_filter_out else 'filter' 1207 opt_value = elem.pattern() 1208 new_arg = '--%s "%s" ' % (opt_name, opt_value.strip('"')) 1209 if new_arg not in autogenerated_note_args: 1210 autogenerated_note_args += new_arg 1211 else: 1212 autogenerated_note_args += action.option_strings[0] + ' ' 1213 if action.const is None: # action takes a parameter 1214 if action.nargs == '+': 1215 value = ' '.join(map(lambda v: '"' + v.strip('"') + '"', value)) 1216 autogenerated_note_args += '%s ' % value 1217 if autogenerated_note_args: 1218 autogenerated_note_args = ' %s %s' % (UTC_ARGS_KEY, autogenerated_note_args[:-1]) 1219 return autogenerated_note_args 1220 1221 1222def check_for_command(line, parser, args, argv, argparse_callback): 1223 cmd_m = UTC_ARGS_CMD.match(line) 1224 if cmd_m: 1225 for option in shlex.split(cmd_m.group('cmd').strip()): 1226 if option: 1227 argv.append(option) 1228 args = parser.parse_args(filter(lambda arg: arg not in args.tests, argv)) 1229 if argparse_callback is not None: 1230 argparse_callback(args) 1231 return args, argv 1232 1233def find_arg_in_test(test_info, get_arg_to_check, arg_string, is_global): 1234 result = get_arg_to_check(test_info.args) 1235 if not result and is_global: 1236 # See if this has been specified via UTC_ARGS. This is a "global" option 1237 # that affects the entire generation of test checks. If it exists anywhere 1238 # in the test, apply it to everything. 1239 saw_line = False 1240 for line_info in test_info.ro_iterlines(): 1241 line = line_info.line 1242 if not line.startswith(';') and line.strip() != '': 1243 saw_line = True 1244 result = get_arg_to_check(line_info.args) 1245 if result: 1246 if warn and saw_line: 1247 # We saw the option after already reading some test input lines. 1248 # Warn about it. 1249 print('WARNING: Found {} in line following test start: '.format(arg_string) 1250 + line, file=sys.stderr) 1251 print('WARNING: Consider moving {} to top of file'.format(arg_string), 1252 file=sys.stderr) 1253 break 1254 return result 1255 1256def dump_input_lines(output_lines, test_info, prefix_set, comment_string): 1257 for input_line_info in test_info.iterlines(output_lines): 1258 line = input_line_info.line 1259 args = input_line_info.args 1260 if line.strip() == comment_string: 1261 continue 1262 if line.strip() == comment_string + SEPARATOR: 1263 continue 1264 if line.lstrip().startswith(comment_string): 1265 m = CHECK_RE.match(line) 1266 if m and m.group(1) in prefix_set: 1267 continue 1268 output_lines.append(line.rstrip('\n')) 1269 1270def add_checks_at_end(output_lines, prefix_list, func_order, 1271 comment_string, check_generator): 1272 added = set() 1273 generated_prefixes = set() 1274 for prefix in prefix_list: 1275 prefixes = prefix[0] 1276 tool_args = prefix[1] 1277 for prefix in prefixes: 1278 for func in func_order[prefix]: 1279 # The func order can contain the same functions multiple times. 1280 # If we see one again we are done. 1281 if (func, prefix) in added: 1282 continue 1283 if added: 1284 output_lines.append(comment_string) 1285 1286 # The add_*_checks routines expect a run list whose items are 1287 # tuples that have a list of prefixes as their first element and 1288 # tool command args string as their second element. They output 1289 # checks for each prefix in the list of prefixes. By doing so, it 1290 # implicitly assumes that for each function every run line will 1291 # generate something for that function. That is not the case for 1292 # generated functions as some run lines might not generate them 1293 # (e.g. -fopenmp vs. no -fopenmp). 1294 # 1295 # Therefore, pass just the prefix we're interested in. This has 1296 # the effect of generating all of the checks for functions of a 1297 # single prefix before moving on to the next prefix. So checks 1298 # are ordered by prefix instead of by function as in "normal" 1299 # mode. 1300 for generated_prefix in check_generator(output_lines, 1301 [([prefix], tool_args)], func): 1302 added.add((func, generated_prefix)) 1303 generated_prefixes.add(generated_prefix) 1304 return generated_prefixes 1305