xref: /openbsd-src/gnu/llvm/llvm/utils/UpdateTestChecks/common.py (revision d415bd752c734aee168c4ee86ff32e8cc249eb16) 
1from __future__ import print_function
2
3import argparse
4import copy
5import glob
6import itertools
7import os
8import re
9import subprocess
10import sys
11import shlex
12
13from typing import List
14
15##### Common utilities for update_*test_checks.py
16
17
18_verbose = False
19_prefix_filecheck_ir_name = ''
20
21class Regex(object):
22  """Wrap a compiled regular expression object to allow deep copy of a regexp.
23  This is required for the deep copy done in do_scrub.
24
25  """
26  def __init__(self, regex):
27    self.regex = regex
28
29  def __deepcopy__(self, memo):
30    result = copy.copy(self)
31    result.regex = self.regex
32    return result
33
34  def search(self, line):
35    return self.regex.search(line)
36
37  def sub(self, repl, line):
38    return self.regex.sub(repl, line)
39
40  def pattern(self):
41    return self.regex.pattern
42
43  def flags(self):
44    return self.regex.flags
45
46class Filter(Regex):
47  """Augment a Regex object with a flag indicating whether a match should be
48    added (!is_filter_out) or removed (is_filter_out) from the generated checks.
49
50  """
51  def __init__(self, regex, is_filter_out):
52    super(Filter, self).__init__(regex)
53    self.is_filter_out = is_filter_out
54
55  def __deepcopy__(self, memo):
56    result = copy.deepcopy(super(Filter, self), memo)
57    result.is_filter_out = copy.deepcopy(self.is_filter_out, memo)
58    return result
59
60def parse_commandline_args(parser):
61  class RegexAction(argparse.Action):
62    """Add a regular expression option value to a list of regular expressions.
63    This compiles the expression, wraps it in a Regex and adds it to the option
64    value list."""
65    def __init__(self, option_strings, dest, nargs=None, **kwargs):
66      if nargs is not None:
67        raise ValueError('nargs not allowed')
68      super(RegexAction, self).__init__(option_strings, dest, **kwargs)
69
70    def do_call(self, namespace, values, flags):
71      value_list = getattr(namespace, self.dest)
72      if value_list is None:
73        value_list = []
74
75      try:
76        value_list.append(Regex(re.compile(values, flags)))
77      except re.error as error:
78        raise ValueError('{}: Invalid regular expression \'{}\' ({})'.format(
79          option_string, error.pattern, error.msg))
80
81      setattr(namespace, self.dest, value_list)
82
83    def __call__(self, parser, namespace, values, option_string=None):
84      self.do_call(namespace, values, 0)
85
86  class FilterAction(RegexAction):
87    """Add a filter to a list of filter option values."""
88    def __init__(self, option_strings, dest, nargs=None, **kwargs):
89      super(FilterAction, self).__init__(option_strings, dest, nargs, **kwargs)
90
91    def __call__(self, parser, namespace, values, option_string=None):
92      super(FilterAction, self).__call__(parser, namespace, values, option_string)
93
94      value_list = getattr(namespace, self.dest)
95
96      is_filter_out = ( option_string == '--filter-out' )
97
98      value_list[-1] = Filter(value_list[-1].regex, is_filter_out)
99
100      setattr(namespace, self.dest, value_list)
101
102  filter_group = parser.add_argument_group(
103    'filtering',
104    """Filters are applied to each output line according to the order given. The
105    first matching filter terminates filter processing for that current line.""")
106
107  filter_group.add_argument('--filter', action=FilterAction, dest='filters',
108                            metavar='REGEX',
109                            help='Only include lines matching REGEX (may be specified multiple times)')
110  filter_group.add_argument('--filter-out', action=FilterAction, dest='filters',
111                            metavar='REGEX',
112                            help='Exclude lines matching REGEX')
113
114  parser.add_argument('--include-generated-funcs', action='store_true',
115                      help='Output checks for functions not in source')
116  parser.add_argument('-v', '--verbose', action='store_true',
117                      help='Show verbose output')
118  parser.add_argument('-u', '--update-only', action='store_true',
119                      help='Only update test if it was already autogened')
120  parser.add_argument('--force-update', action='store_true',
121                      help='Update test even if it was autogened by a different script')
122  parser.add_argument('--enable', action='store_true', dest='enabled', default=True,
123                       help='Activate CHECK line generation from this point forward')
124  parser.add_argument('--disable', action='store_false', dest='enabled',
125                      help='Deactivate CHECK line generation from this point forward')
126  parser.add_argument('--replace-value-regex', nargs='+', default=[],
127                      help='List of regular expressions to replace matching value names')
128  parser.add_argument('--prefix-filecheck-ir-name', default='',
129                      help='Add a prefix to FileCheck IR value names to avoid conflicts with scripted names')
130  parser.add_argument('--global-value-regex', nargs='+', default=[],
131                      help='List of regular expressions that a global value declaration must match to generate a check (has no effect if checking globals is not enabled)')
132  parser.add_argument('--global-hex-value-regex', nargs='+', default=[],
133                      help='List of regular expressions such that, for matching global value declarations, literal integer values should be encoded in hex in the associated FileCheck directives')
134  # FIXME: in 3.9, we can use argparse.BooleanOptionalAction. At that point,
135  # we need to rename the flag to just -generate-body-for-unused-prefixes.
136  parser.add_argument('--no-generate-body-for-unused-prefixes',
137                      action='store_false',
138                      dest='gen_unused_prefix_body',
139                      default=True,
140                      help='Generate a function body that always matches for unused prefixes. This is useful when unused prefixes are desired, and it avoids needing to annotate each FileCheck as allowing them.')
141  args = parser.parse_args()
142  global _verbose, _global_value_regex, _global_hex_value_regex
143  _verbose = args.verbose
144  _global_value_regex = args.global_value_regex
145  _global_hex_value_regex = args.global_hex_value_regex
146  return args
147
148
149class InputLineInfo(object):
150  def __init__(self, line, line_number, args, argv):
151    self.line = line
152    self.line_number = line_number
153    self.args = args
154    self.argv = argv
155
156
157class TestInfo(object):
158  def __init__(self, test, parser, script_name, input_lines, args, argv,
159               comment_prefix, argparse_callback):
160    self.parser = parser
161    self.argparse_callback = argparse_callback
162    self.path = test
163    self.args = args
164    if args.prefix_filecheck_ir_name:
165      global _prefix_filecheck_ir_name
166      _prefix_filecheck_ir_name = args.prefix_filecheck_ir_name
167    self.argv = argv
168    self.input_lines = input_lines
169    self.run_lines = find_run_lines(test, self.input_lines)
170    self.comment_prefix = comment_prefix
171    if self.comment_prefix is None:
172      if self.path.endswith('.mir'):
173        self.comment_prefix = '#'
174      else:
175        self.comment_prefix = ';'
176    self.autogenerated_note_prefix = self.comment_prefix + ' ' + UTC_ADVERT
177    self.test_autogenerated_note = self.autogenerated_note_prefix + script_name
178    self.test_autogenerated_note += get_autogennote_suffix(parser, self.args)
179    self.test_unused_note = self.comment_prefix + self.comment_prefix + ' ' + UNUSED_NOTE
180
181  def ro_iterlines(self):
182    for line_num, input_line in enumerate(self.input_lines):
183      args, argv = check_for_command(input_line, self.parser,
184                                     self.args, self.argv, self.argparse_callback)
185      yield InputLineInfo(input_line, line_num, args, argv)
186
187  def iterlines(self, output_lines):
188    output_lines.append(self.test_autogenerated_note)
189    for line_info in self.ro_iterlines():
190      input_line = line_info.line
191      # Discard any previous script advertising.
192      if input_line.startswith(self.autogenerated_note_prefix):
193        continue
194      self.args = line_info.args
195      self.argv = line_info.argv
196      if not self.args.enabled:
197        output_lines.append(input_line)
198        continue
199      yield line_info
200
201  def get_checks_for_unused_prefixes(self, run_list, used_prefixes: List[str]) -> List[str]:
202    run_list = [element for element in run_list if element[0] is not None]
203    unused_prefixes = set([
204        prefix for sublist in run_list for prefix in sublist[0]
205    ]).difference(set(used_prefixes))
206
207    ret = []
208    if not unused_prefixes:
209      return ret
210    ret.append(self.test_unused_note)
211    for unused in sorted(unused_prefixes):
212      ret.append('{comment} {prefix}: {match_everything}'.format(
213        comment=self.comment_prefix,
214        prefix=unused,
215        match_everything=r"""{{.*}}"""
216      ))
217    return ret
218
219def itertests(test_patterns, parser, script_name, comment_prefix=None, argparse_callback=None):
220  for pattern in test_patterns:
221    # On Windows we must expand the patterns ourselves.
222    tests_list = glob.glob(pattern)
223    if not tests_list:
224      warn("Test file pattern '%s' was not found. Ignoring it." % (pattern,))
225      continue
226    for test in tests_list:
227      with open(test) as f:
228        input_lines = [l.rstrip() for l in f]
229      args = parser.parse_args()
230      if argparse_callback is not None:
231        argparse_callback(args)
232      argv = sys.argv[:]
233      first_line = input_lines[0] if input_lines else ""
234      if UTC_ADVERT in first_line:
235        if script_name not in first_line and not args.force_update:
236          warn("Skipping test which wasn't autogenerated by " + script_name, test)
237          continue
238        args, argv = check_for_command(first_line, parser, args, argv, argparse_callback)
239      elif args.update_only:
240        assert UTC_ADVERT not in first_line
241        warn("Skipping test which isn't autogenerated: " + test)
242        continue
243      final_input_lines = []
244      for l in input_lines:
245        if UNUSED_NOTE in l:
246          break
247        final_input_lines.append(l)
248      yield TestInfo(test, parser, script_name, final_input_lines, args, argv,
249                     comment_prefix, argparse_callback)
250
251
252def should_add_line_to_output(input_line, prefix_set, skip_global_checks = False, comment_marker = ';'):
253  # Skip any blank comment lines in the IR.
254  if not skip_global_checks and input_line.strip() == comment_marker:
255    return False
256  # Skip a special double comment line we use as a separator.
257  if input_line.strip() == comment_marker + SEPARATOR:
258    return False
259  # Skip any blank lines in the IR.
260  #if input_line.strip() == '':
261  #  return False
262  # And skip any CHECK lines. We're building our own.
263  m = CHECK_RE.match(input_line)
264  if m and m.group(1) in prefix_set:
265    if skip_global_checks:
266      global_ir_value_re = re.compile(r'\[\[', flags=(re.M))
267      return not global_ir_value_re.search(input_line)
268    return False
269
270  return True
271
272# Perform lit-like substitutions
273def getSubstitutions(sourcepath):
274  sourcedir = os.path.dirname(sourcepath)
275  return [('%s', sourcepath),
276          ('%S', sourcedir),
277          ('%p', sourcedir),
278          ('%{pathsep}', os.pathsep)]
279
280def applySubstitutions(s, substitutions):
281  for a,b in substitutions:
282    s = s.replace(a, b)
283  return s
284
285# Invoke the tool that is being tested.
286def invoke_tool(exe, cmd_args, ir, preprocess_cmd=None, verbose=False):
287  with open(ir) as ir_file:
288    substitutions = getSubstitutions(ir)
289
290    # TODO Remove the str form which is used by update_test_checks.py and
291    # update_llc_test_checks.py
292    # The safer list form is used by update_cc_test_checks.py
293    if preprocess_cmd:
294      # Allow pre-processing the IR file (e.g. using sed):
295      assert isinstance(preprocess_cmd, str)  # TODO: use a list instead of using shell
296      preprocess_cmd = applySubstitutions(preprocess_cmd, substitutions).strip()
297      if verbose:
298        print('Pre-processing input file: ', ir, " with command '",
299              preprocess_cmd, "'", sep="", file=sys.stderr)
300      # Python 2.7 doesn't have subprocess.DEVNULL:
301      with open(os.devnull, 'w') as devnull:
302        pp = subprocess.Popen(preprocess_cmd, shell=True, stdin=devnull,
303                              stdout=subprocess.PIPE)
304        ir_file = pp.stdout
305
306    if isinstance(cmd_args, list):
307      args = [applySubstitutions(a, substitutions) for a in cmd_args]
308      stdout = subprocess.check_output([exe] + args, stdin=ir_file)
309    else:
310      stdout = subprocess.check_output(exe + ' ' + applySubstitutions(cmd_args, substitutions),
311                                       shell=True, stdin=ir_file)
312    if sys.version_info[0] > 2:
313      # FYI, if you crashed here with a decode error, your run line probably
314      # results in bitcode or other binary format being written to the pipe.
315      # For an opt test, you probably want to add -S or -disable-output.
316      stdout = stdout.decode()
317  # Fix line endings to unix CR style.
318  return stdout.replace('\r\n', '\n')
319
320##### LLVM IR parser
321RUN_LINE_RE = re.compile(r'^\s*(?://|[;#])\s*RUN:\s*(.*)$')
322CHECK_PREFIX_RE = re.compile(r'--?check-prefix(?:es)?[= ](\S+)')
323PREFIX_RE = re.compile('^[a-zA-Z0-9_-]+$')
324CHECK_RE = re.compile(r'^\s*(?://|[;#])\s*([^:]+?)(?:-NEXT|-NOT|-DAG|-LABEL|-SAME|-EMPTY)?:')
325
326UTC_ARGS_KEY = 'UTC_ARGS:'
327UTC_ARGS_CMD = re.compile(r'.*' + UTC_ARGS_KEY + '\s*(?P<cmd>.*)\s*$')
328UTC_ADVERT = 'NOTE: Assertions have been autogenerated by '
329UNUSED_NOTE = 'NOTE: These prefixes are unused and the list is autogenerated. Do not add tests below this line:'
330
331OPT_FUNCTION_RE = re.compile(
332    r'^(\s*;\s*Function\sAttrs:\s(?P<attrs>[\w\s():,]+?))?\s*define\s+(?:internal\s+)?[^@]*@(?P<func>[\w.$-]+?)\s*'
333    r'(?P<args_and_sig>\((\)|(.*?[\w.-]+?)\))[^{]*\{)\n(?P<body>.*?)^\}$',
334    flags=(re.M | re.S))
335
336ANALYZE_FUNCTION_RE = re.compile(
337    r'^\s*\'(?P<analysis>[\w\s-]+?)\'\s+for\s+function\s+\'(?P<func>[\w.$-]+?)\':'
338    r'\s*\n(?P<body>.*)$',
339    flags=(re.X | re.S))
340
341LV_DEBUG_RE = re.compile(
342    r'^\s*\'(?P<func>[\w.$-]+?)\'[^\n]*'
343    r'\s*\n(?P<body>.*)$',
344    flags=(re.X | re.S))
345
346IR_FUNCTION_RE = re.compile(r'^\s*define\s+(?:internal\s+)?[^@]*@"?([\w.$-]+)"?\s*\(')
347TRIPLE_IR_RE = re.compile(r'^\s*target\s+triple\s*=\s*"([^"]+)"$')
348TRIPLE_ARG_RE = re.compile(r'-mtriple[= ]([^ ]+)')
349MARCH_ARG_RE = re.compile(r'-march[= ]([^ ]+)')
350DEBUG_ONLY_ARG_RE = re.compile(r'-debug-only[= ]([^ ]+)')
351
352SCRUB_LEADING_WHITESPACE_RE = re.compile(r'^(\s+)')
353SCRUB_WHITESPACE_RE = re.compile(r'(?!^(|  \w))[ \t]+', flags=re.M)
354SCRUB_TRAILING_WHITESPACE_RE = re.compile(r'[ \t]+$', flags=re.M)
355SCRUB_TRAILING_WHITESPACE_TEST_RE = SCRUB_TRAILING_WHITESPACE_RE
356SCRUB_TRAILING_WHITESPACE_AND_ATTRIBUTES_RE = re.compile(r'([ \t]|(#[0-9]+))+$', flags=re.M)
357SCRUB_KILL_COMMENT_RE = re.compile(r'^ *#+ +kill:.*\n')
358SCRUB_LOOP_COMMENT_RE = re.compile(
359    r'# =>This Inner Loop Header:.*|# in Loop:.*', flags=re.M)
360SCRUB_TAILING_COMMENT_TOKEN_RE = re.compile(r'(?<=\S)+[ \t]*#$', flags=re.M)
361
362SEPARATOR = '.'
363
364def error(msg, test_file=None):
365  if test_file:
366    msg = '{}: {}'.format(msg, test_file)
367  print('ERROR: {}'.format(msg), file=sys.stderr)
368
369def warn(msg, test_file=None):
370  if test_file:
371    msg = '{}: {}'.format(msg, test_file)
372  print('WARNING: {}'.format(msg), file=sys.stderr)
373
374def debug(*args, **kwargs):
375  # Python2 does not allow def debug(*args, file=sys.stderr, **kwargs):
376  if 'file' not in kwargs:
377    kwargs['file'] = sys.stderr
378  if _verbose:
379    print(*args, **kwargs)
380
381def find_run_lines(test, lines):
382  debug('Scanning for RUN lines in test file:', test)
383  raw_lines = [m.group(1)
384               for m in [RUN_LINE_RE.match(l) for l in lines] if m]
385  run_lines = [raw_lines[0]] if len(raw_lines) > 0 else []
386  for l in raw_lines[1:]:
387    if run_lines[-1].endswith('\\'):
388      run_lines[-1] = run_lines[-1].rstrip('\\') + ' ' + l
389    else:
390      run_lines.append(l)
391  debug('Found {} RUN lines in {}:'.format(len(run_lines), test))
392  for l in run_lines:
393    debug('  RUN: {}'.format(l))
394  return run_lines
395
396def get_triple_from_march(march):
397  triples = {
398      'amdgcn': 'amdgcn',
399      'r600': 'r600',
400      'mips': 'mips',
401      'sparc': 'sparc',
402      'hexagon': 'hexagon',
403      've': 've',
404  }
405  for prefix, triple in triples.items():
406    if march.startswith(prefix):
407      return triple
408  print("Cannot find a triple. Assume 'x86'", file=sys.stderr)
409  return 'x86'
410
411def apply_filters(line, filters):
412  has_filter = False
413  for f in filters:
414    if not f.is_filter_out:
415      has_filter = True
416    if f.search(line):
417      return False if f.is_filter_out else True
418  # If we only used filter-out, keep the line, otherwise discard it since no
419  # filter matched.
420  return False if has_filter else True
421
422def do_filter(body, filters):
423  return body if not filters else '\n'.join(filter(
424    lambda line: apply_filters(line, filters), body.splitlines()))
425
426def scrub_body(body):
427  # Scrub runs of whitespace out of the assembly, but leave the leading
428  # whitespace in place.
429  body = SCRUB_WHITESPACE_RE.sub(r' ', body)
430  # Expand the tabs used for indentation.
431  body = str.expandtabs(body, 2)
432  # Strip trailing whitespace.
433  body = SCRUB_TRAILING_WHITESPACE_TEST_RE.sub(r'', body)
434  return body
435
436def do_scrub(body, scrubber, scrubber_args, extra):
437  if scrubber_args:
438    local_args = copy.deepcopy(scrubber_args)
439    local_args[0].extra_scrub = extra
440    return scrubber(body, *local_args)
441  return scrubber(body, *scrubber_args)
442
443# Build up a dictionary of all the function bodies.
444class function_body(object):
445  def __init__(self, string, extra, args_and_sig, attrs, func_name_separator):
446    self.scrub = string
447    self.extrascrub = extra
448    self.args_and_sig = args_and_sig
449    self.attrs = attrs
450    self.func_name_separator = func_name_separator
451  def is_same_except_arg_names(self, extrascrub, args_and_sig, attrs, is_backend):
452    arg_names = set()
453    def drop_arg_names(match):
454      arg_names.add(match.group(variable_group_in_ir_value_match))
455      if match.group(attribute_group_in_ir_value_match):
456        attr = match.group(attribute_group_in_ir_value_match)
457      else:
458        attr = ''
459      return match.group(1) + attr + match.group(match.lastindex)
460    def repl_arg_names(match):
461      if match.group(variable_group_in_ir_value_match) is not None and match.group(variable_group_in_ir_value_match) in arg_names:
462        return match.group(1) + match.group(match.lastindex)
463      return match.group(1) + match.group(2) + match.group(match.lastindex)
464    if self.attrs != attrs:
465      return False
466    ans0 = IR_VALUE_RE.sub(drop_arg_names, self.args_and_sig)
467    ans1 = IR_VALUE_RE.sub(drop_arg_names, args_and_sig)
468    if ans0 != ans1:
469      return False
470    if is_backend:
471      # Check without replacements, the replacements are not applied to the
472      # body for backend checks.
473      return self.extrascrub == extrascrub
474
475    es0 = IR_VALUE_RE.sub(repl_arg_names, self.extrascrub)
476    es1 = IR_VALUE_RE.sub(repl_arg_names, extrascrub)
477    es0 = SCRUB_IR_COMMENT_RE.sub(r'', es0)
478    es1 = SCRUB_IR_COMMENT_RE.sub(r'', es1)
479    return es0 == es1
480
481  def __str__(self):
482    return self.scrub
483
484class FunctionTestBuilder:
485  def __init__(self, run_list, flags, scrubber_args, path):
486    self._verbose = flags.verbose
487    self._record_args = flags.function_signature
488    self._check_attributes = flags.check_attributes
489    # Strip double-quotes if input was read by UTC_ARGS
490    self._filters = list(map(lambda f: Filter(re.compile(f.pattern().strip('"'),
491                                                         f.flags()),
492                                              f.is_filter_out),
493                             flags.filters)) if flags.filters else []
494    self._scrubber_args = scrubber_args
495    self._path = path
496    # Strip double-quotes if input was read by UTC_ARGS
497    self._replace_value_regex = list(map(lambda x: x.strip('"'), flags.replace_value_regex))
498    self._func_dict = {}
499    self._func_order = {}
500    self._global_var_dict = {}
501    self._processed_prefixes = set()
502    for tuple in run_list:
503      for prefix in tuple[0]:
504        self._func_dict.update({prefix: dict()})
505        self._func_order.update({prefix: []})
506        self._global_var_dict.update({prefix: dict()})
507
508  def finish_and_get_func_dict(self):
509    for prefix in self.get_failed_prefixes():
510      warn('Prefix %s had conflicting output from different RUN lines for all functions in test %s' % (prefix,self._path,))
511    return self._func_dict
512
513  def func_order(self):
514    return self._func_order
515
516  def global_var_dict(self):
517    return self._global_var_dict
518
519  def is_filtered(self):
520    return bool(self._filters)
521
522  def process_run_line(self, function_re, scrubber, raw_tool_output, prefixes, is_backend):
523    build_global_values_dictionary(self._global_var_dict, raw_tool_output, prefixes)
524    for m in function_re.finditer(raw_tool_output):
525      if not m:
526        continue
527      func = m.group('func')
528      body = m.group('body')
529      # func_name_separator is the string that is placed right after function name at the
530      # beginning of assembly function definition. In most assemblies, that is just a
531      # colon: `foo:`. But, for example, in nvptx it is a brace: `foo(`. If is_backend is
532      # False, just assume that separator is an empty string.
533      if is_backend:
534        # Use ':' as default separator.
535        func_name_separator = m.group('func_name_separator') if 'func_name_separator' in m.groupdict() else ':'
536      else:
537        func_name_separator = ''
538      attrs = m.group('attrs') if self._check_attributes else ''
539      # Determine if we print arguments, the opening brace, or nothing after the
540      # function name
541      if self._record_args and 'args_and_sig' in m.groupdict():
542        args_and_sig = scrub_body(m.group('args_and_sig').strip())
543      elif 'args_and_sig' in m.groupdict():
544        args_and_sig = '('
545      else:
546        args_and_sig = ''
547      filtered_body = do_filter(body, self._filters)
548      scrubbed_body = do_scrub(filtered_body, scrubber, self._scrubber_args,
549                               extra=False)
550      scrubbed_extra = do_scrub(filtered_body, scrubber, self._scrubber_args,
551                                extra=True)
552      if 'analysis' in m.groupdict():
553        analysis = m.group('analysis')
554        if analysis.lower() != 'cost model analysis':
555          warn('Unsupported analysis mode: %r!' % (analysis,))
556      if func.startswith('stress'):
557        # We only use the last line of the function body for stress tests.
558        scrubbed_body = '\n'.join(scrubbed_body.splitlines()[-1:])
559      if self._verbose:
560        print('Processing function: ' + func, file=sys.stderr)
561        for l in scrubbed_body.splitlines():
562          print('  ' + l, file=sys.stderr)
563      for prefix in prefixes:
564        # Replace function names matching the regex.
565        for regex in self._replace_value_regex:
566          # Pattern that matches capture groups in the regex in leftmost order.
567          group_regex = re.compile(r'\(.*?\)')
568          # Replace function name with regex.
569          match = re.match(regex, func)
570          if match:
571            func_repl = regex
572            # Replace any capture groups with their matched strings.
573            for g in match.groups():
574              func_repl = group_regex.sub(re.escape(g), func_repl, count=1)
575            func = re.sub(func_repl, '{{' + func_repl + '}}', func)
576
577          # Replace all calls to regex matching functions.
578          matches = re.finditer(regex, scrubbed_body)
579          for match in matches:
580            func_repl = regex
581            # Replace any capture groups with their matched strings.
582            for g in match.groups():
583              func_repl = group_regex.sub(re.escape(g), func_repl, count=1)
584            # Substitute function call names that match the regex with the same
585            # capture groups set.
586            scrubbed_body = re.sub(func_repl, '{{' + func_repl + '}}',
587                                   scrubbed_body)
588
589        if func in self._func_dict[prefix]:
590          if (self._func_dict[prefix][func] is not None and
591              (str(self._func_dict[prefix][func]) != scrubbed_body or
592               self._func_dict[prefix][func].args_and_sig != args_and_sig or
593               self._func_dict[prefix][func].attrs != attrs)):
594            if self._func_dict[prefix][func].is_same_except_arg_names(
595                scrubbed_extra,
596                args_and_sig,
597                attrs,
598                is_backend):
599              self._func_dict[prefix][func].scrub = scrubbed_extra
600              self._func_dict[prefix][func].args_and_sig = args_and_sig
601            else:
602              # This means a previous RUN line produced a body for this function
603              # that is different from the one produced by this current RUN line,
604              # so the body can't be common across RUN lines. We use None to
605              # indicate that.
606              self._func_dict[prefix][func] = None
607        else:
608          if prefix not in self._processed_prefixes:
609            self._func_dict[prefix][func] = function_body(
610                scrubbed_body, scrubbed_extra, args_and_sig, attrs,
611                func_name_separator)
612            self._func_order[prefix].append(func)
613          else:
614            # An earlier RUN line used this check prefixes but didn't produce
615            # a body for this function. This happens in Clang tests that use
616            # preprocesser directives to exclude individual functions from some
617            # RUN lines.
618            self._func_dict[prefix][func] = None
619
620  def processed_prefixes(self, prefixes):
621    """
622    Mark a set of prefixes as having had at least one applicable RUN line fully
623    processed. This is used to filter out function bodies that don't have
624    outputs for all RUN lines.
625    """
626    self._processed_prefixes.update(prefixes)
627
628  def get_failed_prefixes(self):
629    # This returns the list of those prefixes that failed to match any function,
630    # because there were conflicting bodies produced by different RUN lines, in
631    # all instances of the prefix.
632    for prefix in self._func_dict:
633      if (self._func_dict[prefix] and
634          (not [fct for fct in self._func_dict[prefix]
635                if self._func_dict[prefix][fct] is not None])):
636        yield prefix
637
638
639##### Generator of LLVM IR CHECK lines
640
641SCRUB_IR_COMMENT_RE = re.compile(r'\s*;.*')
642
643# TODO: We should also derive check lines for global, debug, loop declarations, etc..
644
645class NamelessValue:
646  def __init__(self, check_prefix, check_key, ir_prefix, global_ir_prefix, global_ir_prefix_regexp,
647               ir_regexp, global_ir_rhs_regexp, is_before_functions, *,
648               is_number=False, replace_number_with_counter=False):
649    self.check_prefix = check_prefix
650    self.check_key = check_key
651    self.ir_prefix = ir_prefix
652    self.global_ir_prefix = global_ir_prefix
653    self.global_ir_prefix_regexp = global_ir_prefix_regexp
654    self.ir_regexp = ir_regexp
655    self.global_ir_rhs_regexp = global_ir_rhs_regexp
656    self.is_before_functions = is_before_functions
657    self.is_number = is_number
658    # Some variable numbers (e.g. MCINST1234) will change based on unrelated
659    # modifications to LLVM, replace those with an incrementing counter.
660    self.replace_number_with_counter = replace_number_with_counter
661    self.variable_mapping = {}
662
663  # Return true if this kind of IR value is "local", basically if it matches '%{{.*}}'.
664  def is_local_def_ir_value_match(self, match):
665    return self.ir_prefix == '%'
666
667  # Return true if this kind of IR value is "global", basically if it matches '#{{.*}}'.
668  def is_global_scope_ir_value_match(self, match):
669    return self.global_ir_prefix is not None
670
671  # Return the IR prefix and check prefix we use for this kind or IR value,
672  # e.g., (%, TMP) for locals.
673  def get_ir_prefix_from_ir_value_match(self, match):
674    if self.ir_prefix and match.group(0).strip().startswith(self.ir_prefix):
675      return self.ir_prefix, self.check_prefix
676    return self.global_ir_prefix, self.check_prefix
677
678  # Return the IR regexp we use for this kind or IR value, e.g., [\w.-]+? for locals
679  def get_ir_regex_from_ir_value_re_match(self, match):
680    # for backwards compatibility we check locals with '.*'
681    if self.is_local_def_ir_value_match(match):
682      return '.*'
683    if self.ir_prefix and match.group(0).strip().startswith(self.ir_prefix):
684      return self.ir_regexp
685    return self.global_ir_prefix_regexp
686
687  # Create a FileCheck variable name based on an IR name.
688  def get_value_name(self, var: str, check_prefix: str):
689    var = var.replace('!', '')
690    if self.replace_number_with_counter:
691      assert var.isdigit(), var
692      replacement = self.variable_mapping.get(var, None)
693      if replacement is None:
694        # Replace variable with an incrementing counter
695        replacement = str(len(self.variable_mapping) + 1)
696        self.variable_mapping[var] = replacement
697      var = replacement
698    # This is a nameless value, prepend check_prefix.
699    if var.isdigit():
700      var = check_prefix + var
701    else:
702      # This is a named value that clashes with the check_prefix, prepend with
703      # _prefix_filecheck_ir_name, if it has been defined.
704      if may_clash_with_default_check_prefix_name(check_prefix, var) and _prefix_filecheck_ir_name:
705        var = _prefix_filecheck_ir_name + var
706    var = var.replace('.', '_')
707    var = var.replace('-', '_')
708    return var.upper()
709
710  # Create a FileCheck variable from regex.
711  def get_value_definition(self, var, match):
712    # for backwards compatibility we check locals with '.*'
713    varname = self.get_value_name(var, self.check_prefix)
714    prefix = self.get_ir_prefix_from_ir_value_match(match)[0]
715    if self.is_number:
716      regex = ''  # always capture a number in the default format
717      capture_start = '[[#'
718    else:
719      regex = self.get_ir_regex_from_ir_value_re_match(match)
720      capture_start = '[['
721    if self.is_local_def_ir_value_match(match):
722      return capture_start + varname + ':' + prefix + regex + ']]'
723    return prefix + capture_start + varname + ':' + regex + ']]'
724
725  # Use a FileCheck variable.
726  def get_value_use(self, var, match, var_prefix=None):
727    if var_prefix is None:
728      var_prefix = self.check_prefix
729    capture_start = '[[#' if self.is_number else '[['
730    if self.is_local_def_ir_value_match(match):
731      return capture_start + self.get_value_name(var, var_prefix) + ']]'
732    prefix = self.get_ir_prefix_from_ir_value_match(match)[0]
733    return prefix + capture_start + self.get_value_name(var, var_prefix) + ']]'
734
735# Description of the different "unnamed" values we match in the IR, e.g.,
736# (local) ssa values, (debug) metadata, etc.
737ir_nameless_values = [
738    NamelessValue(r'TMP'        , '%' , r'%'                   , None            , None                   , r'[\w$.-]+?' , None                 , False) ,
739    NamelessValue(r'ATTR'       , '#' , r'#'                   , None            , None                   , r'[0-9]+'    , None                 , False) ,
740    NamelessValue(r'ATTR'       , '#' , None                   , r'attributes #' , r'[0-9]+'              , None         , r'{[^}]*}'           , False) ,
741    NamelessValue(r'GLOB'       , '@' , r'@'                   , None            , None                   , r'[0-9]+'    , None                 , False) ,
742    NamelessValue(r'GLOB'       , '@' , None                   , r'@'            , r'[a-zA-Z0-9_$"\\.-]+' , None         , r'.+'                , True)  ,
743    NamelessValue(r'DBG'        , '!' , r'!dbg '               , None            , None                   , r'![0-9]+'   , None                 , False) ,
744    NamelessValue(r'DIASSIGNID' , '!' , r'!DIAssignID '               , None            , None                   , r'![0-9]+'   , None                 , False) ,
745    NamelessValue(r'PROF'       , '!' , r'!prof '              , None            , None                   , r'![0-9]+'   , None                 , False) ,
746    NamelessValue(r'TBAA'       , '!' , r'!tbaa '              , None            , None                   , r'![0-9]+'   , None                 , False) ,
747    NamelessValue(r'TBAA_STRUCT', '!' , r'!tbaa.struct '       , None            , None                   , r'![0-9]+'   , None                 , False) ,
748    NamelessValue(r'RNG'        , '!' , r'!range '             , None            , None                   , r'![0-9]+'   , None                 , False) ,
749    NamelessValue(r'LOOP'       , '!' , r'!llvm.loop '         , None            , None                   , r'![0-9]+'   , None                 , False) ,
750    NamelessValue(r'META'       , '!' , r'metadata '           , None            , None                   , r'![0-9]+'   , None                 , False) ,
751    NamelessValue(r'META'       , '!' , None                   , r''             , r'![0-9]+'             , None         , r'(?:distinct |)!.*' , False) ,
752    NamelessValue(r'ACC_GRP'    , '!' , r'!llvm.access.group ' , None            , None                   , r'![0-9]+'   , None                 , False) ,
753]
754
755asm_nameless_values = [
756 NamelessValue(r'MCINST', 'Inst#', None, '<MCInst #', r'\d+', None, r'.+',
757               False, is_number=True, replace_number_with_counter=True),
758 NamelessValue(r'MCREG',  'Reg:', None, '<MCOperand Reg:', r'\d+', None, r'.+',
759               False, is_number=True, replace_number_with_counter=True),
760]
761
762def createOrRegexp(old, new):
763  if not old:
764    return new
765  if not new:
766    return old
767  return old + '|' + new
768
769def createPrefixMatch(prefix_str, prefix_re):
770  if prefix_str is None or prefix_re is None:
771    return ''
772  return '(?:' + prefix_str + '(' + prefix_re + '))'
773
774# Build the regexp that matches an "IR value". This can be a local variable,
775# argument, global, or metadata, anything that is "named". It is important that
776# the PREFIX and SUFFIX below only contain a single group, if that changes
777# other locations will need adjustment as well.
778IR_VALUE_REGEXP_PREFIX = r'(\s*)'
779IR_VALUE_REGEXP_STRING = r''
780for nameless_value in ir_nameless_values:
781  lcl_match = createPrefixMatch(nameless_value.ir_prefix, nameless_value.ir_regexp)
782  glb_match = createPrefixMatch(nameless_value.global_ir_prefix, nameless_value.global_ir_prefix_regexp)
783  assert((lcl_match or glb_match) and not (lcl_match and glb_match))
784  if lcl_match:
785    IR_VALUE_REGEXP_STRING = createOrRegexp(IR_VALUE_REGEXP_STRING, lcl_match)
786  elif glb_match:
787    IR_VALUE_REGEXP_STRING = createOrRegexp(IR_VALUE_REGEXP_STRING, '^' + glb_match)
788IR_VALUE_REGEXP_SUFFIX = r'([,\s\(\)]|\Z)'
789IR_VALUE_RE = re.compile(IR_VALUE_REGEXP_PREFIX + r'(' + IR_VALUE_REGEXP_STRING + r')' + IR_VALUE_REGEXP_SUFFIX)
790
791# Build the regexp that matches an "ASM value" (currently only for --asm-show-inst comments).
792ASM_VALUE_REGEXP_STRING = ''
793for nameless_value in asm_nameless_values:
794  glb_match = createPrefixMatch(nameless_value.global_ir_prefix, nameless_value.global_ir_prefix_regexp)
795  assert not nameless_value.ir_prefix and not nameless_value.ir_regexp
796  ASM_VALUE_REGEXP_STRING = createOrRegexp(ASM_VALUE_REGEXP_STRING, glb_match)
797ASM_VALUE_REGEXP_SUFFIX = r'([>\s]|\Z)'
798ASM_VALUE_RE = re.compile(r'((?:#|//)\s*)' + '(' + ASM_VALUE_REGEXP_STRING + ')' + ASM_VALUE_REGEXP_SUFFIX)
799
800# The entire match is group 0, the prefix has one group (=1), the entire
801# IR_VALUE_REGEXP_STRING is one group (=2), and then the nameless values start.
802first_nameless_group_in_ir_value_match = 3
803
804# constants for the group id of special matches
805variable_group_in_ir_value_match = 3
806attribute_group_in_ir_value_match = 4
807
808# Check a match for IR_VALUE_RE and inspect it to determine if it was a local
809# value, %..., global @..., debug number !dbg !..., etc. See the PREFIXES above.
810def get_idx_from_ir_value_match(match):
811  for i in range(first_nameless_group_in_ir_value_match, match.lastindex):
812    if match.group(i) is not None:
813      return i - first_nameless_group_in_ir_value_match
814  error("Unable to identify the kind of IR value from the match!")
815  return 0
816
817# See get_idx_from_ir_value_match
818def get_name_from_ir_value_match(match):
819  return match.group(get_idx_from_ir_value_match(match) + first_nameless_group_in_ir_value_match)
820
821def get_nameless_value_from_match(match, nameless_values) -> NamelessValue:
822  return nameless_values[get_idx_from_ir_value_match(match)]
823
824# Return true if var clashes with the scripted FileCheck check_prefix.
825def may_clash_with_default_check_prefix_name(check_prefix, var):
826  return check_prefix and re.match(r'^' + check_prefix + r'[0-9]+?$', var, re.IGNORECASE)
827
828def generalize_check_lines_common(lines, is_analyze, vars_seen,
829                                  global_vars_seen, nameless_values,
830                                  nameless_value_regex, is_asm):
831  # This gets called for each match that occurs in
832  # a line. We transform variables we haven't seen
833  # into defs, and variables we have seen into uses.
834  def transform_line_vars(match):
835    var = get_name_from_ir_value_match(match)
836    nameless_value = get_nameless_value_from_match(match, nameless_values)
837    if may_clash_with_default_check_prefix_name(nameless_value.check_prefix, var):
838      warn("Change IR value name '%s' or use --prefix-filecheck-ir-name to prevent possible conflict"
839           " with scripted FileCheck name." % (var,))
840    key = (var, nameless_value.check_key)
841    is_local_def = nameless_value.is_local_def_ir_value_match(match)
842    if is_local_def and key in vars_seen:
843      rv = nameless_value.get_value_use(var, match)
844    elif not is_local_def and key in global_vars_seen:
845      # We could have seen a different prefix for the global variables first,
846      # ensure we use that one instead of the prefix for the current match.
847      rv = nameless_value.get_value_use(var, match, global_vars_seen[key])
848    else:
849      if is_local_def:
850        vars_seen.add(key)
851      else:
852        global_vars_seen[key] = nameless_value.check_prefix
853      rv = nameless_value.get_value_definition(var, match)
854    # re.sub replaces the entire regex match
855    # with whatever you return, so we have
856    # to make sure to hand it back everything
857    # including the commas and spaces.
858    return match.group(1) + rv + match.group(match.lastindex)
859
860  lines_with_def = []
861
862  for i, line in enumerate(lines):
863    if not is_asm:
864      # An IR variable named '%.' matches the FileCheck regex string.
865      line = line.replace('%.', '%dot')
866      for regex in _global_hex_value_regex:
867        if re.match('^@' + regex + ' = ', line):
868          line = re.sub(r'\bi([0-9]+) ([0-9]+)',
869              lambda m : 'i' + m.group(1) + ' [[#' + hex(int(m.group(2))) + ']]',
870              line)
871          break
872      # Ignore any comments, since the check lines will too.
873      scrubbed_line = SCRUB_IR_COMMENT_RE.sub(r'', line)
874      lines[i] = scrubbed_line
875    if is_asm or not is_analyze:
876      # It can happen that two matches are back-to-back and for some reason sub
877      # will not replace both of them. For now we work around this by
878      # substituting until there is no more match.
879      changed = True
880      while changed:
881        (lines[i], changed) = nameless_value_regex.subn(transform_line_vars,
882                                                        lines[i], count=1)
883  return lines
884
885# Replace IR value defs and uses with FileCheck variables.
886def generalize_check_lines(lines, is_analyze, vars_seen, global_vars_seen):
887  return generalize_check_lines_common(lines, is_analyze, vars_seen,
888                                       global_vars_seen, ir_nameless_values,
889                                       IR_VALUE_RE, False)
890
891def generalize_asm_check_lines(lines, vars_seen, global_vars_seen):
892  return generalize_check_lines_common(lines, False, vars_seen,
893                                       global_vars_seen, asm_nameless_values,
894                                       ASM_VALUE_RE, True)
895
896def add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name, check_label_format, is_backend, is_analyze, global_vars_seen_dict, is_filtered):
897  # prefix_exclusions are prefixes we cannot use to print the function because it doesn't exist in run lines that use these prefixes as well.
898  prefix_exclusions = set()
899  printed_prefixes = []
900  for p in prefix_list:
901    checkprefixes = p[0]
902    # If not all checkprefixes of this run line produced the function we cannot check for it as it does not
903    # exist for this run line. A subset of the check prefixes might know about the function but only because
904    # other run lines created it.
905    if any(map(lambda checkprefix: func_name not in func_dict[checkprefix], checkprefixes)):
906      prefix_exclusions |= set(checkprefixes)
907      continue
908
909  # prefix_exclusions is constructed, we can now emit the output
910  for p in prefix_list:
911    global_vars_seen = {}
912    checkprefixes = p[0]
913    for checkprefix in checkprefixes:
914      if checkprefix in global_vars_seen_dict:
915        global_vars_seen.update(global_vars_seen_dict[checkprefix])
916      else:
917        global_vars_seen_dict[checkprefix] = {}
918      if checkprefix in printed_prefixes:
919        break
920
921      # Check if the prefix is excluded.
922      if checkprefix in prefix_exclusions:
923        continue
924
925      # If we do not have output for this prefix we skip it.
926      if not func_dict[checkprefix][func_name]:
927        continue
928
929      # Add some space between different check prefixes, but not after the last
930      # check line (before the test code).
931      if is_backend:
932        if len(printed_prefixes) != 0:
933          output_lines.append(comment_marker)
934
935      if checkprefix not in global_vars_seen_dict:
936        global_vars_seen_dict[checkprefix] = {}
937
938      global_vars_seen_before = [key for key in global_vars_seen.keys()]
939
940      vars_seen = set()
941      printed_prefixes.append(checkprefix)
942      attrs = str(func_dict[checkprefix][func_name].attrs)
943      attrs = '' if attrs == 'None' else attrs
944      if attrs:
945        output_lines.append('%s %s: Function Attrs: %s' % (comment_marker, checkprefix, attrs))
946      args_and_sig = str(func_dict[checkprefix][func_name].args_and_sig)
947      if args_and_sig:
948        args_and_sig = generalize_check_lines([args_and_sig], is_analyze, vars_seen, global_vars_seen)[0]
949      func_name_separator = func_dict[checkprefix][func_name].func_name_separator
950      if '[[' in args_and_sig:
951        output_lines.append(check_label_format % (checkprefix, func_name, '', func_name_separator))
952        output_lines.append('%s %s-SAME: %s' % (comment_marker, checkprefix, args_and_sig))
953      else:
954        output_lines.append(check_label_format % (checkprefix, func_name, args_and_sig, func_name_separator))
955      func_body = str(func_dict[checkprefix][func_name]).splitlines()
956      if not func_body:
957        # We have filtered everything.
958        continue
959
960      # For ASM output, just emit the check lines.
961      if is_backend:
962        body_start = 1
963        if is_filtered:
964          # For filtered output we don't add "-NEXT" so don't add extra spaces
965          # before the first line.
966          body_start = 0
967        else:
968          output_lines.append('%s %s:       %s' % (comment_marker, checkprefix, func_body[0]))
969        func_lines = generalize_asm_check_lines(func_body[body_start:],
970                                                vars_seen, global_vars_seen)
971        for func_line in func_lines:
972          if func_line.strip() == '':
973            output_lines.append('%s %s-EMPTY:' % (comment_marker, checkprefix))
974          else:
975            check_suffix = '-NEXT' if not is_filtered else ''
976            output_lines.append('%s %s%s:  %s' % (comment_marker, checkprefix,
977                                                  check_suffix, func_line))
978        # Remember new global variables we have not seen before
979        for key in global_vars_seen:
980          if key not in global_vars_seen_before:
981            global_vars_seen_dict[checkprefix][key] = global_vars_seen[key]
982        break
983
984      # For IR output, change all defs to FileCheck variables, so we're immune
985      # to variable naming fashions.
986      func_body = generalize_check_lines(func_body, is_analyze, vars_seen, global_vars_seen)
987
988      # This could be selectively enabled with an optional invocation argument.
989      # Disabled for now: better to check everything. Be safe rather than sorry.
990
991      # Handle the first line of the function body as a special case because
992      # it's often just noise (a useless asm comment or entry label).
993      #if func_body[0].startswith("#") or func_body[0].startswith("entry:"):
994      #  is_blank_line = True
995      #else:
996      #  output_lines.append('%s %s:       %s' % (comment_marker, checkprefix, func_body[0]))
997      #  is_blank_line = False
998
999      is_blank_line = False
1000
1001      for func_line in func_body:
1002        if func_line.strip() == '':
1003          is_blank_line = True
1004          continue
1005        # Do not waste time checking IR comments.
1006        func_line = SCRUB_IR_COMMENT_RE.sub(r'', func_line)
1007
1008        # Skip blank lines instead of checking them.
1009        if is_blank_line:
1010          output_lines.append('{} {}:       {}'.format(
1011              comment_marker, checkprefix, func_line))
1012        else:
1013          check_suffix = '-NEXT' if not is_filtered else ''
1014          output_lines.append('{} {}{}:  {}'.format(
1015              comment_marker, checkprefix, check_suffix, func_line))
1016        is_blank_line = False
1017
1018      # Add space between different check prefixes and also before the first
1019      # line of code in the test function.
1020      output_lines.append(comment_marker)
1021
1022      # Remember new global variables we have not seen before
1023      for key in global_vars_seen:
1024        if key not in global_vars_seen_before:
1025          global_vars_seen_dict[checkprefix][key] = global_vars_seen[key]
1026      break
1027  return printed_prefixes
1028
1029def add_ir_checks(output_lines, comment_marker, prefix_list, func_dict,
1030                  func_name, preserve_names, function_sig,
1031                  global_vars_seen_dict, is_filtered):
1032  # Label format is based on IR string.
1033  function_def_regex = 'define {{[^@]+}}' if function_sig else ''
1034  check_label_format = '{} %s-LABEL: {}@%s%s%s'.format(comment_marker, function_def_regex)
1035  return add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name,
1036                    check_label_format, False, preserve_names, global_vars_seen_dict,
1037                    is_filtered)
1038
1039def add_analyze_checks(output_lines, comment_marker, prefix_list, func_dict, func_name, is_filtered):
1040  check_label_format = '{} %s-LABEL: \'%s%s%s\''.format(comment_marker)
1041  global_vars_seen_dict = {}
1042  return add_checks(output_lines, comment_marker, prefix_list, func_dict, func_name,
1043                    check_label_format, False, True, global_vars_seen_dict,
1044                    is_filtered)
1045
1046def build_global_values_dictionary(glob_val_dict, raw_tool_output, prefixes):
1047  for nameless_value in itertools.chain(ir_nameless_values, asm_nameless_values):
1048    if nameless_value.global_ir_prefix is None:
1049      continue
1050
1051    lhs_re_str = nameless_value.global_ir_prefix + nameless_value.global_ir_prefix_regexp
1052    rhs_re_str = nameless_value.global_ir_rhs_regexp
1053
1054    global_ir_value_re_str = r'^' + lhs_re_str + r'\s=\s' + rhs_re_str + r'$'
1055    global_ir_value_re = re.compile(global_ir_value_re_str, flags=(re.M))
1056    lines = []
1057    for m in global_ir_value_re.finditer(raw_tool_output):
1058      lines.append(m.group(0))
1059
1060    for prefix in prefixes:
1061      if glob_val_dict[prefix] is None:
1062        continue
1063      if nameless_value.check_prefix in glob_val_dict[prefix]:
1064        if lines == glob_val_dict[prefix][nameless_value.check_prefix]:
1065          continue
1066        if prefix == prefixes[-1]:
1067          warn('Found conflicting asm under the same prefix: %r!' % (prefix,))
1068        else:
1069          glob_val_dict[prefix][nameless_value.check_prefix] = None
1070          continue
1071      glob_val_dict[prefix][nameless_value.check_prefix] = lines
1072
1073def add_global_checks(glob_val_dict, comment_marker, prefix_list, output_lines, global_vars_seen_dict, is_analyze, is_before_functions):
1074  printed_prefixes = set()
1075  for nameless_value in ir_nameless_values:
1076    if nameless_value.global_ir_prefix is None:
1077      continue
1078    if nameless_value.is_before_functions != is_before_functions:
1079      continue
1080    for p in prefix_list:
1081      global_vars_seen = {}
1082      checkprefixes = p[0]
1083      if checkprefixes is None:
1084        continue
1085      for checkprefix in checkprefixes:
1086        if checkprefix in global_vars_seen_dict:
1087          global_vars_seen.update(global_vars_seen_dict[checkprefix])
1088        else:
1089          global_vars_seen_dict[checkprefix] = {}
1090        if (checkprefix, nameless_value.check_prefix) in printed_prefixes:
1091          break
1092        if not glob_val_dict[checkprefix]:
1093          continue
1094        if nameless_value.check_prefix not in glob_val_dict[checkprefix]:
1095          continue
1096        if not glob_val_dict[checkprefix][nameless_value.check_prefix]:
1097          continue
1098
1099        check_lines = []
1100        global_vars_seen_before = [key for key in global_vars_seen.keys()]
1101        for line in glob_val_dict[checkprefix][nameless_value.check_prefix]:
1102          if _global_value_regex:
1103            matched = False
1104            for regex in _global_value_regex:
1105              if re.match('^@' + regex + ' = ', line):
1106                matched = True
1107                break
1108            if not matched:
1109              continue
1110          tmp = generalize_check_lines([line], is_analyze, set(), global_vars_seen)
1111          check_line = '%s %s: %s' % (comment_marker, checkprefix, tmp[0])
1112          check_lines.append(check_line)
1113        if not check_lines:
1114          continue
1115
1116        output_lines.append(comment_marker + SEPARATOR)
1117        for check_line in check_lines:
1118          output_lines.append(check_line)
1119
1120        printed_prefixes.add((checkprefix, nameless_value.check_prefix))
1121
1122        # Remembe new global variables we have not seen before
1123        for key in global_vars_seen:
1124          if key not in global_vars_seen_before:
1125            global_vars_seen_dict[checkprefix][key] = global_vars_seen[key]
1126        break
1127
1128  if printed_prefixes:
1129    output_lines.append(comment_marker + SEPARATOR)
1130  return printed_prefixes
1131
1132
1133def check_prefix(prefix):
1134  if not PREFIX_RE.match(prefix):
1135    hint = ""
1136    if ',' in prefix:
1137      hint = " Did you mean '--check-prefixes=" + prefix + "'?"
1138    warn(("Supplied prefix '%s' is invalid. Prefix must contain only alphanumeric characters, hyphens and underscores." + hint) %
1139         (prefix))
1140
1141
1142def verify_filecheck_prefixes(fc_cmd):
1143  fc_cmd_parts = fc_cmd.split()
1144  for part in fc_cmd_parts:
1145    if "check-prefix=" in part:
1146      prefix = part.split('=', 1)[1]
1147      check_prefix(prefix)
1148    elif "check-prefixes=" in part:
1149      prefixes = part.split('=', 1)[1].split(',')
1150      for prefix in prefixes:
1151        check_prefix(prefix)
1152        if prefixes.count(prefix) > 1:
1153          warn("Supplied prefix '%s' is not unique in the prefix list." % (prefix,))
1154
1155
1156def get_autogennote_suffix(parser, args):
1157  autogenerated_note_args = ''
1158  for action in parser._actions:
1159    if not hasattr(args, action.dest):
1160      continue  # Ignore options such as --help that aren't included in args
1161    # Ignore parameters such as paths to the binary or the list of tests
1162    if action.dest in ('tests', 'update_only', 'tool_binary', 'opt_binary',
1163                       'llc_binary', 'clang', 'opt', 'llvm_bin', 'verbose',
1164                       'force_update'):
1165      continue
1166    value = getattr(args, action.dest)
1167    if action.const is not None:  # action stores a constant (usually True/False)
1168      # Skip actions with different constant values (this happens with boolean
1169      # --foo/--no-foo options)
1170      if value != action.const:
1171        continue
1172    if parser.get_default(action.dest) == value:
1173      continue  # Don't add default values
1174    if action.dest == 'filters':
1175      # Create a separate option for each filter element.  The value is a list
1176      # of Filter objects.
1177      for elem in value:
1178        opt_name = 'filter-out' if elem.is_filter_out else 'filter'
1179        opt_value = elem.pattern()
1180        new_arg = '--%s "%s" ' % (opt_name, opt_value.strip('"'))
1181        if new_arg not in autogenerated_note_args:
1182          autogenerated_note_args += new_arg
1183    else:
1184      autogenerated_note_args += action.option_strings[0] + ' '
1185      if action.const is None:  # action takes a parameter
1186        if action.nargs == '+':
1187          value = ' '.join(map(lambda v: '"' + v.strip('"') + '"', value))
1188        autogenerated_note_args += '%s ' % value
1189  if autogenerated_note_args:
1190    autogenerated_note_args = ' %s %s' % (UTC_ARGS_KEY, autogenerated_note_args[:-1])
1191  return autogenerated_note_args
1192
1193
1194def check_for_command(line, parser, args, argv, argparse_callback):
1195  cmd_m = UTC_ARGS_CMD.match(line)
1196  if cmd_m:
1197    for option in shlex.split(cmd_m.group('cmd').strip()):
1198      if option:
1199        argv.append(option)
1200    args = parser.parse_args(filter(lambda arg: arg not in args.tests, argv))
1201    if argparse_callback is not None:
1202      argparse_callback(args)
1203  return args, argv
1204
1205def find_arg_in_test(test_info, get_arg_to_check, arg_string, is_global):
1206  result = get_arg_to_check(test_info.args)
1207  if not result and is_global:
1208    # See if this has been specified via UTC_ARGS.  This is a "global" option
1209    # that affects the entire generation of test checks.  If it exists anywhere
1210    # in the test, apply it to everything.
1211    saw_line = False
1212    for line_info in test_info.ro_iterlines():
1213      line = line_info.line
1214      if not line.startswith(';') and line.strip() != '':
1215        saw_line = True
1216      result = get_arg_to_check(line_info.args)
1217      if result:
1218        if warn and saw_line:
1219          # We saw the option after already reading some test input lines.
1220          # Warn about it.
1221          print('WARNING: Found {} in line following test start: '.format(arg_string)
1222                + line, file=sys.stderr)
1223          print('WARNING: Consider moving {} to top of file'.format(arg_string),
1224                file=sys.stderr)
1225        break
1226  return result
1227
1228def dump_input_lines(output_lines, test_info, prefix_set, comment_string):
1229  for input_line_info in test_info.iterlines(output_lines):
1230    line = input_line_info.line
1231    args = input_line_info.args
1232    if line.strip() == comment_string:
1233      continue
1234    if line.strip() == comment_string + SEPARATOR:
1235      continue
1236    if line.lstrip().startswith(comment_string):
1237      m = CHECK_RE.match(line)
1238      if m and m.group(1) in prefix_set:
1239        continue
1240    output_lines.append(line.rstrip('\n'))
1241
1242def add_checks_at_end(output_lines, prefix_list, func_order,
1243                      comment_string, check_generator):
1244  added = set()
1245  generated_prefixes = set()
1246  for prefix in prefix_list:
1247    prefixes = prefix[0]
1248    tool_args = prefix[1]
1249    for prefix in prefixes:
1250      for func in func_order[prefix]:
1251        # The func order can contain the same functions multiple times.
1252        # If we see one again we are done.
1253        if (func, prefix) in added:
1254          continue
1255        if added:
1256          output_lines.append(comment_string)
1257
1258        # The add_*_checks routines expect a run list whose items are
1259        # tuples that have a list of prefixes as their first element and
1260        # tool command args string as their second element.  They output
1261        # checks for each prefix in the list of prefixes.  By doing so, it
1262        # implicitly assumes that for each function every run line will
1263        # generate something for that function.  That is not the case for
1264        # generated functions as some run lines might not generate them
1265        # (e.g. -fopenmp vs. no -fopenmp).
1266        #
1267        # Therefore, pass just the prefix we're interested in.  This has
1268        # the effect of generating all of the checks for functions of a
1269        # single prefix before moving on to the next prefix.  So checks
1270        # are ordered by prefix instead of by function as in "normal"
1271        # mode.
1272        for generated_prefix in check_generator(output_lines,
1273                        [([prefix], tool_args)], func):
1274          added.add((func, generated_prefix))
1275          generated_prefixes.add(generated_prefix)
1276  return generated_prefixes
1277