1#!/usr/bin/python 2# 3# Copyright (C) 2014-2024 Free Software Foundation, Inc. 4# 5# This script is free software; you can redistribute it and/or modify 6# it under the terms of the GNU General Public License as published by 7# the Free Software Foundation; either version 3, or (at your option) 8# any later version. 9 10import sys 11import getopt 12import re 13import io 14from datetime import datetime 15from operator import attrgetter 16 17# True if unrecognised lines should cause a fatal error. Might want to turn 18# this on by default later. 19strict = False 20 21# True if the order of .log segments should match the .sum file, false if 22# they should keep the original order. 23sort_logs = True 24 25# A version of open() that is safe against whatever binary output 26# might be added to the log. 27def safe_open (filename): 28 if sys.version_info >= (3, 0): 29 return open (filename, 'r', errors = 'surrogateescape') 30 return open (filename, 'r') 31 32# Force stdout to handle escape sequences from a safe_open file. 33if sys.version_info >= (3, 0): 34 sys.stdout = io.TextIOWrapper (sys.stdout.buffer, 35 errors = 'surrogateescape') 36 37class Named: 38 def __init__ (self, name): 39 self.name = name 40 41class ToolRun (Named): 42 def __init__ (self, name): 43 Named.__init__ (self, name) 44 # The variations run for this tool, mapped by --target_board name. 45 self.variations = dict() 46 47 # Return the VariationRun for variation NAME. 48 def get_variation (self, name): 49 if name not in self.variations: 50 self.variations[name] = VariationRun (name) 51 return self.variations[name] 52 53class VariationRun (Named): 54 def __init__ (self, name): 55 Named.__init__ (self, name) 56 # A segment of text before the harness runs start, describing which 57 # baseboard files were loaded for the target. 58 self.header = None 59 # The harnesses run for this variation, mapped by filename. 60 self.harnesses = dict() 61 # A list giving the number of times each type of result has 62 # been seen. 63 self.counts = [] 64 65 # Return the HarnessRun for harness NAME. 66 def get_harness (self, name): 67 if name not in self.harnesses: 68 self.harnesses[name] = HarnessRun (name) 69 return self.harnesses[name] 70 71class HarnessRun (Named): 72 def __init__ (self, name): 73 Named.__init__ (self, name) 74 # Segments of text that make up the harness run, mapped by a test-based 75 # key that can be used to order them. 76 self.segments = dict() 77 # Segments of text that make up the harness run but which have 78 # no recognized test results. These are typically harnesses that 79 # are completely skipped for the target. 80 self.empty = [] 81 # A list of results. Each entry is a pair in which the first element 82 # is a unique sorting key and in which the second is the full 83 # PASS/FAIL line. 84 self.results = [] 85 86 # Add a segment of text to the harness run. If the segment includes 87 # test results, KEY is an example of one of them, and can be used to 88 # combine the individual segments in order. If the segment has no 89 # test results (e.g. because the harness doesn't do anything for the 90 # current configuration) then KEY is None instead. In that case 91 # just collect the segments in the order that we see them. 92 def add_segment (self, key, segment): 93 if key: 94 assert key not in self.segments 95 self.segments[key] = segment 96 else: 97 self.empty.append (segment) 98 99class Segment: 100 def __init__ (self, filename, start): 101 self.filename = filename 102 self.start = start 103 self.lines = 0 104 105class Prog: 106 def __init__ (self): 107 # The variations specified on the command line. 108 self.variations = [] 109 # The variations seen in the input files. 110 self.known_variations = set() 111 # The tools specified on the command line. 112 self.tools = [] 113 # Whether to create .sum rather than .log output. 114 self.do_sum = True 115 # Regexps used while parsing. 116 self.test_run_re = re.compile (r'^Test run by (\S+) on (.*)$', 117 re.IGNORECASE) 118 self.tool_re = re.compile (r'^\t\t=== (.*) tests ===$') 119 self.result_re = re.compile (r'^(PASS|XPASS|FAIL|XFAIL|UNRESOLVED' 120 r'|WARNING|ERROR|UNSUPPORTED|UNTESTED' 121 r'|KFAIL|KPASS|PATH|DUPLICATE):\s*(.+)') 122 self.completed_re = re.compile (r'.* completed at (.*)') 123 # Pieces of text to write at the head of the output. 124 # start_line is a pair in which the first element is a datetime 125 # and in which the second is the associated 'Test Run By' line. 126 self.start_line = None 127 self.native_line = '' 128 self.target_line = '' 129 self.host_line = '' 130 self.acats_premable = '' 131 # Pieces of text to write at the end of the output. 132 # end_line is like start_line but for the 'runtest completed' line. 133 self.acats_failures = [] 134 self.version_output = '' 135 self.end_line = None 136 # Known summary types. 137 self.count_names = [ 138 '# of DejaGnu errors\t\t', 139 '# of expected passes\t\t', 140 '# of unexpected failures\t', 141 '# of unexpected successes\t', 142 '# of expected failures\t\t', 143 '# of unknown successes\t\t', 144 '# of known failures\t\t', 145 '# of untested testcases\t\t', 146 '# of unresolved testcases\t', 147 '# of unsupported tests\t\t', 148 '# of paths in test names\t', 149 '# of duplicate test names\t' 150 ] 151 self.runs = dict() 152 153 def usage (self): 154 name = sys.argv[0] 155 sys.stderr.write ('Usage: ' + name 156 + ''' [-t tool] [-l variant-list] [-L] log-or-sum-file ... 157 158 tool The tool (e.g. g++, libffi) for which to create a 159 new test summary file. If not specified then output 160 is created for all tools. 161 variant-list One or more test variant names. If the list is 162 not specified then one is constructed from all 163 variants in the files for <tool>. 164 sum-file A test summary file with the format of those 165 created by runtest from DejaGnu. 166 If -L is used, merge *.log files instead of *.sum. In this 167 mode the exact order of lines may not be preserved, just different 168 Running *.exp chunks should be in correct order. 169''') 170 sys.exit (1) 171 172 def fatal (self, what, string): 173 if not what: 174 what = sys.argv[0] 175 sys.stderr.write (what + ': ' + string + '\n') 176 sys.exit (1) 177 178 # Parse the command-line arguments. 179 def parse_cmdline (self): 180 try: 181 (options, self.files) = getopt.getopt (sys.argv[1:], 'l:t:L') 182 if len (self.files) == 0: 183 self.usage() 184 for (option, value) in options: 185 if option == '-l': 186 self.variations.append (value) 187 elif option == '-t': 188 self.tools.append (value) 189 else: 190 self.do_sum = False 191 except getopt.GetoptError as e: 192 self.fatal (None, e.msg) 193 194 # Try to parse time string TIME, returning an arbitrary time on failure. 195 # Getting this right is just a nice-to-have so failures should be silent. 196 def parse_time (self, time): 197 try: 198 return datetime.strptime (time, '%c') 199 except ValueError: 200 return datetime.now() 201 202 # Parse an integer and abort on failure. 203 def parse_int (self, filename, value): 204 try: 205 return int (value) 206 except ValueError: 207 self.fatal (filename, 'expected an integer, got: ' + value) 208 209 # Return a list that represents no test results. 210 def zero_counts (self): 211 return [0 for x in self.count_names] 212 213 # Return the ToolRun for tool NAME. 214 def get_tool (self, name): 215 if name not in self.runs: 216 self.runs[name] = ToolRun (name) 217 return self.runs[name] 218 219 # Add the result counts in list FROMC to TOC. 220 def accumulate_counts (self, toc, fromc): 221 for i in range (len (self.count_names)): 222 toc[i] += fromc[i] 223 224 # Parse the list of variations after 'Schedule of variations:'. 225 # Return the number seen. 226 def parse_variations (self, filename, file): 227 num_variations = 0 228 while True: 229 line = file.readline() 230 if line == '': 231 self.fatal (filename, 'could not parse variation list') 232 if line == '\n': 233 break 234 self.known_variations.add (line.strip()) 235 num_variations += 1 236 return num_variations 237 238 # Parse from the first line after 'Running target ...' to the end 239 # of the run's summary. 240 def parse_run (self, filename, file, tool, variation, num_variations): 241 header = None 242 harness = None 243 segment = None 244 final_using = 0 245 has_warning = 0 246 247 # If this is the first run for this variation, add any text before 248 # the first harness to the header. 249 if not variation.header: 250 segment = Segment (filename, file.tell()) 251 variation.header = segment 252 253 # Parse the rest of the summary (the '# of ' lines). 254 if len (variation.counts) == 0: 255 variation.counts = self.zero_counts() 256 257 # Parse up until the first line of the summary. 258 if num_variations == 1: 259 end = '\t\t=== ' + tool.name + ' Summary ===\n' 260 else: 261 end = ('\t\t=== ' + tool.name + ' Summary for ' 262 + variation.name + ' ===\n') 263 while True: 264 line = file.readline() 265 if line == '': 266 self.fatal (filename, 'no recognised summary line') 267 if line == end: 268 break 269 270 # Look for the start of a new harness. 271 if line.startswith ('Running ') and line.endswith (' ...\n'): 272 # Close off the current harness segment, if any. 273 if harness: 274 segment.lines -= final_using 275 harness.add_segment (first_key, segment) 276 name = line[len ('Running '):-len(' ...\n')] 277 harness = variation.get_harness (name) 278 segment = Segment (filename, file.tell()) 279 first_key = None 280 final_using = 0 281 continue 282 283 # Record test results. Associate the first test result with 284 # the harness segment, so that if a run for a particular harness 285 # has been split up, we can reassemble the individual segments 286 # in a sensible order. 287 # 288 # dejagnu sometimes issues warnings about the testing environment 289 # before running any tests. Treat them as part of the header 290 # rather than as a test result. 291 match = self.result_re.match (line) 292 if match and (harness or not line.startswith ('WARNING:')): 293 if not harness: 294 self.fatal (filename, 'saw test result before harness name') 295 name = match.group (2) 296 # Ugly hack to get the right order for gfortran. 297 if name.startswith ('gfortran.dg/g77/'): 298 name = 'h' + name 299 # If we have a time out warning, make sure it appears 300 # before the following testcase diagnostic: we insert 301 # the testname before 'program' so that sort faces a 302 # list of testnames. 303 if line.startswith ('WARNING: program timed out'): 304 has_warning = 1 305 else: 306 if has_warning == 1: 307 key = (name, len (harness.results)) 308 myline = 'WARNING: %s program timed out.\n' % name 309 harness.results.append ((key, myline)) 310 has_warning = 0 311 key = (name, len (harness.results)) 312 harness.results.append ((key, line)) 313 if not first_key and sort_logs: 314 first_key = key 315 if line.startswith ('ERROR: (DejaGnu)'): 316 for i in range (len (self.count_names)): 317 if 'DejaGnu errors' in self.count_names[i]: 318 variation.counts[i] += 1 319 break 320 321 # 'Using ...' lines are only interesting in a header. Splitting 322 # the test up into parallel runs leads to more 'Using ...' lines 323 # than there would be in a single log. 324 if line.startswith ('Using '): 325 final_using += 1 326 else: 327 final_using = 0 328 329 # Add other text to the current segment, if any. 330 if segment: 331 segment.lines += 1 332 333 # Close off the final harness segment, if any. 334 if harness: 335 segment.lines -= final_using 336 harness.add_segment (first_key, segment) 337 338 while True: 339 before = file.tell() 340 line = file.readline() 341 if line == '': 342 break 343 if line == '\n': 344 continue 345 if not line.startswith ('# '): 346 file.seek (before) 347 break 348 found = False 349 for i in range (len (self.count_names)): 350 if line.startswith (self.count_names[i]): 351 count = line[len (self.count_names[i]):-1].strip() 352 variation.counts[i] += self.parse_int (filename, count) 353 found = True 354 break 355 if not found: 356 self.fatal (filename, 'unknown test result: ' + line[:-1]) 357 358 # Parse an acats run, which uses a different format from dejagnu. 359 # We have just skipped over '=== acats configuration ==='. 360 def parse_acats_run (self, filename, file): 361 # Parse the preamble, which describes the configuration and logs 362 # the creation of support files. 363 record = (self.acats_premable == '') 364 if record: 365 self.acats_premable = '\t\t=== acats configuration ===\n' 366 while True: 367 line = file.readline() 368 if line == '': 369 self.fatal (filename, 'could not parse acats preamble') 370 if line == '\t\t=== acats tests ===\n': 371 break 372 if record: 373 self.acats_premable += line 374 375 # Parse the test results themselves, using a dummy variation name. 376 tool = self.get_tool ('acats') 377 variation = tool.get_variation ('none') 378 self.parse_run (filename, file, tool, variation, 1) 379 380 # Parse the failure list. 381 while True: 382 before = file.tell() 383 line = file.readline() 384 if line.startswith ('*** FAILURES: '): 385 self.acats_failures.append (line[len ('*** FAILURES: '):-1]) 386 continue 387 file.seek (before) 388 break 389 390 # Parse the final summary at the end of a log in order to capture 391 # the version output that follows it. 392 def parse_final_summary (self, filename, file): 393 record = (self.version_output == '') 394 while True: 395 line = file.readline() 396 if line == '': 397 break 398 if line.startswith ('# of '): 399 continue 400 if record: 401 self.version_output += line 402 if line == '\n': 403 break 404 405 # Parse a .log or .sum file. 406 def parse_file (self, filename, file): 407 tool = None 408 target = None 409 num_variations = 1 410 while True: 411 line = file.readline() 412 if line == '': 413 return 414 415 # Parse the list of variations, which comes before the test 416 # runs themselves. 417 if line.startswith ('Schedule of variations:'): 418 num_variations = self.parse_variations (filename, file) 419 continue 420 421 # Parse a testsuite run for one tool/variation combination. 422 if line.startswith ('Running target '): 423 name = line[len ('Running target '):-1] 424 if not tool: 425 self.fatal (filename, 'could not parse tool name') 426 if name not in self.known_variations: 427 self.fatal (filename, 'unknown target: ' + name) 428 self.parse_run (filename, file, tool, 429 tool.get_variation (name), 430 num_variations) 431 # If there is only one variation then there is no separate 432 # summary for it. Record any following version output. 433 if num_variations == 1: 434 self.parse_final_summary (filename, file) 435 continue 436 437 # Parse the start line. In the case where several files are being 438 # parsed, pick the one with the earliest time. 439 match = self.test_run_re.match (line) 440 if match: 441 time = self.parse_time (match.group (2)) 442 if not self.start_line or self.start_line[0] > time: 443 self.start_line = (time, line) 444 continue 445 446 # Parse the form used for native testing. 447 if line.startswith ('Native configuration is '): 448 self.native_line = line 449 continue 450 451 # Parse the target triplet. 452 if line.startswith ('Target is '): 453 self.target_line = line 454 continue 455 456 # Parse the host triplet. 457 if line.startswith ('Host is '): 458 self.host_line = line 459 continue 460 461 # Parse the acats premable. 462 if line == '\t\t=== acats configuration ===\n': 463 self.parse_acats_run (filename, file) 464 continue 465 466 # Parse the tool name. 467 match = self.tool_re.match (line) 468 if match: 469 tool = self.get_tool (match.group (1)) 470 continue 471 472 # Skip over the final summary (which we instead create from 473 # individual runs) and parse the version output. 474 if tool and line == '\t\t=== ' + tool.name + ' Summary ===\n': 475 if file.readline() != '\n': 476 self.fatal (filename, 'expected blank line after summary') 477 self.parse_final_summary (filename, file) 478 continue 479 480 # Parse the completion line. In the case where several files 481 # are being parsed, pick the one with the latest time. 482 match = self.completed_re.match (line) 483 if match: 484 time = self.parse_time (match.group (1)) 485 if not self.end_line or self.end_line[0] < time: 486 self.end_line = (time, line) 487 continue 488 489 # Sanity check to make sure that important text doesn't get 490 # dropped accidentally. 491 if strict and line.strip() != '': 492 self.fatal (filename, 'unrecognised line: ' + line[:-1]) 493 494 # Output a segment of text. 495 def output_segment (self, segment): 496 with safe_open (segment.filename) as file: 497 file.seek (segment.start) 498 for i in range (segment.lines): 499 sys.stdout.write (file.readline()) 500 501 # Output a summary giving the number of times each type of result has 502 # been seen. 503 def output_summary (self, tool, counts): 504 for i in range (len (self.count_names)): 505 name = self.count_names[i] 506 # dejagnu only prints result types that were seen at least once, 507 # but acats always prints a number of unexpected failures. 508 if (counts[i] > 0 509 or (tool.name == 'acats' 510 and name.startswith ('# of unexpected failures'))): 511 sys.stdout.write ('%s%d\n' % (name, counts[i])) 512 513 # Output unified .log or .sum information for a particular variation, 514 # with a summary at the end. 515 def output_variation (self, tool, variation): 516 self.output_segment (variation.header) 517 for harness in sorted (variation.harnesses.values(), 518 key = attrgetter ('name')): 519 sys.stdout.write ('Running ' + harness.name + ' ...\n') 520 if self.do_sum: 521 harness.results.sort() 522 for (key, line) in harness.results: 523 sys.stdout.write (line) 524 else: 525 # Rearrange the log segments into test order (but without 526 # rearranging text within those segments). 527 for key in sorted (harness.segments.keys()): 528 self.output_segment (harness.segments[key]) 529 for segment in harness.empty: 530 self.output_segment (segment) 531 if len (self.variations) > 1: 532 sys.stdout.write ('\t\t=== ' + tool.name + ' Summary for ' 533 + variation.name + ' ===\n\n') 534 self.output_summary (tool, variation.counts) 535 536 # Output unified .log or .sum information for a particular tool, 537 # with a summary at the end. 538 def output_tool (self, tool): 539 counts = self.zero_counts() 540 if tool.name == 'acats': 541 # acats doesn't use variations, so just output everything. 542 # It also has a different approach to whitespace. 543 sys.stdout.write ('\t\t=== ' + tool.name + ' tests ===\n') 544 for variation in tool.variations.values(): 545 self.output_variation (tool, variation) 546 self.accumulate_counts (counts, variation.counts) 547 sys.stdout.write ('\t\t=== ' + tool.name + ' Summary ===\n') 548 else: 549 # Output the results in the usual dejagnu runtest format. 550 sys.stdout.write ('\n\t\t=== ' + tool.name + ' tests ===\n\n' 551 'Schedule of variations:\n') 552 for name in self.variations: 553 if name in tool.variations: 554 sys.stdout.write (' ' + name + '\n') 555 sys.stdout.write ('\n') 556 for name in self.variations: 557 if name in tool.variations: 558 variation = tool.variations[name] 559 sys.stdout.write ('Running target ' 560 + variation.name + '\n') 561 self.output_variation (tool, variation) 562 self.accumulate_counts (counts, variation.counts) 563 sys.stdout.write ('\n\t\t=== ' + tool.name + ' Summary ===\n\n') 564 self.output_summary (tool, counts) 565 566 def main (self): 567 self.parse_cmdline() 568 try: 569 # Parse the input files. 570 for filename in self.files: 571 with safe_open (filename) as file: 572 self.parse_file (filename, file) 573 574 # Decide what to output. 575 if len (self.variations) == 0: 576 self.variations = sorted (self.known_variations) 577 else: 578 for name in self.variations: 579 if name not in self.known_variations: 580 self.fatal (None, 'no results for ' + name) 581 if len (self.tools) == 0: 582 self.tools = sorted (self.runs.keys()) 583 584 # Output the header. 585 if self.start_line: 586 sys.stdout.write (self.start_line[1]) 587 sys.stdout.write (self.native_line) 588 sys.stdout.write (self.target_line) 589 sys.stdout.write (self.host_line) 590 sys.stdout.write (self.acats_premable) 591 592 # Output the main body. 593 for name in self.tools: 594 if name not in self.runs: 595 self.fatal (None, 'no results for ' + name) 596 self.output_tool (self.runs[name]) 597 598 # Output the footer. 599 if len (self.acats_failures) > 0: 600 sys.stdout.write ('*** FAILURES: ' 601 + ' '.join (self.acats_failures) + '\n') 602 sys.stdout.write (self.version_output) 603 if self.end_line: 604 sys.stdout.write (self.end_line[1]) 605 except IOError as e: 606 self.fatal (e.filename, e.strerror) 607 608Prog().main() 609