1""" 2The LLVM Compiler Infrastructure 3 4This file is distributed under the University of Illinois Open Source 5License. See LICENSE.TXT for details. 6 7Provides classes used by the test results reporting infrastructure 8within the LLDB test suite. 9 10 11This module provides process-management support for the LLDB test 12running infrasructure. 13""" 14 15# System imports 16import os 17import re 18import signal 19import subprocess 20import sys 21import threading 22 23 24class CommunicatorThread(threading.Thread): 25 """Provides a thread class that communicates with a subprocess.""" 26 27 def __init__(self, process, event, output_file): 28 super(CommunicatorThread, self).__init__() 29 # Don't let this thread prevent shutdown. 30 self.daemon = True 31 self.process = process 32 self.pid = process.pid 33 self.event = event 34 self.output_file = output_file 35 self.output = None 36 37 def run(self): 38 try: 39 # Communicate with the child process. 40 # This will not complete until the child process terminates. 41 self.output = self.process.communicate() 42 except Exception as exception: # pylint: disable=broad-except 43 if self.output_file: 44 self.output_file.write( 45 "exception while using communicate() for pid: {}\n".format( 46 exception)) 47 finally: 48 # Signal that the thread's run is complete. 49 self.event.set() 50 51 52# Provides a regular expression for matching gtimeout-based durations. 53TIMEOUT_REGEX = re.compile(r"(^\d+)([smhd])?$") 54 55 56def timeout_to_seconds(timeout): 57 """Converts timeout/gtimeout timeout values into seconds. 58 59 @param timeout a timeout in the form of xm representing x minutes. 60 61 @return None if timeout is None, or the number of seconds as a float 62 if a valid timeout format was specified. 63 """ 64 if timeout is None: 65 return None 66 else: 67 match = TIMEOUT_REGEX.match(timeout) 68 if match: 69 value = float(match.group(1)) 70 units = match.group(2) 71 if units is None: 72 # default is seconds. No conversion necessary. 73 return value 74 elif units == 's': 75 # Seconds. No conversion necessary. 76 return value 77 elif units == 'm': 78 # Value is in minutes. 79 return 60.0 * value 80 elif units == 'h': 81 # Value is in hours. 82 return (60.0 * 60.0) * value 83 elif units == 'd': 84 # Value is in days. 85 return 24 * (60.0 * 60.0) * value 86 else: 87 raise Exception("unexpected units value '{}'".format(units)) 88 else: 89 raise Exception("could not parse TIMEOUT spec '{}'".format( 90 timeout)) 91 92 93class ProcessHelper(object): 94 """Provides an interface for accessing process-related functionality. 95 96 This class provides a factory method that gives the caller a 97 platform-specific implementation instance of the class. 98 99 Clients of the class should stick to the methods provided in this 100 base class. 101 102 @see ProcessHelper.process_helper() 103 """ 104 105 def __init__(self): 106 super(ProcessHelper, self).__init__() 107 108 @classmethod 109 def process_helper(cls): 110 """Returns a platform-specific ProcessHelper instance. 111 @return a ProcessHelper instance that does the right thing for 112 the current platform. 113 """ 114 115 # If you add a new platform, create an instance here and 116 # return it. 117 if os.name == "nt": 118 return WindowsProcessHelper() 119 else: 120 # For all POSIX-like systems. 121 return UnixProcessHelper() 122 123 def create_piped_process(self, command, new_process_group=True): 124 # pylint: disable=no-self-use,unused-argument 125 # As expected. We want derived classes to implement this. 126 """Creates a subprocess.Popen-based class with I/O piped to the parent. 127 128 @param command the command line list as would be passed to 129 subprocess.Popen(). Use the list form rather than the string form. 130 131 @param new_process_group indicates if the caller wants the 132 process to be created in its own process group. Each OS handles 133 this concept differently. It provides a level of isolation and 134 can simplify or enable terminating the process tree properly. 135 136 @return a subprocess.Popen-like object. 137 """ 138 raise Exception("derived class must implement") 139 140 def supports_soft_terminate(self): 141 # pylint: disable=no-self-use 142 # As expected. We want derived classes to implement this. 143 """Indicates if the platform supports soft termination. 144 145 Soft termination is the concept of a terminate mechanism that 146 allows the target process to shut down nicely, but with the 147 catch that the process might choose to ignore it. 148 149 Platform supporter note: only mark soft terminate as supported 150 if the target process has some way to evade the soft terminate 151 request; otherwise, just support the hard terminate method. 152 153 @return True if the platform supports a soft terminate mechanism. 154 """ 155 # By default, we do not support a soft terminate mechanism. 156 return False 157 158 def soft_terminate(self, popen_process, log_file=None, want_core=True): 159 # pylint: disable=no-self-use,unused-argument 160 # As expected. We want derived classes to implement this. 161 """Attempts to terminate the process in a polite way. 162 163 This terminate method is intended to give the child process a 164 chance to clean up and exit on its own, possibly with a request 165 to drop a core file or equivalent (i.e. [mini-]crashdump, crashlog, 166 etc.) If new_process_group was set in the process creation method 167 and the platform supports it, this terminate call will attempt to 168 kill the whole process tree rooted in this child process. 169 170 @param popen_process the subprocess.Popen-like object returned 171 by one of the process-creation methods of this class. 172 173 @param log_file file-like object used to emit error-related 174 logging info. May be None if no error-related info is desired. 175 176 @param want_core True if the caller would like to get a core 177 dump (or the analogous crash report) from the terminated process. 178 """ 179 popen_process.terminate() 180 181 def hard_terminate(self, popen_process, log_file=None): 182 # pylint: disable=no-self-use,unused-argument 183 # As expected. We want derived classes to implement this. 184 """Attempts to terminate the process immediately. 185 186 This terminate method is intended to kill child process in 187 a manner in which the child process has no ability to block, 188 and also has no ability to clean up properly. If new_process_group 189 was specified when creating the process, and if the platform 190 implementation supports it, this will attempt to kill the 191 whole process tree rooted in the child process. 192 193 @param popen_process the subprocess.Popen-like object returned 194 by one of the process-creation methods of this class. 195 196 @param log_file file-like object used to emit error-related 197 logging info. May be None if no error-related info is desired. 198 """ 199 popen_process.kill() 200 201 def was_soft_terminate(self, returncode, with_core): 202 # pylint: disable=no-self-use,unused-argument 203 # As expected. We want derived classes to implement this. 204 """Returns if Popen-like object returncode matches soft terminate. 205 206 @param returncode the returncode from the Popen-like object that 207 terminated with a given return code. 208 209 @param with_core indicates whether the returncode should match 210 a core-generating return signal. 211 212 @return True when the returncode represents what the system would 213 issue when a soft_terminate() with the given with_core arg occurred; 214 False otherwise. 215 """ 216 if not self.supports_soft_terminate(): 217 # If we don't support soft termination on this platform, 218 # then this should always be False. 219 return False 220 else: 221 # Once a platform claims to support soft terminate, it 222 # needs to be able to identify it by overriding this method. 223 raise Exception("platform needs to implement") 224 225 def was_hard_terminate(self, returncode): 226 # pylint: disable=no-self-use,unused-argument 227 # As expected. We want derived classes to implement this. 228 """Returns if Popen-like object returncode matches that of a hard 229 terminate attempt. 230 231 @param returncode the returncode from the Popen-like object that 232 terminated with a given return code. 233 234 @return True when the returncode represents what the system would 235 issue when a hard_terminate() occurred; False 236 otherwise. 237 """ 238 raise Exception("platform needs to implement") 239 240 def soft_terminate_signals(self): 241 # pylint: disable=no-self-use 242 """Retrieve signal numbers that can be sent to soft terminate. 243 @return a list of signal numbers that can be sent to soft terminate 244 a process, or None if not applicable. 245 """ 246 return None 247 248 def is_exceptional_exit(self, popen_status): 249 """Returns whether the program exit status is exceptional. 250 251 Returns whether the return code from a Popen process is exceptional 252 (e.g. signals on POSIX systems). 253 254 Derived classes should override this if they can detect exceptional 255 program exit. 256 257 @return True if the given popen_status represents an exceptional 258 program exit; False otherwise. 259 """ 260 return False 261 262 def exceptional_exit_details(self, popen_status): 263 """Returns the normalized exceptional exit code and a description. 264 265 Given an exceptional exit code, returns the integral value of the 266 exception (e.g. signal number for POSIX) and a description (e.g. 267 signal name on POSIX) for the result. 268 269 Derived classes should override this if they can detect exceptional 270 program exit. 271 272 It is fine to not implement this so long as is_exceptional_exit() 273 always returns False. 274 275 @return (normalized exception code, symbolic exception description) 276 """ 277 raise Exception("exception_exit_details() called on unsupported class") 278 279 280class UnixProcessHelper(ProcessHelper): 281 """Provides a ProcessHelper for Unix-like operating systems. 282 283 This implementation supports anything that looks Posix-y 284 (e.g. Darwin, Linux, *BSD, etc.) 285 """ 286 287 def __init__(self): 288 super(UnixProcessHelper, self).__init__() 289 290 @classmethod 291 def _create_new_process_group(cls): 292 """Creates a new process group for the calling process.""" 293 os.setpgid(os.getpid(), os.getpid()) 294 295 def create_piped_process(self, command, new_process_group=True): 296 # Determine what to run after the fork but before the exec. 297 if new_process_group: 298 preexec_func = self._create_new_process_group 299 else: 300 preexec_func = None 301 302 # Create the process. 303 process = subprocess.Popen( 304 command, 305 stdin=subprocess.PIPE, 306 stdout=subprocess.PIPE, 307 stderr=subprocess.PIPE, 308 universal_newlines=True, # Elicits automatic byte -> string decoding in Py3 309 close_fds=True, 310 preexec_fn=preexec_func) 311 312 # Remember whether we're using process groups for this 313 # process. 314 process.using_process_groups = new_process_group 315 return process 316 317 def supports_soft_terminate(self): 318 # POSIX does support a soft terminate via: 319 # * SIGTERM (no core requested) 320 # * SIGQUIT (core requested if enabled, see ulimit -c) 321 return True 322 323 @classmethod 324 def _validate_pre_terminate(cls, popen_process, log_file): 325 # Validate args. 326 if popen_process is None: 327 raise ValueError("popen_process is None") 328 329 # Ensure we have something that looks like a valid process. 330 if popen_process.pid < 1: 331 if log_file: 332 log_file.write("skipping soft_terminate(): no process id") 333 return False 334 335 # We only do the process liveness check if we're not using 336 # process groups. With process groups, checking if the main 337 # inferior process is dead and short circuiting here is no 338 # good - children of it in the process group could still be 339 # alive, and they should be killed during a timeout. 340 if not popen_process.using_process_groups: 341 # Don't kill if it's already dead. 342 popen_process.poll() 343 if popen_process.returncode is not None: 344 # It has a returncode. It has already stopped. 345 if log_file: 346 log_file.write( 347 "requested to terminate pid {} but it has already " 348 "terminated, returncode {}".format( 349 popen_process.pid, popen_process.returncode)) 350 # Move along... 351 return False 352 353 # Good to go. 354 return True 355 356 def _kill_with_signal(self, popen_process, log_file, signum): 357 # Validate we're ready to terminate this. 358 if not self._validate_pre_terminate(popen_process, log_file): 359 return 360 361 # Choose kill mechanism based on whether we're targeting 362 # a process group or just a process. 363 try: 364 if popen_process.using_process_groups: 365 # if log_file: 366 # log_file.write( 367 # "sending signum {} to process group {} now\n".format( 368 # signum, popen_process.pid)) 369 os.killpg(popen_process.pid, signum) 370 else: 371 # if log_file: 372 # log_file.write( 373 # "sending signum {} to process {} now\n".format( 374 # signum, popen_process.pid)) 375 os.kill(popen_process.pid, signum) 376 except OSError as error: 377 import errno 378 if error.errno == errno.ESRCH: 379 # This is okay - failed to find the process. It may be that 380 # that the timeout pre-kill hook eliminated the process. We'll 381 # ignore. 382 pass 383 else: 384 raise 385 386 def soft_terminate(self, popen_process, log_file=None, want_core=True): 387 # Choose signal based on desire for core file. 388 if want_core: 389 # SIGQUIT will generate core by default. Can be caught. 390 signum = signal.SIGQUIT 391 else: 392 # SIGTERM is the traditional nice way to kill a process. 393 # Can be caught, doesn't generate a core. 394 signum = signal.SIGTERM 395 396 self._kill_with_signal(popen_process, log_file, signum) 397 398 def hard_terminate(self, popen_process, log_file=None): 399 self._kill_with_signal(popen_process, log_file, signal.SIGKILL) 400 401 def was_soft_terminate(self, returncode, with_core): 402 if with_core: 403 return returncode == -signal.SIGQUIT 404 else: 405 return returncode == -signal.SIGTERM 406 407 def was_hard_terminate(self, returncode): 408 return returncode == -signal.SIGKILL 409 410 def soft_terminate_signals(self): 411 return [signal.SIGQUIT, signal.SIGTERM] 412 413 def is_exceptional_exit(self, popen_status): 414 return popen_status < 0 415 416 @classmethod 417 def _signal_names_by_number(cls): 418 return dict( 419 (k, v) for v, k in reversed(sorted(signal.__dict__.items())) 420 if v.startswith('SIG') and not v.startswith('SIG_')) 421 422 def exceptional_exit_details(self, popen_status): 423 signo = -popen_status 424 signal_names_by_number = self._signal_names_by_number() 425 signal_name = signal_names_by_number.get(signo, "") 426 return (signo, signal_name) 427 428 429class WindowsProcessHelper(ProcessHelper): 430 """Provides a Windows implementation of the ProcessHelper class.""" 431 432 def __init__(self): 433 super(WindowsProcessHelper, self).__init__() 434 435 def create_piped_process(self, command, new_process_group=True): 436 if new_process_group: 437 # We need this flag if we want os.kill() to work on the subprocess. 438 creation_flags = subprocess.CREATE_NEW_PROCESS_GROUP 439 else: 440 creation_flags = 0 441 442 return subprocess.Popen( 443 command, 444 stdin=subprocess.PIPE, 445 stdout=subprocess.PIPE, 446 stderr=subprocess.PIPE, 447 universal_newlines=True, # Elicits automatic byte -> string decoding in Py3 448 creationflags=creation_flags) 449 450 def was_hard_terminate(self, returncode): 451 return returncode != 0 452 453 454class ProcessDriver(object): 455 """Drives a child process, notifies on important events, and can timeout. 456 457 Clients are expected to derive from this class and override the 458 on_process_started and on_process_exited methods if they want to 459 hook either of those. 460 461 This class supports timing out the child process in a platform-agnostic 462 way. The on_process_exited method is informed if the exit was natural 463 or if it was due to a timeout. 464 """ 465 466 def __init__(self, soft_terminate_timeout=10.0): 467 super(ProcessDriver, self).__init__() 468 self.process_helper = ProcessHelper.process_helper() 469 self.pid = None 470 # Create the synchronization event for notifying when the 471 # inferior dotest process is complete. 472 self.done_event = threading.Event() 473 self.io_thread = None 474 self.process = None 475 # Number of seconds to wait for the soft terminate to 476 # wrap up, before moving to more drastic measures. 477 # Might want this longer if core dumps are generated and 478 # take a long time to write out. 479 self.soft_terminate_timeout = soft_terminate_timeout 480 # Number of seconds to wait for the hard terminate to 481 # wrap up, before giving up on the io thread. This should 482 # be fast. 483 self.hard_terminate_timeout = 5.0 484 self.returncode = None 485 486 # ============================================= 487 # Methods for subclasses to override if desired. 488 # ============================================= 489 490 def on_process_started(self): 491 pass 492 493 def on_process_exited(self, command, output, was_timeout, exit_status): 494 pass 495 496 def on_timeout_pre_kill(self): 497 """Called after the timeout interval elapses but before killing it. 498 499 This method is added to enable derived classes the ability to do 500 something to the process prior to it being killed. For example, 501 this would be a good spot to run a program that samples the process 502 to see what it was doing (or not doing). 503 504 Do not attempt to reap the process (i.e. use wait()) in this method. 505 That will interfere with the kill mechanism and return code processing. 506 """ 507 pass 508 509 def write(self, content): 510 # pylint: disable=no-self-use 511 # Intended - we want derived classes to be able to override 512 # this and use any self state they may contain. 513 sys.stdout.write(content) 514 515 # ============================================================== 516 # Operations used to drive processes. Clients will want to call 517 # one of these. 518 # ============================================================== 519 520 def run_command(self, command): 521 # Start up the child process and the thread that does the 522 # communication pump. 523 self._start_process_and_io_thread(command) 524 525 # Wait indefinitely for the child process to finish 526 # communicating. This indicates it has closed stdout/stderr 527 # pipes and is done. 528 self.io_thread.join() 529 self.returncode = self.process.wait() 530 if self.returncode is None: 531 raise Exception( 532 "no exit status available for pid {} after the " 533 " inferior dotest.py should have completed".format( 534 self.process.pid)) 535 536 # Notify of non-timeout exit. 537 self.on_process_exited( 538 command, 539 self.io_thread.output, 540 False, 541 self.returncode) 542 543 def run_command_with_timeout(self, command, timeout, want_core): 544 # Figure out how many seconds our timeout description is requesting. 545 timeout_seconds = timeout_to_seconds(timeout) 546 547 # Start up the child process and the thread that does the 548 # communication pump. 549 self._start_process_and_io_thread(command) 550 551 self._wait_with_timeout(timeout_seconds, command, want_core) 552 553 # ================ 554 # Internal details. 555 # ================ 556 557 def _start_process_and_io_thread(self, command): 558 # Create the process. 559 self.process = self.process_helper.create_piped_process(command) 560 self.pid = self.process.pid 561 self.on_process_started() 562 563 # Ensure the event is cleared that is used for signaling 564 # from the communication() thread when communication is 565 # complete (i.e. the inferior process has finished). 566 self.done_event.clear() 567 568 self.io_thread = CommunicatorThread( 569 self.process, self.done_event, self.write) 570 self.io_thread.start() 571 572 def _attempt_soft_kill(self, want_core): 573 # The inferior dotest timed out. Attempt to clean it 574 # with a non-drastic method (so it can clean up properly 575 # and/or generate a core dump). Often the OS can't guarantee 576 # that the process will really terminate after this. 577 self.process_helper.soft_terminate( 578 self.process, 579 want_core=want_core, 580 log_file=self) 581 582 # Now wait up to a certain timeout period for the io thread 583 # to say that the communication ended. If that wraps up 584 # within our soft terminate timeout, we're all done here. 585 self.io_thread.join(self.soft_terminate_timeout) 586 if not self.io_thread.is_alive(): 587 # stdout/stderr were closed on the child process side. We 588 # should be able to wait and reap the child process here. 589 self.returncode = self.process.wait() 590 # We terminated, and the done_trying result is n/a 591 terminated = True 592 done_trying = None 593 else: 594 self.write("soft kill attempt of process {} timed out " 595 "after {} seconds\n".format( 596 self.process.pid, self.soft_terminate_timeout)) 597 terminated = False 598 done_trying = False 599 return terminated, done_trying 600 601 def _attempt_hard_kill(self): 602 # Instruct the process to terminate and really force it to 603 # happen. Don't give the process a chance to ignore. 604 self.process_helper.hard_terminate( 605 self.process, 606 log_file=self) 607 608 # Reap the child process. This should not hang as the 609 # hard_kill() mechanism is supposed to really kill it. 610 # Improvement option: 611 # If this does ever hang, convert to a self.process.poll() 612 # loop checking on self.process.returncode until it is not 613 # None or the timeout occurs. 614 self.returncode = self.process.wait() 615 616 # Wait a few moments for the io thread to finish... 617 self.io_thread.join(self.hard_terminate_timeout) 618 if self.io_thread.is_alive(): 619 # ... but this is not critical if it doesn't end for some 620 # reason. 621 self.write( 622 "hard kill of process {} timed out after {} seconds waiting " 623 "for the io thread (ignoring)\n".format( 624 self.process.pid, self.hard_terminate_timeout)) 625 626 # Set if it terminated. (Set up for optional improvement above). 627 terminated = self.returncode is not None 628 # Nothing else to try. 629 done_trying = True 630 631 return terminated, done_trying 632 633 def _attempt_termination(self, attempt_count, want_core): 634 if self.process_helper.supports_soft_terminate(): 635 # When soft termination is supported, we first try to stop 636 # the process with a soft terminate. Failing that, we try 637 # the hard terminate option. 638 if attempt_count == 1: 639 return self._attempt_soft_kill(want_core) 640 elif attempt_count == 2: 641 return self._attempt_hard_kill() 642 else: 643 # We don't have anything else to try. 644 terminated = self.returncode is not None 645 done_trying = True 646 return terminated, done_trying 647 else: 648 # We only try the hard terminate option when there 649 # is no soft terminate available. 650 if attempt_count == 1: 651 return self._attempt_hard_kill() 652 else: 653 # We don't have anything else to try. 654 terminated = self.returncode is not None 655 done_trying = True 656 return terminated, done_trying 657 658 def _wait_with_timeout(self, timeout_seconds, command, want_core): 659 # Allow up to timeout seconds for the io thread to wrap up. 660 # If that completes, the child process should be done. 661 completed_normally = self.done_event.wait(timeout_seconds) 662 if completed_normally: 663 # Reap the child process here. 664 self.returncode = self.process.wait() 665 else: 666 667 # Allow derived classes to do some work after we detected 668 # a timeout but before we touch the timed-out process. 669 self.on_timeout_pre_kill() 670 671 # Prepare to stop the process 672 process_terminated = completed_normally 673 terminate_attempt_count = 0 674 675 # Try as many attempts as we support for trying to shut down 676 # the child process if it's not already shut down. 677 while not process_terminated: 678 terminate_attempt_count += 1 679 # Attempt to terminate. 680 process_terminated, done_trying = self._attempt_termination( 681 terminate_attempt_count, want_core) 682 # Check if there's nothing more to try. 683 if done_trying: 684 # Break out of our termination attempt loop. 685 break 686 687 # At this point, we're calling it good. The process 688 # finished gracefully, was shut down after one or more 689 # attempts, or we failed but gave it our best effort. 690 self.on_process_exited( 691 command, 692 self.io_thread.output, 693 not completed_normally, 694 self.returncode) 695 696 697def patched_init(self, *args, **kwargs): 698 self.original_init(*args, **kwargs) 699 # Initialize our condition variable that protects wait()/poll(). 700 self.wait_condition = threading.Condition() 701 702 703def patched_wait(self, *args, **kwargs): 704 self.wait_condition.acquire() 705 try: 706 result = self.original_wait(*args, **kwargs) 707 # The process finished. Signal the condition. 708 self.wait_condition.notify_all() 709 return result 710 finally: 711 self.wait_condition.release() 712 713 714def patched_poll(self, *args, **kwargs): 715 self.wait_condition.acquire() 716 try: 717 result = self.original_poll(*args, **kwargs) 718 if self.returncode is not None: 719 # We did complete, and we have the return value. 720 # Signal the event to indicate we're done. 721 self.wait_condition.notify_all() 722 return result 723 finally: 724 self.wait_condition.release() 725 726 727def patch_up_subprocess_popen(): 728 subprocess.Popen.original_init = subprocess.Popen.__init__ 729 subprocess.Popen.__init__ = patched_init 730 731 subprocess.Popen.original_wait = subprocess.Popen.wait 732 subprocess.Popen.wait = patched_wait 733 734 subprocess.Popen.original_poll = subprocess.Popen.poll 735 subprocess.Popen.poll = patched_poll 736 737# Replace key subprocess.Popen() threading-unprotected methods with 738# threading-protected versions. 739patch_up_subprocess_popen() 740