| #pylint: disable-msg=C0111 | |
| """ | |
| Pidfile monitor. | |
| """ | |
| import logging | |
| import time | |
| import traceback | |
| import common | |
| from autotest_lib.client.common_lib import utils | |
| from autotest_lib.client.common_lib import global_config | |
| from autotest_lib.scheduler import drone_manager | |
| from autotest_lib.scheduler import scheduler_config | |
| try: | |
| from chromite.lib import metrics | |
| except ImportError: | |
| metrics = utils.metrics_mock | |
| def _get_pidfile_timeout_secs(): | |
| """@returns How long to wait for autoserv to write pidfile.""" | |
| pidfile_timeout_mins = global_config.global_config.get_config_value( | |
| scheduler_config.CONFIG_SECTION, 'pidfile_timeout_mins', type=int) | |
| return pidfile_timeout_mins * 60 | |
| class PidfileRunMonitor(object): | |
| """ | |
| Client must call either run() to start a new process or | |
| attach_to_existing_process(). | |
| """ | |
| class _PidfileException(Exception): | |
| """ | |
| Raised when there's some unexpected behavior with the pid file, but only | |
| used internally (never allowed to escape this class). | |
| """ | |
| def __init__(self): | |
| self._drone_manager = drone_manager.instance() | |
| self.lost_process = False | |
| self._start_time = None | |
| self.pidfile_id = None | |
| self._killed = False | |
| self._state = drone_manager.PidfileContents() | |
| def _add_nice_command(self, command, nice_level): | |
| if not nice_level: | |
| return command | |
| return ['nice', '-n', str(nice_level)] + command | |
| def _set_start_time(self): | |
| self._start_time = time.time() | |
| def run(self, command, working_directory, num_processes, nice_level=None, | |
| log_file=None, pidfile_name=None, paired_with_pidfile=None, | |
| username=None, drone_hostnames_allowed=None): | |
| assert command is not None | |
| if nice_level is not None: | |
| command = ['nice', '-n', str(nice_level)] + command | |
| self._set_start_time() | |
| self.pidfile_id = self._drone_manager.execute_command( | |
| command, working_directory, pidfile_name=pidfile_name, | |
| num_processes=num_processes, log_file=log_file, | |
| paired_with_pidfile=paired_with_pidfile, username=username, | |
| drone_hostnames_allowed=drone_hostnames_allowed) | |
| def attach_to_existing_process(self, execution_path, | |
| pidfile_name=drone_manager.AUTOSERV_PID_FILE, | |
| num_processes=None): | |
| self._set_start_time() | |
| self.pidfile_id = self._drone_manager.get_pidfile_id_from( | |
| execution_path, pidfile_name=pidfile_name) | |
| if num_processes is not None: | |
| self._drone_manager.declare_process_count(self.pidfile_id, num_processes) | |
| def kill(self): | |
| if self.has_process(): | |
| self._drone_manager.kill_process(self.get_process()) | |
| self._killed = True | |
| def has_process(self): | |
| self._get_pidfile_info() | |
| return self._state.process is not None | |
| def get_process(self): | |
| self._get_pidfile_info() | |
| assert self._state.process is not None | |
| return self._state.process | |
| def _read_pidfile(self, use_second_read=False): | |
| assert self.pidfile_id is not None, ( | |
| 'You must call run() or attach_to_existing_process()') | |
| contents = self._drone_manager.get_pidfile_contents( | |
| self.pidfile_id, use_second_read=use_second_read) | |
| if contents.is_invalid(): | |
| self._state = drone_manager.PidfileContents() | |
| raise self._PidfileException(contents) | |
| self._state = contents | |
| def _handle_pidfile_error(self, error, message=''): | |
| self.on_lost_process(self._state.process) | |
| def _get_pidfile_info_helper(self): | |
| if self.lost_process: | |
| return | |
| self._read_pidfile() | |
| if self._state.process is None: | |
| self._handle_no_process() | |
| return | |
| if self._state.exit_status is None: | |
| # double check whether or not autoserv is running | |
| if self._drone_manager.is_process_running(self._state.process): | |
| return | |
| # pid but no running process - maybe process *just* exited | |
| self._read_pidfile(use_second_read=True) | |
| if self._state.exit_status is None: | |
| # autoserv exited without writing an exit code | |
| # to the pidfile | |
| self._handle_pidfile_error( | |
| 'autoserv died without writing exit code') | |
| def _get_pidfile_info(self): | |
| """\ | |
| After completion, self._state will contain: | |
| pid=None, exit_status=None if autoserv has not yet run | |
| pid!=None, exit_status=None if autoserv is running | |
| pid!=None, exit_status!=None if autoserv has completed | |
| """ | |
| try: | |
| self._get_pidfile_info_helper() | |
| except self._PidfileException, exc: | |
| self._handle_pidfile_error('Pidfile error', traceback.format_exc()) | |
| def _handle_no_process(self): | |
| """\ | |
| Called when no pidfile is found or no pid is in the pidfile. | |
| """ | |
| if time.time() - self._start_time > _get_pidfile_timeout_secs(): | |
| # If we aborted the process, and we find that it has exited without | |
| # writing a pidfile, then it's because we killed it, and thus this | |
| # isn't a surprising situation. | |
| if not self._killed: | |
| metrics.Counter('chromeos/autotest/errors/scheduler/no_pidfile' | |
| ).increment() | |
| else: | |
| logging.warning("%s didn't exit after SIGTERM", self.pidfile_id) | |
| self.on_lost_process() | |
| def on_lost_process(self, process=None): | |
| """\ | |
| Called when autoserv has exited without writing an exit status, | |
| or we've timed out waiting for autoserv to write a pid to the | |
| pidfile. In either case, we just return failure and the caller | |
| should signal some kind of warning. | |
| process is unimportant here, as it shouldn't be used by anyone. | |
| """ | |
| self.lost_process = True | |
| self._state.process = process | |
| self._state.exit_status = 1 | |
| self._state.num_tests_failed = 0 | |
| def exit_code(self): | |
| self._get_pidfile_info() | |
| return self._state.exit_status | |
| def num_tests_failed(self): | |
| """@returns The number of tests that failed or -1 if unknown.""" | |
| self._get_pidfile_info() | |
| if self._state.num_tests_failed is None: | |
| return -1 | |
| return self._state.num_tests_failed | |
| def try_copy_results_on_drone(self, **kwargs): | |
| if self.has_process(): | |
| # copy results logs into the normal place for job results | |
| self._drone_manager.copy_results_on_drone(self.get_process(), **kwargs) | |
| def try_copy_to_results_repository(self, source, **kwargs): | |
| if self.has_process(): | |
| self._drone_manager.copy_to_results_repository(self.get_process(), | |
| source, **kwargs) | |