| # Copyright (c) 2013 The Chromium OS Authors. All rights reserved. | |
| # Use of this source code is governed by a BSD-style license that can be | |
| # found in the LICENSE file. | |
| import logging | |
| import csv | |
| import cStringIO | |
| import random | |
| import re | |
| import collections | |
| from autotest_lib.client.common_lib.cros import path_utils | |
| class ResourceMonitorRawResult(object): | |
| """Encapsulates raw resource_monitor results.""" | |
| def __init__(self, raw_results_filename): | |
| self._raw_results_filename = raw_results_filename | |
| def get_parsed_results(self): | |
| """Constructs parsed results from the raw ones. | |
| @return ResourceMonitorParsedResult object | |
| """ | |
| return ResourceMonitorParsedResult(self.raw_results_filename) | |
| @property | |
| def raw_results_filename(self): | |
| """@return string filename storing the raw top command output.""" | |
| return self._raw_results_filename | |
| class IncorrectTopFormat(Exception): | |
| """Thrown if top output format is not as expected""" | |
| pass | |
| def _extract_value_before_single_keyword(line, keyword): | |
| """Extract word occurring immediately before the specified keyword. | |
| @param line string the line in which to search for the keyword. | |
| @param keyword string the keyword to look for. Can be a regexp. | |
| @return string the word just before the keyword. | |
| """ | |
| pattern = ".*?(\S+) " + keyword | |
| matches = re.match(pattern, line) | |
| if matches is None or len(matches.groups()) != 1: | |
| raise IncorrectTopFormat | |
| return matches.group(1) | |
| def _extract_values_before_keywords(line, *args): | |
| """Extract the words occuring immediately before each specified | |
| keyword in args. | |
| @param line string the string to look for the keywords. | |
| @param args variable number of string args the keywords to look for. | |
| @return string list the words occuring just before each keyword. | |
| """ | |
| line_nocomma = re.sub(",", " ", line) | |
| line_singlespace = re.sub("\s+", " ", line_nocomma) | |
| return [_extract_value_before_single_keyword( | |
| line_singlespace, arg) for arg in args] | |
| def _find_top_output_identifying_pattern(line): | |
| """Return true iff the line looks like the first line of top output. | |
| @param line string to look for the pattern | |
| @return boolean | |
| """ | |
| pattern ="\s*top\s*-.*up.*users.*" | |
| matches = re.match(pattern, line) | |
| return matches is not None | |
| class ResourceMonitorParsedResult(object): | |
| """Encapsulates logic to parse and represent top command results.""" | |
| _columns = ["Time", "UserCPU", "SysCPU", "NCPU", "Idle", | |
| "IOWait", "IRQ", "SoftIRQ", "Steal", | |
| "MemUnits", "UsedMem", "FreeMem", | |
| "SwapUnits", "UsedSwap", "FreeSwap"] | |
| UtilValues = collections.namedtuple('UtilValues', ' '.join(_columns)) | |
| def __init__(self, raw_results_filename): | |
| """Construct a ResourceMonitorResult. | |
| @param raw_results_filename string filename of raw batch top output. | |
| """ | |
| self._raw_results_filename = raw_results_filename | |
| self.parse_resource_monitor_results() | |
| def parse_resource_monitor_results(self): | |
| """Extract utilization metrics from output file.""" | |
| self._utils_over_time = [] | |
| with open(self._raw_results_filename, "r") as results_file: | |
| while True: | |
| curr_line = '\n' | |
| while curr_line != '' and \ | |
| not _find_top_output_identifying_pattern(curr_line): | |
| curr_line = results_file.readline() | |
| if curr_line == '': | |
| break | |
| try: | |
| time, = _extract_values_before_keywords(curr_line, "up") | |
| # Ignore one line. | |
| _ = results_file.readline() | |
| # Get the cpu usage. | |
| curr_line = results_file.readline() | |
| (cpu_user, cpu_sys, cpu_nice, cpu_idle, io_wait, irq, sirq, | |
| steal) = _extract_values_before_keywords(curr_line, | |
| "us", "sy", "ni", "id", "wa", "hi", "si", "st") | |
| # Get memory usage. | |
| curr_line = results_file.readline() | |
| (mem_units, mem_free, | |
| mem_used) = _extract_values_before_keywords( | |
| curr_line, "Mem", "free", "used") | |
| # Get swap usage. | |
| curr_line = results_file.readline() | |
| (swap_units, swap_free, | |
| swap_used) = _extract_values_before_keywords( | |
| curr_line, "Swap", "free", "used") | |
| curr_util_values = ResourceMonitorParsedResult.UtilValues( | |
| Time=time, UserCPU=cpu_user, | |
| SysCPU=cpu_sys, NCPU=cpu_nice, Idle=cpu_idle, | |
| IOWait=io_wait, IRQ=irq, SoftIRQ=sirq, Steal=steal, | |
| MemUnits=mem_units, UsedMem=mem_used, | |
| FreeMem=mem_free, | |
| SwapUnits=swap_units, UsedSwap=swap_used, | |
| FreeSwap=swap_free) | |
| self._utils_over_time.append(curr_util_values) | |
| except IncorrectTopFormat: | |
| logging.error( | |
| "Top output format incorrect. Aborting parse.") | |
| return | |
| def __repr__(self): | |
| output_stringfile = cStringIO.StringIO() | |
| self.save_to_file(output_stringfile) | |
| return output_stringfile.getvalue() | |
| def save_to_file(self, file): | |
| """Save parsed top results to file | |
| @param file file object to write to | |
| """ | |
| if len(self._utils_over_time) < 1: | |
| logging.warning("Tried to save parsed results, but they were " | |
| "empty. Skipping the save.") | |
| return | |
| csvwriter = csv.writer(file, delimiter=',') | |
| csvwriter.writerow(self._utils_over_time[0]._fields) | |
| for row in self._utils_over_time: | |
| csvwriter.writerow(row) | |
| def save_to_filename(self, filename): | |
| """Save parsed top results to filename | |
| @param filename string filepath to write to | |
| """ | |
| out_file = open(filename, "wb") | |
| self.save_to_file(out_file) | |
| out_file.close() | |
| class ResourceMonitorConfig(object): | |
| """Defines a single top run.""" | |
| DEFAULT_MONITOR_PERIOD = 3 | |
| def __init__(self, monitor_period=DEFAULT_MONITOR_PERIOD, | |
| rawresult_output_filename=None): | |
| """Construct a ResourceMonitorConfig. | |
| @param monitor_period float seconds between successive top refreshes. | |
| @param rawresult_output_filename string filename to output the raw top | |
| results to | |
| """ | |
| if monitor_period < 0.1: | |
| logging.info('Monitor period must be at least 0.1s.' | |
| ' Given: %r. Defaulting to 0.1s', monitor_period) | |
| monitor_period = 0.1 | |
| self._monitor_period = monitor_period | |
| self._server_outfile = rawresult_output_filename | |
| class ResourceMonitor(object): | |
| """Delegate to run top on a client. | |
| Usage example (call from a test): | |
| rmc = resource_monitor.ResourceMonitorConfig(monitor_period=1, | |
| rawresult_output_filename=os.path.join(self.resultsdir, | |
| 'topout.txt')) | |
| with resource_monitor.ResourceMonitor(self.context.client.host, rmc) as rm: | |
| rm.start() | |
| <operation_to_monitor> | |
| rm_raw_res = rm.stop() | |
| rm_res = rm_raw_res.get_parsed_results() | |
| rm_res.save_to_filename( | |
| os.path.join(self.resultsdir, 'resource_mon.csv')) | |
| """ | |
| def __init__(self, client_host, config): | |
| """Construct a ResourceMonitor. | |
| @param client_host: SSHHost object representing a remote ssh host | |
| """ | |
| self._client_host = client_host | |
| self._config = config | |
| self._command_top = path_utils.must_be_installed( | |
| 'top', host=self._client_host) | |
| self._top_pid = None | |
| def __enter__(self): | |
| return self | |
| def __exit__(self, exc_type, exc_value, traceback): | |
| if self._top_pid is not None: | |
| self._client_host.run('kill %s && rm %s' % | |
| (self._top_pid, self._client_outfile), ignore_status=True) | |
| return True | |
| def start(self): | |
| """Run top and save results to a temp file on the client.""" | |
| if self._top_pid is not None: | |
| logging.debug("Tried to start monitoring before stopping. " | |
| "Ignoring request.") | |
| return | |
| # Decide where to write top's output to (on the client). | |
| random_suffix = random.random() | |
| self._client_outfile = '/tmp/topcap-%r' % random_suffix | |
| # Run top on the client. | |
| top_command = '%s -b -d%d > %s' % (self._command_top, | |
| self._config._monitor_period, self._client_outfile) | |
| logging.info('Running top.') | |
| self._top_pid = self._client_host.run_background(top_command) | |
| logging.info('Top running with pid %s', self._top_pid) | |
| def stop(self): | |
| """Stop running top and return the results. | |
| @return ResourceMonitorRawResult object | |
| """ | |
| logging.debug("Stopping monitor") | |
| if self._top_pid is None: | |
| logging.debug("Tried to stop monitoring before starting. " | |
| "Ignoring request.") | |
| return | |
| # Stop top on the client. | |
| self._client_host.run('kill %s' % self._top_pid, ignore_status=True) | |
| # Get the top output file from the client onto the server. | |
| if self._config._server_outfile is None: | |
| self._config._server_outfile = self._client_outfile | |
| self._client_host.get_file( | |
| self._client_outfile, self._config._server_outfile) | |
| # Delete the top output file from client. | |
| self._client_host.run('rm %s' % self._client_outfile, | |
| ignore_status=True) | |
| self._top_pid = None | |
| logging.info("Saved resource monitor results at %s", | |
| self._config._server_outfile) | |
| return ResourceMonitorRawResult(self._config._server_outfile) |