| #!/usr/bin/python |
| # |
| # Copyright 2007 Google Inc. Released under the GPL v2 |
| |
| """ |
| This module defines the Autotest class |
| |
| Autotest: software to run tests automatically |
| """ |
| |
| __author__ = """ |
| [email protected] (Martin J. Bligh), |
| [email protected] (Benjamin Poirier), |
| [email protected] (Ryan Stutsman) |
| """ |
| |
| import re, os, sys, traceback, subprocess, tempfile, shutil, time |
| |
| from autotest_lib.server import installable_object, utils, server_job |
| from autotest_lib.client.common_lib import logging |
| from autotest_lib.client.common_lib import error |
| |
| |
| AUTOTEST_SVN = 'svn://test.kernel.org/autotest/trunk/client' |
| AUTOTEST_HTTP = 'http://test.kernel.org/svn/autotest/trunk/client' |
| |
| # Timeouts for powering down and up respectively |
| HALT_TIME = 300 |
| BOOT_TIME = 1800 |
| CRASH_RECOVERY_TIME = 9000 |
| |
| |
| class BaseAutotest(installable_object.InstallableObject): |
| """ |
| This class represents the Autotest program. |
| |
| Autotest is used to run tests automatically and collect the results. |
| It also supports profilers. |
| |
| Implementation details: |
| This is a leaf class in an abstract class hierarchy, it must |
| implement the unimplemented methods in parent classes. |
| """ |
| job = None |
| |
| |
| def __init__(self, host = None): |
| self.host = host |
| self.got = False |
| self.installed = False |
| self.serverdir = utils.get_server_dir() |
| super(BaseAutotest, self).__init__() |
| |
| |
| @logging.record |
| def install(self, host = None): |
| """ |
| Install autotest. If get() was not called previously, an |
| attempt will be made to install from the autotest svn |
| repository. |
| |
| Args: |
| host: a Host instance on which autotest will be |
| installed |
| |
| Raises: |
| AutoservError: if a tarball was not specified and |
| the target host does not have svn installed in its path |
| |
| TODO(poirier): check dependencies |
| autotest needs: |
| bzcat |
| liboptdev (oprofile) |
| binutils-dev (oprofile) |
| make |
| psutils (netperf) |
| """ |
| if not host: |
| host = self.host |
| if not self.got: |
| self.get() |
| host.wait_up(timeout=30) |
| host.setup() |
| print "Installing autotest on %s" % host.hostname |
| |
| # Let's try to figure out where autotest is installed. If we can't, |
| # (autotest not installed) just assume '/usr/local/autotest' and |
| # proceed. |
| try: |
| autodir = _get_autodir(host) |
| except error.AutotestRunError: |
| autodir = '/usr/local/autotest' |
| |
| host.run('mkdir -p "%s"' % utils.sh_escape(autodir)) |
| |
| if getattr(host, 'site_install_autotest', None): |
| if host.site_install_autotest(): |
| self.installed = True |
| return |
| |
| # try to install from file or directory |
| if self.source_material: |
| if os.path.isdir(self.source_material): |
| # Copy autotest recursively |
| host.send_file(self.source_material, autodir) |
| else: |
| # Copy autotest via tarball |
| e_msg = 'Installation method not yet implemented!' |
| raise NotImplementedError(e_msg) |
| print "Installation of autotest completed" |
| self.installed = True |
| return |
| |
| # if that fails try to install using svn |
| if utils.run('which svn').exit_status: |
| raise error.AutoservError('svn not found in path on \ |
| target machine: %s' % host.name) |
| try: |
| host.run('svn checkout %s %s' % |
| (AUTOTEST_SVN, autodir)) |
| except error.AutoservRunError, e: |
| host.run('svn checkout %s %s' % |
| (AUTOTEST_HTTP, autodir)) |
| print "Installation of autotest completed" |
| self.installed = True |
| |
| |
| def get(self, location = None): |
| if not location: |
| location = os.path.join(self.serverdir, '../client') |
| location = os.path.abspath(location) |
| # If there's stuff run on our client directory already, it |
| # can cause problems. Try giving it a quick clean first. |
| cwd = os.getcwd() |
| os.chdir(location) |
| os.system('tools/make_clean') |
| os.chdir(cwd) |
| super(BaseAutotest, self).get(location) |
| self.got = True |
| |
| |
| def run(self, control_file, results_dir = '.', host = None, |
| timeout=None, tag=None, parallel_flag=False): |
| """ |
| Run an autotest job on the remote machine. |
| |
| Args: |
| control_file: an open file-like-obj of the control file |
| results_dir: a str path where the results should be stored |
| on the local filesystem |
| host: a Host instance on which the control file should |
| be run |
| tag: tag name for the client side instance of autotest |
| parallel_flag: flag set when multiple jobs are run at the |
| same time |
| Raises: |
| AutotestRunError: if there is a problem executing |
| the control file |
| """ |
| host = self._get_host_and_setup(host) |
| results_dir = os.path.abspath(results_dir) |
| |
| if tag: |
| results_dir = os.path.join(results_dir, tag) |
| |
| atrun = _Run(host, results_dir, tag, parallel_flag) |
| self._do_run(control_file, results_dir, host, atrun, timeout) |
| |
| |
| def _get_host_and_setup(self, host): |
| if not host: |
| host = self.host |
| if not self.installed: |
| self.install(host) |
| |
| host.wait_up(timeout=30) |
| return host |
| |
| |
| def prepare_for_copying_logs(self, src, dest, host): |
| keyval_path = '' |
| if not os.path.exists(os.path.join(dest, 'keyval')): |
| # Client-side keyval file can be copied directly |
| return keyval_path |
| # Copy client-side keyval to temporary location |
| try: |
| try: |
| # Create temp file |
| fd, keyval_path = tempfile.mkstemp( |
| '.keyval_%s' % host.hostname) |
| host.get_file(os.path.join(src, 'keyval'), |
| keyval_path) |
| finally: |
| # We will squirrel away the client side keyval |
| # away and move it back when we are done |
| self.temp_keyval_path = tempfile.mktemp() |
| host.run('mv %s %s' % |
| (os.path.join(src, 'keyval'), |
| self.temp_keyval_path)) |
| except (error.AutoservRunError, error.AutoservSSHTimeout): |
| print "Prepare for copying logs failed" |
| return keyval_path |
| |
| |
| def process_copied_logs(self, dest, host, keyval_path): |
| if not os.path.exists(os.path.join(dest, 'keyval')): |
| # Client-side keyval file was copied directly |
| return |
| # Append contents of keyval_<host> file to keyval file |
| try: |
| # Read in new and old keyval files |
| new_keyval = utils.read_keyval(keyval_path) |
| old_keyval = utils.read_keyval(dest) |
| # 'Delete' from new keyval entries that are in both |
| tmp_keyval = {} |
| for key, val in new_keyval.iteritems(): |
| if key not in old_keyval: |
| tmp_keyval[key] = val |
| # Append new info to keyval file |
| utils.write_keyval(dest, tmp_keyval) |
| # Delete keyval_<host> file |
| os.remove(keyval_path) |
| except IOError: |
| print "Process copied logs failed" |
| |
| |
| def postprocess_copied_logs(self, src, host): |
| # we can now put our keyval file back |
| try: |
| host.run('mv %s %s' % (self.temp_keyval_path, |
| os.path.join(src, 'keyval'))) |
| except: |
| pass |
| |
| |
| def _do_run(self, control_file, results_dir, host, atrun, timeout): |
| try: |
| atrun.verify_machine() |
| except: |
| print "Verify machine failed on %s. Reinstalling" % \ |
| host.hostname |
| self.install(host) |
| atrun.verify_machine() |
| debug = os.path.join(results_dir, 'debug') |
| try: |
| os.makedirs(debug) |
| except: |
| pass |
| |
| # Ready .... Aim .... |
| for control in [atrun.remote_control_file, |
| atrun.remote_control_file + '.state', |
| atrun.manual_control_file, |
| atrun.manual_control_file + '.state']: |
| host.run('rm -f ' + control) |
| |
| # Copy control_file to remote_control_file on the host |
| tmppath = utils.get(control_file) |
| host.send_file(tmppath, atrun.remote_control_file) |
| if os.path.abspath(tmppath) != os.path.abspath(control_file): |
| os.remove(tmppath) |
| |
| try: |
| atrun.execute_control(timeout=timeout) |
| finally: |
| # make an effort to wait for the machine to come up |
| try: |
| host.wait_up(timeout=30) |
| except error.AutoservError: |
| # don't worry about any errors, we'll try and |
| # get the results anyway |
| pass |
| |
| # get the results |
| if not atrun.tag: |
| results = os.path.join(atrun.autodir, |
| 'results', 'default') |
| else: |
| results = os.path.join(atrun.autodir, |
| 'results', atrun.tag) |
| |
| # Copy all dirs in default to results_dir |
| keyval_path = self.prepare_for_copying_logs(results, |
| results_dir, host) |
| host.get_file(results + '/', results_dir) |
| self.process_copied_logs(results_dir, host, keyval_path) |
| self.postprocess_copied_logs(results, host) |
| |
| |
| def run_timed_test(self, test_name, results_dir='.', host=None, |
| timeout=None, tag=None, *args, **dargs): |
| """ |
| Assemble a tiny little control file to just run one test, |
| and run it as an autotest client-side test |
| """ |
| if not host: |
| host = self.host |
| if not self.installed: |
| self.install(host) |
| opts = ["%s=%s" % (o[0], repr(o[1])) for o in dargs.items()] |
| cmd = ", ".join([repr(test_name)] + map(repr, args) + opts) |
| control = "job.run_test(%s)\n" % cmd |
| self.run(control, results_dir, host, timeout=timeout, tag=tag) |
| |
| |
| def run_test(self, test_name, results_dir='.', host=None, |
| tag=None, *args, **dargs): |
| self.run_timed_test(test_name, results_dir, host, timeout=None, |
| tag=tag, *args, **dargs) |
| |
| |
| class _Run(object): |
| """ |
| Represents a run of autotest control file. This class maintains |
| all the state necessary as an autotest control file is executed. |
| |
| It is not intended to be used directly, rather control files |
| should be run using the run method in Autotest. |
| """ |
| def __init__(self, host, results_dir, tag, parallel_flag): |
| self.host = host |
| self.results_dir = results_dir |
| self.env = host.env |
| self.tag = tag |
| self.parallel_flag = parallel_flag |
| self.autodir = _get_autodir(self.host) |
| if tag: |
| self.manual_control_file = os.path.join(self.autodir, |
| 'control.%s' % tag) |
| self.remote_control_file = os.path.join(self.autodir, |
| 'control.%s.autoserv' % tag) |
| else: |
| self.manual_control_file = os.path.join(self.autodir, |
| 'control') |
| self.remote_control_file = os.path.join(self.autodir, |
| 'control.autoserv') |
| |
| |
| def verify_machine(self): |
| binary = os.path.join(self.autodir, 'bin/autotest') |
| try: |
| self.host.run('ls %s > /dev/null 2>&1' % binary) |
| except: |
| raise "Autotest does not appear to be installed" |
| |
| if not self.parallel_flag: |
| tmpdir = os.path.join(self.autodir, 'tmp') |
| download = os.path.join(self.autodir, 'tests/download') |
| self.host.run('umount %s' % tmpdir, ignore_status=True) |
| self.host.run('umount %s' % download, ignore_status=True) |
| |
| def get_full_cmd(self, section): |
| # build up the full command we want to run over the host |
| cmd = [os.path.join(self.autodir, 'bin/autotest_client')] |
| if section > 0: |
| cmd.append('-c') |
| if self.tag: |
| cmd.append('-t %s' % self.tag) |
| if self.host.job.use_external_logging(): |
| cmd.append('-l') |
| cmd.append(self.remote_control_file) |
| return ' '.join(cmd) |
| |
| |
| def get_client_log(self, section): |
| # open up the files we need for our logging |
| client_log_file = os.path.join(self.results_dir, 'debug', |
| 'client.log.%d' % section) |
| return open(client_log_file, 'w', 0) |
| |
| |
| def execute_section(self, section, timeout): |
| print "Executing %s/bin/autotest %s/control phase %d" % \ |
| (self.autodir, self.autodir, |
| section) |
| |
| full_cmd = self.get_full_cmd(section) |
| client_log = self.get_client_log(section) |
| redirector = server_job.client_logger(self.host.job) |
| |
| try: |
| old_resultdir = self.host.job.resultdir |
| self.host.job.resultdir = self.results_dir |
| result = self.host.run(full_cmd, ignore_status=True, |
| timeout=timeout, |
| stdout_tee=client_log, |
| stderr_tee=redirector) |
| finally: |
| redirector.close() |
| self.host.job.resultdir = old_resultdir |
| |
| if result.exit_status == 1: |
| self.host.job.aborted = True |
| if not result.stderr: |
| raise error.AutotestRunError( |
| "execute_section: %s failed to return anything\n" |
| "stdout:%s\n" % (full_cmd, result.stdout)) |
| |
| return redirector.last_line |
| |
| |
| def execute_control(self, timeout=None): |
| section = 0 |
| time_left = None |
| if timeout: |
| end_time = time.time() + timeout |
| time_left = end_time - time.time() |
| while not timeout or time_left > 0: |
| last = self.execute_section(section, time_left) |
| if timeout: |
| time_left = end_time - time.time() |
| if time_left <= 0: |
| break |
| section += 1 |
| if re.match(r'^END .*\t----\t----\t.*$', last): |
| print "Client complete" |
| return |
| elif re.match('^\t*GOOD\t----\treboot\.start.*$', last): |
| print "Client is rebooting" |
| print "Waiting for client to halt" |
| if not self.host.wait_down(HALT_TIME): |
| raise error.AutotestRunError("%s \ |
| failed to shutdown after %ds" % |
| (self.host.hostname, |
| HALT_TIME)) |
| print "Client down, waiting for restart" |
| if not self.host.wait_up(BOOT_TIME): |
| # since reboot failed |
| # hardreset the machine once if possible |
| # before failing this control file |
| print "Hardresetting %s" % ( |
| self.host.hostname,) |
| try: |
| self.host.hardreset(wait=False) |
| except error.AutoservUnsupportedError: |
| print "Hardreset unsupported on %s" % ( |
| self.host.hostname,) |
| raise error.AutotestRunError("%s failed" |
| " to boot after %ds" % ( |
| self.host.hostname, |
| BOOT_TIME,)) |
| self.host.reboot_followup() |
| continue |
| self.host.job.record("ABORT", None, None, |
| "Autotest client terminated " + |
| "unexpectedly") |
| # give the client machine a chance to recover from |
| # possible crash |
| self.host.wait_up(CRASH_RECOVERY_TIME) |
| raise error.AutotestRunError("Aborting - unexpected " |
| "final status message " |
| "from client: %s\n" |
| % last) |
| |
| # should only get here if we timed out |
| assert timeout |
| raise error.AutotestTimeoutError() |
| |
| |
| def _get_autodir(host): |
| dir = host.get_autodir() |
| if dir: |
| return dir |
| try: |
| # There's no clean way to do this. readlink may not exist |
| cmd = "python -c 'import os,sys; print os.readlink(sys.argv[1])' /etc/autotest.conf 2> /dev/null" |
| dir = os.path.dirname(host.run(cmd).stdout) |
| if dir: |
| return dir |
| except error.AutoservRunError: |
| pass |
| for path in ['/usr/local/autotest', '/home/autotest']: |
| try: |
| host.run('ls %s > /dev/null 2>&1' % \ |
| os.path.join(path, 'bin/autotest')) |
| return path |
| except error.AutoservRunError: |
| pass |
| raise error.AutotestRunError("Cannot figure out autotest directory") |
| |
| |
| # site_autotest.py may be non-existant or empty, make sure that an appropriate |
| # SiteAutotest class is created nevertheless |
| try: |
| from site_autotest import SiteAutotest |
| except ImportError: |
| class SiteAutotest(BaseAutotest): |
| pass |
| |
| |
| class Autotest(SiteAutotest): |
| pass |
