Adds support for using client-side profilers from the server side.
This makes a few significant infrastructure changes in order the
support this:
- it modifies autotest.py to allow "background" clients to be
launched, where the server simply kicks off a client as a
background process without bothering to monitor it
- it adds a profiler_test test which is not a "real" test but is
instead used to allow the server to start & stop the profilers
in a reasonably controlled way
Currently, this still lacks any support for dealing with reboots; if
a test reboots the remote machine then the profilers will not be
restarted and the logs collected will only go as far as the first
reboot.
Risk: Medium
Visibility: You can use profilers from the server side (via
job.profilers.add and job.profilers.delete)
Signed-off-by: John Admanski <[email protected]>
git-svn-id: http://test.kernel.org/svn/autotest/trunk@2514 592f7852-d20e-0410-864c-8624ca9c26a4
diff --git a/server/autotest.py b/server/autotest.py
index 4629a73..c339d21 100644
--- a/server/autotest.py
+++ b/server/autotest.py
@@ -58,8 +58,8 @@
@log.record
- def install(self, host=None):
- self._install(host=host)
+ def install(self, host=None, autodir=None):
+ self._install(host=host, autodir=autodir)
def install_base(self, host=None, autodir=None):
@@ -181,7 +181,7 @@
def run(self, control_file, results_dir = '.', host = None,
- timeout=None, tag=None, parallel_flag=False):
+ timeout=None, tag=None, parallel_flag=False, background=False):
"""
Run an autotest job on the remote machine.
@@ -194,6 +194,10 @@
tag: tag name for the client side instance of autotest
parallel_flag: flag set when multiple jobs are run at the
same time
+ background: indicates that the client should be launched as
+ a background job; the code calling run will be
+ responsible for monitoring the client and
+ collecting the results
Raises:
AutotestRunError: if there is a problem executing
the control file
@@ -204,7 +208,7 @@
if tag:
results_dir = os.path.join(results_dir, tag)
- atrun = _Run(host, results_dir, tag, parallel_flag)
+ atrun = _Run(host, results_dir, tag, parallel_flag, background)
self._do_run(control_file, results_dir, host, atrun, timeout)
@@ -270,9 +274,10 @@
try:
atrun.execute_control(timeout=timeout)
finally:
- collector = log_collector(host, atrun.tag, results_dir)
- collector.collect_client_job_results()
- self._process_client_state_file(host, atrun, results_dir)
+ if not atrun.background:
+ collector = log_collector(host, atrun.tag, results_dir)
+ collector.collect_client_job_results()
+ self._process_client_state_file(host, atrun, results_dir)
def _create_state_file(self, job, state_dict):
@@ -349,12 +354,13 @@
It is not intended to be used directly, rather control files
should be run using the run method in Autotest.
"""
- def __init__(self, host, results_dir, tag, parallel_flag):
+ def __init__(self, host, results_dir, tag, parallel_flag, background):
self.host = host
self.results_dir = results_dir
self.env = host.env
self.tag = tag
self.parallel_flag = parallel_flag
+ self.background = background
self.autodir = _get_autodir(self.host)
control = os.path.join(self.autodir, 'control')
if tag:
@@ -379,8 +385,9 @@
def get_full_cmd(self, section):
# build up the full command we want to run over the host
- cmd = [os.path.join(self.autodir, 'bin/autotest_client'),
- '-H autoserv']
+ cmd = [os.path.join(self.autodir, 'bin/autotest_client')]
+ if not self.background:
+ cmd.append('-H autoserv')
if section > 0:
cmd.append('-c')
if self.tag:
@@ -388,6 +395,8 @@
if self.host.job.use_external_logging():
cmd.append('-l')
cmd.append(self.remote_control_file)
+ if self.background:
+ cmd = ['nohup'] + cmd + ['>/dev/null 2>/dev/null &']
return ' '.join(cmd)
@@ -417,7 +426,7 @@
if result.exit_status == 1:
raise error.AutotestRunError("client job was aborted")
- if not result.stderr:
+ if not self.background and not result.stderr:
raise error.AutotestRunError(
"execute_section: %s failed to return anything\n"
"stdout:%s\n" % (full_cmd, result.stdout))
@@ -464,6 +473,8 @@
section_timeout = None
last = self.execute_section(section, section_timeout,
logger)
+ if self.background:
+ return
section += 1
if re.match(r'^END .*\t----\t----\t.*$', last):
print "Client complete"
diff --git a/server/autotest_unittest.py b/server/autotest_unittest.py
index 619106d..14e5a14 100644
--- a/server/autotest_unittest.py
+++ b/server/autotest_unittest.py
@@ -132,7 +132,7 @@
self.base_autotest.install.expect_call(self.host)
self.host.wait_up.expect_call(timeout=30)
os.path.abspath.expect_call('.').and_return('.')
- run_obj = autotest._Run.expect_new(self.host, '.', None, False)
+ run_obj = autotest._Run.expect_new(self.host, '.', None, False, False)
tag = None
run_obj.manual_control_file = os.path.join('autodir',
'control.%s' % tag)
@@ -142,6 +142,7 @@
run_obj.autodir = 'autodir'
run_obj.verify_machine.expect_call()
run_obj.verify_machine.expect_call()
+ run_obj.background = False
debug = os.path.join('.', 'debug')
os.makedirs.expect_call(debug)
delete_file_list = [run_obj.remote_control_file,
diff --git a/server/profiler.py b/server/profiler.py
index a23c655..63b2e4d 100644
--- a/server/profiler.py
+++ b/server/profiler.py
@@ -1,7 +1,17 @@
-import itertools
+import os, itertools
from autotest_lib.server import autotest
+PROFILER_TMPDIR = "/tmp/profilers"
+
+
+# control file template for running a job that uses profiler 'name'
+run_profiler_control = """\
+job.profilers.add(%s)
+job.run_test("profiler_test")
+job.profilers.delete(%r)
+"""
+
def get_unpassable_types(arg):
""" Given an argument, returns a set of types contained in arg that are
@@ -36,6 +46,13 @@
raise TypeError(msg)
+def encode_args(profiler, args, dargs):
+ parts = [repr(profiler)]
+ parts += [repr(arg) for arg in dargs]
+ parts += ["%s=%r" % darg for darg in dargs.iteritems()]
+ return ", ".join(parts)
+
+
class profiler_proxy(object):
""" This is a server-side class that acts as a proxy to a real client-side
profiler class."""
@@ -48,11 +65,13 @@
def _install(self):
""" Install autotest on any current job hosts. """
- current_job_hosts = self.job.hosts
+ current_job_hosts = set(host for host in self.job.hosts
+ if not host.get_autodir() or
+ host.get_autodir().startswith(PROFILER_TMPDIR))
current_profiler_hosts = set(self.installed_hosts.keys())
# install autotest on any new hosts in job.hosts
for host in current_job_hosts - current_profiler_hosts:
- tmp_dir = host.get_tmp_dir(parent="/tmp/profilers")
+ tmp_dir = host.get_tmp_dir(parent=PROFILER_TMPDIR)
at = autotest.Autotest(host)
at.install(autodir=tmp_dir)
self.installed_hosts[host] = at
@@ -61,24 +80,50 @@
del self.installed_hosts[host]
- def setup(self, *args, **dargs):
- validate_args(args)
- validate_args(dargs)
- self._install()
-
-
def initialize(self, *args, **dargs):
validate_args(args)
validate_args(dargs)
+ self.args, self.dargs = args, dargs
+
+
+ def setup(self, *args, **dargs):
+ assert self.args == args and self.dargs == dargs
+ # the actual setup happens lazily at start()
+
+
+ def _signal_clients(self, command):
+ """ Signal to each client that it should execute profilers.command
+ by writing a byte into AUTODIR/profilers.command. """
+ for host in self.installed_hosts.iterkeys():
+ autodir = host.get_autodir()
+ path = os.path.join(autodir, "profiler.%s" % command)
+ host.run("echo A > %s" % path)
def start(self, test):
- pass
+ self._install()
+ encoded_args = encode_args(self.name, self.args, self.dargs)
+ control_script = run_profiler_control % (encoded_args, self.name)
+ for at in self.installed_hosts.itervalues():
+ at.run(control_script, background=True)
+ self._signal_clients("start")
def stop(self, test):
- pass
+ self._signal_clients("stop")
def report(self, test):
- pass
+ self._signal_clients("report")
+ # pull back all the results
+ for host in self.installed_hosts.iterkeys():
+ results_dir = os.path.join(host.get_autodir(), "results",
+ "default", "profiler_test",
+ "profiling") + "/"
+ local_dir = os.path.join(test.profdir, host.hostname)
+ if not os.path.exists(local_dir):
+ os.makedirs(local_dir)
+ try:
+ host.get_file(results_dir, local_dir)
+ except error.AutoservRunError:
+ pass # no files to pull back, nothing we can do
