[autotest] Add support for scheduler to honor require_ssp attribute in control file
This CL adds changes to pipe require_ssp attribute in control file to autoserv
command. Following are the work flow:
1. The control file parser stores require_ssp attribute value in afe_jobs table.
2. QueueTask compiles command line list, --require-ssp option will be added to
the command line list if following conditions are met:
a. AUTOSERV/enable_ssp_container in global config is True
b. The test is a server-side test
c. require_ssp for the job entry is None or True.
3. When agent_task tries to call run method to run the command, it will check if
there is any drone supporting server-side packaging first. If no drone is found,
the agent task will will run the command in a drone without using server-side
packaging. A warning will be posted in the autoserv log.
4. If a drone without SSP supported is assigned to a test requires SSP, the test
will be run without ssp.
BUG=chromium:453624
TEST=unittest, local test:
set AUTOSERV/enable_ssp_container to True in shadow config;
Create a job for dummy_PassServer in AFE, check require SSP, confirm the job
succeeds but with a warning in the autoserv log.
Create a job for dummy_PassServer_nossp in AFE, uncheck require SSP, confirm
the job passes without warning in the autoserv log.
set AUTOSERV/enable_ssp_container to False in shadow config, restart scheduler.
Create a job for dummy_PassServer in AFE, check require SSP, confirm the job
succeeds without warning in the autoserv log.
also run test_that in local chroot to make sure test_that is not affected.
DEPLOY=apache,scheduler, db migrate must be done before push this CL to prod.
Change-Id: I02f3d137186676ae570e8380d975a1bcd9ffbb94
Reviewed-on: https://chromium-review.googlesource.com/249841
Reviewed-by: Dan Shi <[email protected]>
Commit-Queue: Dan Shi <[email protected]>
Trybot-Ready: Dan Shi <[email protected]>
Tested-by: Dan Shi <[email protected]>
diff --git a/cli/job.py b/cli/job.py
index ddf4e68..6277f38 100644
--- a/cli/job.py
+++ b/cli/job.py
@@ -364,7 +364,7 @@
[--one-time-hosts <hosts>] [--email <email>]
[--dependencies <labels this job is dependent on>]
[--atomic_group <atomic group name>] [--parse-failed-repair <option>]
- [--image <http://path/to/image>]
+ [--image <http://path/to/image>] [--require-ssp]
job_name
Creating a job is rather different from the other create operations,
@@ -430,6 +430,9 @@
self.parser.add_option('-i', '--image',
help='OS image to install before running the '
'test.')
+ self.parser.add_option('--require-ssp',
+ help='Require server-side packaging',
+ default=False, action='store_true')
@staticmethod
@@ -520,6 +523,8 @@
else:
self.data['control_type'] = control_data.CONTROL_TYPE_NAMES.CLIENT
+ self.data['require_ssp'] = options.require_ssp
+
return options, leftover
diff --git a/cli/job_unittest.py b/cli/job_unittest.py
index 34ef429..bac0ceb 100755
--- a/cli/job_unittest.py
+++ b/cli/job_unittest.py
@@ -735,7 +735,8 @@
data = {'priority': 'Medium', 'control_file': ctrl_file, 'hosts': ['host0'],
'name': 'test_job0', 'control_type': CLIENT, 'email_list': '',
- 'meta_hosts': [], 'synch_count': 1, 'dependencies': []}
+ 'meta_hosts': [], 'synch_count': 1, 'dependencies': [],
+ 'require_ssp': False}
def test_execute_create_job(self):
diff --git a/client/common_lib/control_data.py b/client/common_lib/control_data.py
index 943d54d4..4eaae61 100644
--- a/client/common_lib/control_data.py
+++ b/client/common_lib/control_data.py
@@ -56,6 +56,11 @@
self.test_class = ''
self.retries = 0
self.job_retries = 0
+ # Default to require server-side package. Unless require_ssp is
+ # explicitly set to False, server-side package will be used for the
+ # job. This can be overridden by global config
+ # AUTOSERV/enable_ssp_container
+ self.require_ssp = None
diff = REQUIRED_VARS - set(vars)
if diff:
@@ -188,6 +193,7 @@
def set_retries(self, val):
self._set_int('retries', val)
+
def set_job_retries(self, val):
self._set_int('job_retries', val)
@@ -197,6 +203,10 @@
setattr(self, 'bug_template', val)
+ def set_require_ssp(self, val):
+ self._set_bool('require_ssp', val)
+
+
def _extract_const(expr):
assert(expr.__class__ == compiler.ast.Const)
assert(expr.value.__class__ in (str, int, float, unicode))
diff --git a/client/common_lib/control_data_unittest.py b/client/common_lib/control_data_unittest.py
index f4fd6d4..50664da 100755
--- a/client/common_lib/control_data_unittest.py
+++ b/client/common_lib/control_data_unittest.py
@@ -24,6 +24,7 @@
TEST_CATEGORY='Stress'
TEST_TYPE='client'
RETRIES = 5
+REQUIRE_SSP = False
"""
@@ -52,6 +53,7 @@
self.assertEquals(cd.test_category, "stress")
self.assertEquals(cd.test_type, "client")
self.assertEquals(cd.retries, 5)
+ self.assertEquals(cd.require_ssp, False)
class ParseControlFileBugTemplate(unittest.TestCase):
diff --git a/frontend/afe/rpc_interface.py b/frontend/afe/rpc_interface.py
index 193389d..232af52 100644
--- a/frontend/afe/rpc_interface.py
+++ b/frontend/afe/rpc_interface.py
@@ -636,7 +636,8 @@
run_verify=False, email_list='', dependencies=(),
reboot_before=None, reboot_after=None,
parse_failed_repair=None, hostless=False,
- keyvals=None, drone_set=None, run_reset=True):
+ keyvals=None, drone_set=None, run_reset=True,
+ require_ssq=None):
"""
Creates and enqueues a parameterized job.
@@ -743,8 +744,8 @@
run_verify=False, email_list='', dependencies=(),
reboot_before=None, reboot_after=None, parse_failed_repair=None,
hostless=False, keyvals=None, drone_set=None, image=None,
- parent_job_id=None, test_retry=0, run_reset=True, args=(),
- **kwargs):
+ parent_job_id=None, test_retry=0, run_reset=True,
+ require_ssp=None, args=(), **kwargs):
"""\
Create and enqueue a job.
@@ -780,6 +781,12 @@
@param test_retry Number of times to retry test if the test did not
complete successfully. (optional, default: 0)
@param run_reset Should the host be reset before running the test?
+ @param require_ssp Set to True to require server-side packaging to run the
+ test. If it's set to None, drone will still try to run
+ the server side with server-side packaging. If the
+ autotest-server package doesn't exist for the build or
+ image is not set, drone will run the test without server-
+ side packaging. Default is None.
@param args A list of args to be injected into control file.
@param kwargs extra keyword args. NOT USED.
diff --git a/frontend/afe/rpc_utils.py b/frontend/afe/rpc_utils.py
index 6804c32..70dd91a 100644
--- a/frontend/afe/rpc_utils.py
+++ b/frontend/afe/rpc_utils.py
@@ -781,7 +781,8 @@
dependencies=(), reboot_before=None, reboot_after=None,
parse_failed_repair=None, hostless=False, keyvals=None,
drone_set=None, parameterized_job=None,
- parent_job_id=None, test_retry=0, run_reset=True):
+ parent_job_id=None, test_retry=0, run_reset=True,
+ require_ssp=None):
#pylint: disable-msg=C0111
"""
Common code between creating "standard" jobs and creating parameterized jobs
@@ -896,7 +897,8 @@
parameterized_job=parameterized_job,
parent_job_id=parent_job_id,
test_retry=test_retry,
- run_reset=run_reset)
+ run_reset=run_reset,
+ require_ssp=require_ssp)
return create_new_job(owner=owner,
options=options,
host_objects=host_objects,
diff --git a/scheduler/drone_manager.py b/scheduler/drone_manager.py
index 065949d..402a232 100644
--- a/scheduler/drone_manager.py
+++ b/scheduler/drone_manager.py
@@ -234,7 +234,7 @@
def _add_drone(self, hostname):
- logging.info('Adding drone %s' % hostname)
+ logging.info('Adding drone %s', hostname)
drone = drones.get_drone(hostname)
if drone:
self._drones[drone.hostname] = drone
@@ -277,7 +277,6 @@
disabled = config.get_config_value(
section, '%s_disabled' % hostname, default='')
drone.enabled = not bool(disabled)
-
drone.max_processes = config.get_config_value(
section, '%s_max_processes' % hostname, type=int,
default=scheduler_config.config.max_processes_per_drone)
@@ -610,11 +609,25 @@
def _choose_drone_for_execution(self, num_processes, username,
- drone_hostnames_allowed):
+ drone_hostnames_allowed,
+ require_ssp=False):
+ """Choose a drone to execute command.
+
+ @param num_processes: Number of processes needed for execution.
+ @param username: Name of the user to execute the command.
+ @param drone_hostnames_allowed: A list of names of drone allowed.
+ @param require_ssp: Require server-side packaging to execute the,
+ command, default to False.
+
+ @return: A drone object to be used for execution.
+ """
# cycle through drones is order of increasing used capacity until
# we find one that can handle these processes
checked_drones = []
usable_drones = []
+ # Drones do not support server-side packaging, used as backup if no
+ # drone is found to run command requires server-side packaging.
+ no_ssp_drones = []
drone_to_use = None
while self._drone_queue:
drone = heapq.heappop(self._drone_queue).drone
@@ -628,6 +641,11 @@
if not drone_allowed:
logging.debug('Drone %s not allowed: ', drone.hostname)
continue
+ if require_ssp and not drone.support_ssp:
+ logging.debug('Drone %s does not support server-side '
+ 'packaging.', drone.hostname)
+ no_ssp_drones.append(drone)
+ continue
usable_drones.append(drone)
@@ -639,6 +657,7 @@
drone.max_processes)
if not drone_to_use and usable_drones:
+ # Drones are all over loaded, pick the one with least load.
drone_summary = ','.join('%s %s/%s' % (drone.hostname,
drone.active_processes,
drone.max_processes)
@@ -646,6 +665,9 @@
logging.error('No drone has capacity to handle %d processes (%s) '
'for user %s', num_processes, drone_summary, username)
drone_to_use = self._least_loaded_drone(usable_drones)
+ elif not drone_to_use and require_ssp and no_ssp_drones:
+ # No drone supports server-side packaging, choose the least loaded.
+ drone_to_use = self._least_loaded_drone(no_ssp_drones)
# refill _drone_queue
for drone in checked_drones:
@@ -696,15 +718,21 @@
if paired_with_pidfile:
drone = self._get_drone_for_pidfile_id(paired_with_pidfile)
else:
- drone = self._choose_drone_for_execution(num_processes, username,
- drone_hostnames_allowed)
+ require_ssp = '--require-ssp' in command
+ drone = self._choose_drone_for_execution(
+ num_processes, username, drone_hostnames_allowed,
+ require_ssp=require_ssp)
+ # Enable --warn-no-ssp option for autoserv to log a warning and run
+ # the command without using server-side packaging.
+ if require_ssp and not drone.support_ssp:
+ command.append('--warn-no-ssp')
if not drone:
raise DroneManagerError('command failed; no drones available: %s'
% command)
- logging.info("command = %s" % command)
- logging.info('log file = %s:%s' % (drone.hostname, log_file))
+ logging.info("command = %s", command)
+ logging.info('log file = %s:%s', drone.hostname, log_file)
self._write_attached_files(working_directory, drone)
drone.queue_call('execute_command', command, abs_working_directory,
log_file, pidfile_name)
diff --git a/scheduler/drone_manager_unittest.py b/scheduler/drone_manager_unittest.py
index 5a30600..e18f696 100755
--- a/scheduler/drone_manager_unittest.py
+++ b/scheduler/drone_manager_unittest.py
@@ -17,13 +17,15 @@
class MockDrone(drones._AbstractDrone):
def __init__(self, name, active_processes=0, max_processes=10,
- allowed_users=None):
+ allowed_users=None, support_ssp=False):
super(MockDrone, self).__init__()
self.name = name
self.hostname = name
self.active_processes = active_processes
self.max_processes = max_processes
self.allowed_users = allowed_users
+ self._host = 'mock_drone'
+ self._support_ssp = support_ssp
# maps method names list of tuples containing method arguments
self._recorded_calls = {'queue_call': [],
'send_file_to': []}
@@ -102,14 +104,20 @@
def _test_choose_drone_for_execution_helper(self, processes_info_list,
- requested_processes):
+ requested_processes,
+ require_ssp=False):
for index, process_info in enumerate(processes_info_list):
- active_processes, max_processes = process_info
- self.manager._enqueue_drone(MockDrone(index, active_processes,
- max_processes))
+ if len(process_info) == 2:
+ active_processes, max_processes = process_info
+ support_ssp = False
+ else:
+ active_processes, max_processes, support_ssp = process_info
+ self.manager._enqueue_drone(MockDrone(
+ index, active_processes, max_processes, allowed_users=None,
+ support_ssp=support_ssp))
- return self.manager._choose_drone_for_execution(requested_processes,
- self._USERNAME, None)
+ return self.manager._choose_drone_for_execution(
+ requested_processes, self._USERNAME, None, require_ssp)
def test_choose_drone_for_execution(self):
@@ -136,6 +144,19 @@
self.assertEquals(drone.name, 1)
+ def test_choose_drone_for_execution_no_ssp_support(self):
+ drone = self._test_choose_drone_for_execution_helper(
+ [(0, 1), (1, 3)], 1, True)
+ self.assertEquals(drone.name, 0)
+
+
+ def test_choose_drone_for_execution_with_ssp_support(self):
+ self.mock_drone._support_ssp = True
+ drone = self._test_choose_drone_for_execution_helper(
+ [(0, 1), (1, 3, True)], 1, True)
+ self.assertEquals(drone.name, 1)
+
+
def test_user_restrictions(self):
# this drone is restricted to a different user
self.manager._enqueue_drone(MockDrone(1, max_processes=10,
diff --git a/scheduler/drones.py b/scheduler/drones.py
index d3d3562..e62d3f0 100644
--- a/scheduler/drones.py
+++ b/scheduler/drones.py
@@ -42,6 +42,9 @@
self._autotest_install_dir = AUTOTEST_INSTALL_DIR
self._host = None
self.timestamp_remote_calls = timestamp_remote_calls
+ # If drone supports server-side packaging. The property support_ssp will
+ # init self._support_ssp later.
+ self._support_ssp = None
def shutdown(self):
@@ -138,6 +141,24 @@
pass
+ @property
+ def support_ssp(self):
+ """Check if the drone supports server-side packaging with container.
+
+ @return: True if the drone supports server-side packaging with container
+ """
+ if not self._host:
+ raise ValueError('Can not determine if drone supports server-side '
+ 'packaging before host is set.')
+ if self._support_ssp is None:
+ try:
+ self._host.run('which lxc-start')
+ self._support_ssp = True
+ except error.AutotestHostRunError:
+ self._support_ssp = False
+ return self._support_ssp
+
+
SiteDrone = utils.import_site_class(
__file__, 'autotest_lib.scheduler.site_drones',
'_SiteAbstractDrone', _BaseAbstractDrone)
diff --git a/scheduler/monitor_db.py b/scheduler/monitor_db.py
index 59af937..a29023a 100755
--- a/scheduler/monitor_db.py
+++ b/scheduler/monitor_db.py
@@ -19,6 +19,7 @@
import django.db
+from autotest_lib.client.common_lib import control_data
from autotest_lib.client.common_lib import global_config
from autotest_lib.client.common_lib import utils
from autotest_lib.client.common_lib.cros.graphite import autotest_stats
@@ -70,6 +71,9 @@
scheduler_config.CONFIG_SECTION, 'inline_host_acquisition', type=bool,
default=True)
+_enable_ssp_container = global_config.global_config.get_config_value(
+ 'AUTOSERV', 'enable_ssp_container', type=bool,
+ default=True)
def _site_init_monitor_db_dummy():
return {}
@@ -1291,6 +1295,11 @@
def _command_line(self):
invocation = super(QueueTask, self)._command_line()
+ # Check if server-side packaging is needed.
+ if (_enable_ssp_container and
+ self.job.control_type == control_data.CONTROL_TYPE.SERVER and
+ self.job.require_ssp != False):
+ invocation += ['--require-ssp']
return invocation + ['--verify_job_repo_url']
diff --git a/server/autoserv b/server/autoserv
index 24ec567..63748b0 100755
--- a/server/autoserv
+++ b/server/autoserv
@@ -65,6 +65,14 @@
sys.exit(1)
def run_autoserv(pid_file_manager, results, parser):
+ if parser.options.warn_no_ssp:
+ # Post a warning in the log and force to not use server-side packaging.
+ parser.options.require_ssp = False
+ logging.warn('Autoserv is required to run with server-side packaging. '
+ 'However, no drone is found to support server-side '
+ 'packaging. The test will be executed in a drone without '
+ 'server-side packaging supported.')
+
# send stdin to /dev/null
dev_null = os.open(os.devnull, os.O_RDONLY)
os.dup2(dev_null, sys.stdin.fileno())
@@ -188,7 +196,8 @@
% group_name)
kwargs = {'group_name': group_name, 'tag': execution_tag,
- 'disable_sysinfo': parser.options.disable_sysinfo}
+ 'disable_sysinfo': parser.options.disable_sysinfo,
+ 'require_ssp': parser.options.require_ssp}
if control_filename:
kwargs['control_filename'] = control_filename
job = server_job.server_job(control, parser.args[1:], results, label,
diff --git a/server/autoserv_parser.py b/server/autoserv_parser.py
index 5633bfb..3d706a4 100644
--- a/server/autoserv_parser.py
+++ b/server/autoserv_parser.py
@@ -168,6 +168,16 @@
dest="ssh_options", default='',
help=("A string giving command line flags "
"that will be included in ssh commands"))
+ self.parser.add_option("--require-ssp", action="store_true",
+ dest="require_ssp", default=False,
+ help=("Force the autoserv process to run with "
+ "server-side packaging"))
+ self.parser.add_option("--warn-no-ssp", action="store_true",
+ dest="warn_no_ssp", default=False,
+ help=("Post a warning in autoserv log that the "
+ "process runs in a drone without server-"
+ "side packaging support, even though the "
+ "job requires server-side packaging"))
def parse_args(self):
diff --git a/server/cros/dynamic_suite/fakes.py b/server/cros/dynamic_suite/fakes.py
index ff4e1f6..0a36069 100644
--- a/server/cros/dynamic_suite/fakes.py
+++ b/server/cros/dynamic_suite/fakes.py
@@ -25,6 +25,7 @@
self.sync_count = 1
self.job_retries = job_retries
self.bug_template = {}
+ self.require_ssp = None
class FakeJob(object):
diff --git a/server/cros/dynamic_suite/suite.py b/server/cros/dynamic_suite/suite.py
index 4ae7d60..bfc6d79 100644
--- a/server/cros/dynamic_suite/suite.py
+++ b/server/cros/dynamic_suite/suite.py
@@ -669,7 +669,8 @@
parent_job_id=self._suite_job_id,
test_retry=test.retries,
priority=self._priority,
- synch_count=test.sync_count)
+ synch_count=test.sync_count,
+ require_ssp=test.require_ssp)
setattr(test_obj, 'test_name', test.name)
diff --git a/server/cros/dynamic_suite/suite_unittest.py b/server/cros/dynamic_suite/suite_unittest.py
index a130e35..081cbd6 100755
--- a/server/cros/dynamic_suite/suite_unittest.py
+++ b/server/cros/dynamic_suite/suite_unittest.py
@@ -261,7 +261,8 @@
parent_job_id=None,
test_retry=0,
priority=priorities.Priority.DEFAULT,
- synch_count=test.sync_count
+ synch_count=test.sync_count,
+ require_ssp=test.require_ssp
)
if raises:
job_mock.AndRaise(error.NoEligibleHostException())
diff --git a/server/server_job.py b/server/server_job.py
index c573047..2488587 100644
--- a/server/server_job.py
+++ b/server/server_job.py
@@ -153,7 +153,8 @@
ssh_options=host_factory.DEFAULT_SSH_OPTIONS,
test_retry=0, group_name='',
tag='', disable_sysinfo=False,
- control_filename=SERVER_CONTROL_FILENAME):
+ control_filename=SERVER_CONTROL_FILENAME,
+ require_ssp=False):
"""
Create a server side job object.
@@ -181,6 +182,7 @@
tests for a modest shortening of test time. [optional]
@param control_filename: The filename where the server control file
should be written in the results directory.
+ @param require_ssp: Require to use server-side packaging to run the test.
"""
super(base_server_job, self).__init__(resultdir=resultdir,
test_retry=test_retry)
@@ -216,6 +218,7 @@
self.drop_caches_between_iterations = False
self._control_filename = control_filename
self._disable_sysinfo = disable_sysinfo
+ self._require_ssp = require_ssp
self.logging = logging_manager.get_logging_manager(
manage_stdout_and_stderr=True, redirect_fds=True)
@@ -510,6 +513,12 @@
return wrapper
+ def _run_with_ssp(self):
+ """Run the server job with server-side packaging.
+ """
+ raise NotImplementedError('Server-side packaging is not supported yet.')
+
+
def parallel_simple(self, function, machines, log=True, timeout=None,
return_results=False):
"""
@@ -640,7 +649,10 @@
else:
utils.open_write_close(server_control_file, control)
logging.info("Processing control file")
- self._execute_code(server_control_file, namespace)
+ if self._require_ssp:
+ self._run_with_ssp()
+ else:
+ self._execute_code(server_control_file, namespace)
logging.info("Finished processing control file")
# If no device error occured, no need to collect crashinfo.
