Autotest: Use SSH For Communication to Moblab
Create an ssh tunnel for autoserv to communicate to moblab. MoblabHost calls
rpc_server_tracker to create a safe ssh connection to moblab when it is
created or moblab is rebooted.
Related Change:
1. Refactoring rpc_server_tracker to seperate tunnel creating process
|_setup_tracker_for_rpc| and URL/host construction functions,
|_setup_rpc| or |tunnel_connect|.
2. Add a |enable_ssh_tunnel_for_moblab| to enable/disable ssh tunnel for
moblab.
BUG=chromium:593965
TEST=Ran verify&repair, moblab_RunSuite:dummyServer, on local moblab to
verify ssh tunnel for moblab. Ran network_WiFi_SimpleConnect.wifi_check11g
to make sure that rpc_server_tracker still works.
Change-Id: I0d35209c3ba7b27cca00bbdff1a714ed6d8d40d3
Reviewed-on: https://chromium-review.googlesource.com/331792
Commit-Ready: Xixuan Wu <[email protected]>
Tested-by: Xixuan Wu <[email protected]>
Reviewed-by: Richard Barnette <[email protected]>
diff --git a/global_config.ini b/global_config.ini
index c4a142e..ec75011 100644
--- a/global_config.ini
+++ b/global_config.ini
@@ -331,6 +331,9 @@
# Flags to enable/disable SSH connection for devserver.
enable_ssh_connection_for_devserver: False
+# Flags to enable/disable SSH tunnel connection for moblab host.
+enable_ssh_tunnel_for_moblab: False
+
[BUG_REPORTING]
gs_domain: https://storage.cloud.google.com/
chromeos_image_archive: chromeos-image-archive/
diff --git a/server/cros/dynamic_suite/frontend_wrappers.py b/server/cros/dynamic_suite/frontend_wrappers.py
index 491a162..807042e 100644
--- a/server/cros/dynamic_suite/frontend_wrappers.py
+++ b/server/cros/dynamic_suite/frontend_wrappers.py
@@ -56,6 +56,14 @@
super(RetryingAFE, self).__init__(**dargs)
+ def set_timeout(self, timeout_min):
+ """Set timeout minutes for the AFE server.
+
+ @param timeout_min: The timeout minutes for AFE server.
+ """
+ self.timeout_min = timeout_min
+
+
def run(self, call, **dargs):
if retry_util is None:
raise ImportError('Unable to import chromite. Please consider to '
diff --git a/server/hosts/moblab_host.py b/server/hosts/moblab_host.py
index 7f0faff..12cb311 100644
--- a/server/hosts/moblab_host.py
+++ b/server/hosts/moblab_host.py
@@ -16,6 +16,10 @@
AUTOTEST_INSTALL_DIR = global_config.global_config.get_config_value(
'SCHEDULER', 'drone_installation_directory')
+
+ENABLE_SSH_TUNNEL_FOR_MOBLAB = global_config.global_config.get_config_value(
+ 'CROS', 'enable_ssh_tunnel_for_moblab', type=bool, default=False)
+
#'/usr/local/autotest'
SHADOW_CONFIG_PATH = '%s/shadow_config.ini' % AUTOTEST_INSTALL_DIR
ATEST_PATH = '%s/cli/atest' % AUTOTEST_INSTALL_DIR
@@ -36,12 +40,34 @@
DUT_VERIFY_SLEEP_SECS = 5
DUT_VERIFY_TIMEOUT = 15 * 60
MOBLAB_TMP_DIR = '/mnt/moblab/tmp'
+MOBLAB_PORT = 80
class MoblabHost(cros_host.CrosHost):
"""Moblab specific host class."""
+ def _initialize_frontend_rpcs(self, timeout_min):
+ """Initialize frontends for AFE and TKO for a moblab host.
+
+ AFE and TKO are initialized differently based on |_use_tunnel|,
+ which indicates that whether to use ssh tunnel to connect to moblab.
+
+ @param timeout_min: The timeout minuties for AFE services.
+ """
+ if self._use_tunnel:
+ self.web_address = self.rpc_server_tracker.tunnel_connect(
+ MOBLAB_PORT)
+ # Pass timeout_min to self.afe
+ self.afe = frontend_wrappers.RetryingAFE(timeout_min=timeout_min,
+ user='moblab',
+ server=self.web_address)
+ # Use default timeout_min of MoblabHost for self.tko
+ self.tko = frontend_wrappers.RetryingTKO(timeout_min=self.timeout_min,
+ user='moblab',
+ server=self.web_address)
+
+
def _initialize(self, *args, **dargs):
super(MoblabHost, self)._initialize(*args, **dargs)
# Clear the Moblab Image Storage so that staging an image is properly
@@ -50,13 +76,10 @@
self.run('rm -rf %s/*' % MOBLAB_IMAGE_STORAGE)
self._dhcpd_leasefile = None
self.web_address = dargs.get('web_address', self.hostname)
- timeout_min = dargs.get('rpc_timeout_min', 1)
- self.afe = frontend_wrappers.RetryingAFE(timeout_min=timeout_min,
- user='moblab',
- server=self.web_address)
- self.tko = frontend_wrappers.RetryingTKO(timeout_min=timeout_min,
- user='moblab',
- server=self.web_address)
+ self._use_tunnel = (ENABLE_SSH_TUNNEL_FOR_MOBLAB and
+ self.web_address == self.hostname)
+ self.timeout_min = dargs.get('rpc_timeout_min', 1)
+ self._initialize_frontend_rpcs(self.timeout_min)
@staticmethod
@@ -138,12 +161,14 @@
@raises urllib2.HTTPError if AFE does not respond within the timeout.
"""
- # Use a new AFE object with a longer timeout to wait for the AFE to
- # load.
- afe = frontend_wrappers.RetryingAFE(timeout_min=timeout_min,
- server=self.hostname)
+ # Use moblabhost's own AFE object with a longer timeout to wait for the
+ # AFE to load. Also re-create the ssh tunnel for connections to moblab.
+ # Set the timeout_min to be longer than self.timeout_min for rebooting.
+ self._initialize_frontend_rpcs(timeout_min)
# Verify the AFE can handle a simple request.
- afe.get_hosts()
+ self.afe.get_hosts()
+ # Reset the timeout_min after rebooting checks for afe services.
+ self.afe.set_timeout(self.timeout_min)
def _wake_devices(self):
diff --git a/server/hosts/rpc_server_tracker.py b/server/hosts/rpc_server_tracker.py
index f771576..859ef7c 100644
--- a/server/hosts/rpc_server_tracker.py
+++ b/server/hosts/rpc_server_tracker.py
@@ -28,10 +28,10 @@
"""
_RPC_PROXY_URL_FORMAT = 'http://localhost:%d'
+ _RPC_HOST_ADDRESS_FORMAT = 'localhost:%d'
_RPC_SHUTDOWN_POLLING_PERIOD_SECONDS = 2
_RPC_SHUTDOWN_TIMEOUT_SECONDS = 10
-
def __init__(self, host):
"""
@param port: The host object associated with this instance of
@@ -41,8 +41,8 @@
self._rpc_proxy_map = {}
- def _setup_rpc(self, port, command_name, remote_pid=None):
- """Sets up a tunnel process and performs rpc connection book keeping.
+ def _setup_port(self, port, command_name, remote_pid=None):
+ """Sets up a tunnel process and register it to rpc_server_tracker.
Chrome OS on the target closes down most external ports for security.
We could open the port, but doing that would conflict with security
@@ -72,15 +72,43 @@
@param port: The remote forwarding port.
@param command_name: The name of the remote process, to terminate
- using pkill.
+ using pkill.
+ @param remote_pid: The PID of the remote background process
+ as a string.
- @return A url that we can use to initiate the rpc connection.
+ @return the local port which is used for port forwarding on the ssh
+ client.
"""
self.disconnect(port)
local_port = utils.get_unused_port()
tunnel_proc = self._host.create_ssh_tunnel(port, local_port)
self._rpc_proxy_map[port] = (command_name, tunnel_proc, remote_pid)
- return self._RPC_PROXY_URL_FORMAT % local_port
+ return local_port
+
+
+ def _setup_rpc(self, port, command_name, remote_pid=None):
+ """Construct a URL for an rpc connection using ssh tunnel.
+
+ @param port: The remote forwarding port.
+ @param command_name: The name of the remote process, to terminate
+ using pkill.
+ @param remote_pid: The PID of the remote background process
+ as a string.
+
+ @return a url that we can use to initiate the rpc connection.
+ """
+ return self._RPC_PROXY_URL_FORMAT % self._setup_port(
+ port, command_name, remote_pid=remote_pid)
+
+
+ def tunnel_connect(self, port):
+ """Construct a host address using ssh tunnel.
+
+ @param port: The remote forwarding port.
+
+ @return a host address using ssh tunnel.
+ """
+ return self._RPC_HOST_ADDRESS_FORMAT % self._setup_port(port, None)
def xmlrpc_connect(self, command, port, command_name=None,
