[moblab] Run dut ssh test in parallel
The Manage DUTs page becomes slow with many DUTs connected, as
it checks each connected DUT for SSH connectivity in sequence.
This change has the checks run in parallel subprocesses and
provides a good speedup.
Also updated some messaging on UI for DUTs that couldn't be
reached by SSH, to make it clear what the problem is.
BUG=chromium:815275,chromium:814425
TEST=Added unit test for parallel test execution logic, and tested
functionality on local device with 24 connected DUTs.
Change-Id: Id087f02e9a5bf20e06cd536642164c356c747b74
Reviewed-on: https://chromium-review.googlesource.com/935881
Commit-Ready: Matt Mallett <[email protected]>
Tested-by: Matt Mallett <[email protected]>
Reviewed-by: Keith Haddow <[email protected]>
diff --git a/frontend/afe/moblab_rpc_interface.py b/frontend/afe/moblab_rpc_interface.py
index 29ad8b6..d4c1315 100644
--- a/frontend/afe/moblab_rpc_interface.py
+++ b/frontend/afe/moblab_rpc_interface.py
@@ -18,6 +18,8 @@
import StringIO
import subprocess
import time
+import multiprocessing
+import ctypes
from autotest_lib.client.common_lib import error
from autotest_lib.client.common_lib import global_config
@@ -544,13 +546,8 @@
# Make a list of the connected DUT's
leases = _get_dhcp_dut_leases()
- connected_duts = {}
- for ip in leases:
- ssh_connection_ok = _test_dut_ssh_connection(ip)
- connected_duts[ip] = {
- 'mac_address': leases[ip],
- 'ssh_connection_ok': ssh_connection_ok
- }
+
+ connected_duts = _test_all_dut_connections(leases)
# Get a list of the AFE configured DUT's
hosts = list(rpc_utils.get_host_query((), False, True, {}))
@@ -589,6 +586,46 @@
leases[ipaddress] = mac_address_search.group(1)
return leases
+def _test_all_dut_connections(leases):
+ """ Test ssh connection of all connected DUTs in parallel
+
+ @param leases: dict containing key value pairs of ip and mac address
+
+ @return: dict containing {
+ ip: {mac_address:[string], ssh_connection_ok:[boolean]}
+ }
+ """
+ # target function for parallel process
+ def _test_dut(ip, result):
+ result.value = _test_dut_ssh_connection(ip)
+
+ processes = []
+ for ip in leases:
+ # use a shared variable to get the ssh test result from child process
+ ssh_test_result = multiprocessing.Value(ctypes.c_bool)
+ # create a subprocess to test each DUT
+ process = multiprocessing.Process(
+ target=_test_dut, args=(ip, ssh_test_result))
+ process.start()
+
+ processes.append({
+ 'ip': ip,
+ 'ssh_test_result': ssh_test_result,
+ 'process': process
+ })
+
+ connected_duts = {}
+ for process in processes:
+ process['process'].join()
+ ip = process['ip']
+ connected_duts[ip] = {
+ 'mac_address': leases[ip],
+ 'ssh_connection_ok': process['ssh_test_result'].value
+ }
+
+ return connected_duts
+
+
def _test_dut_ssh_connection(ip):
""" Test if a connected dut is accessible via ssh.
The primary use case is to verify that the dut has a test image.
diff --git a/frontend/afe/moblab_rpc_interface_unittest.py b/frontend/afe/moblab_rpc_interface_unittest.py
index f9a05e8..28b62e8 100644
--- a/frontend/afe/moblab_rpc_interface_unittest.py
+++ b/frontend/afe/moblab_rpc_interface_unittest.py
@@ -619,6 +619,89 @@
# test sorting
self.assertEquals(output[0]['model'], 'bruce')
+ def testAllDutConnections(self):
+ leases = {
+ '192.168.0.20': '3c:52:82:5f:15:20',
+ '192.168.0.30': '3c:52:82:5f:15:21'
+ }
+
+ # stub out all of the multiprocessing
+ mock_value = self.mox.CreateMockAnything()
+ mock_value.value = True
+ mock_process = self.mox.CreateMockAnything()
+
+ for key in leases:
+ mock_process.start()
+ for key in leases:
+ mock_process.join()
+
+ self.mox.StubOutWithMock(
+ moblab_rpc_interface, 'multiprocessing')
+
+ for key in leases:
+ moblab_rpc_interface.multiprocessing.Value(
+ mox.IgnoreArg()).AndReturn(mock_value)
+ moblab_rpc_interface.multiprocessing.Process(
+ target=mox.IgnoreArg(), args=mox.IgnoreArg()).AndReturn(
+ mock_process)
+
+ self.mox.ReplayAll()
+
+ expected = {
+ '192.168.0.20': {
+ 'mac_address': '3c:52:82:5f:15:20',
+ 'ssh_connection_ok': True
+ },
+ '192.168.0.30': {
+ 'mac_address': '3c:52:82:5f:15:21',
+ 'ssh_connection_ok': True
+ }
+ }
+
+ connected_duts = moblab_rpc_interface._test_all_dut_connections(leases)
+ self.assertDictEqual(expected, connected_duts)
+
+ def testAllDutConnectionsFailure(self):
+ leases = {
+ '192.168.0.20': '3c:52:82:5f:15:20',
+ '192.168.0.30': '3c:52:82:5f:15:21'
+ }
+
+ # stub out all of the multiprocessing
+ mock_value = self.mox.CreateMockAnything()
+ mock_value.value = False
+ mock_process = self.mox.CreateMockAnything()
+
+ for key in leases:
+ mock_process.start()
+ for key in leases:
+ mock_process.join()
+
+ self.mox.StubOutWithMock(
+ moblab_rpc_interface, 'multiprocessing')
+
+ for key in leases:
+ moblab_rpc_interface.multiprocessing.Value(
+ mox.IgnoreArg()).AndReturn(mock_value)
+ moblab_rpc_interface.multiprocessing.Process(
+ target=mox.IgnoreArg(), args=mox.IgnoreArg()).AndReturn(
+ mock_process)
+
+ self.mox.ReplayAll()
+
+ expected = {
+ '192.168.0.20': {
+ 'mac_address': '3c:52:82:5f:15:20',
+ 'ssh_connection_ok': False
+ },
+ '192.168.0.30': {
+ 'mac_address': '3c:52:82:5f:15:21',
+ 'ssh_connection_ok': False
+ }
+ }
+
+ connected_duts = moblab_rpc_interface._test_all_dut_connections(leases)
+ self.assertDictEqual(expected, connected_duts)
def testDutSshConnection(self):
good_ip = '192.168.0.20'
diff --git a/frontend/client/src/autotest/moblab/DutManagementView.java b/frontend/client/src/autotest/moblab/DutManagementView.java
index b72b1ab..e324380 100644
--- a/frontend/client/src/autotest/moblab/DutManagementView.java
+++ b/frontend/client/src/autotest/moblab/DutManagementView.java
@@ -225,7 +225,8 @@
boolean sshOk =
info.getConnectedIpsToSshConnection().get(dutIpAddress);
labelString = sshOk ? "DUT Not Configured in Autotest" :
- "Unable to connect to DUT over SSH, is this a test image?";
+ "Unable to connect to DUT over SSH, check device is powered " +
+ "on, connected and has a test image on it.";
}
addRow(row, dutIpAddress, info.getConnectedIpsToMacAddress().get(dutIpAddress), labelString);
row++;
