[moblab] Run dut ssh test in parallel
The Manage DUTs page becomes slow with many DUTs connected, as
it checks each connected DUT for SSH connectivity in sequence.
This change has the checks run in parallel subprocesses and
provides a good speedup.
Also updated some messaging on UI for DUTs that couldn't be
reached by SSH, to make it clear what the problem is.
BUG=chromium:815275,chromium:814425
TEST=Added unit test for parallel test execution logic, and tested
functionality on local device with 24 connected DUTs.
Change-Id: Id087f02e9a5bf20e06cd536642164c356c747b74
Reviewed-on: https://chromium-review.googlesource.com/935881
Commit-Ready: Matt Mallett <[email protected]>
Tested-by: Matt Mallett <[email protected]>
Reviewed-by: Keith Haddow <[email protected]>
diff --git a/frontend/afe/moblab_rpc_interface.py b/frontend/afe/moblab_rpc_interface.py
index 29ad8b6..d4c1315 100644
--- a/frontend/afe/moblab_rpc_interface.py
+++ b/frontend/afe/moblab_rpc_interface.py
@@ -18,6 +18,8 @@
import StringIO
import subprocess
import time
+import multiprocessing
+import ctypes
from autotest_lib.client.common_lib import error
from autotest_lib.client.common_lib import global_config
@@ -544,13 +546,8 @@
# Make a list of the connected DUT's
leases = _get_dhcp_dut_leases()
- connected_duts = {}
- for ip in leases:
- ssh_connection_ok = _test_dut_ssh_connection(ip)
- connected_duts[ip] = {
- 'mac_address': leases[ip],
- 'ssh_connection_ok': ssh_connection_ok
- }
+
+ connected_duts = _test_all_dut_connections(leases)
# Get a list of the AFE configured DUT's
hosts = list(rpc_utils.get_host_query((), False, True, {}))
@@ -589,6 +586,46 @@
leases[ipaddress] = mac_address_search.group(1)
return leases
+def _test_all_dut_connections(leases):
+ """ Test ssh connection of all connected DUTs in parallel
+
+ @param leases: dict containing key value pairs of ip and mac address
+
+ @return: dict containing {
+ ip: {mac_address:[string], ssh_connection_ok:[boolean]}
+ }
+ """
+ # target function for parallel process
+ def _test_dut(ip, result):
+ result.value = _test_dut_ssh_connection(ip)
+
+ processes = []
+ for ip in leases:
+ # use a shared variable to get the ssh test result from child process
+ ssh_test_result = multiprocessing.Value(ctypes.c_bool)
+ # create a subprocess to test each DUT
+ process = multiprocessing.Process(
+ target=_test_dut, args=(ip, ssh_test_result))
+ process.start()
+
+ processes.append({
+ 'ip': ip,
+ 'ssh_test_result': ssh_test_result,
+ 'process': process
+ })
+
+ connected_duts = {}
+ for process in processes:
+ process['process'].join()
+ ip = process['ip']
+ connected_duts[ip] = {
+ 'mac_address': leases[ip],
+ 'ssh_connection_ok': process['ssh_test_result'].value
+ }
+
+ return connected_duts
+
+
def _test_dut_ssh_connection(ip):
""" Test if a connected dut is accessible via ssh.
The primary use case is to verify that the dut has a test image.
