[autotest] Add collection of /var/log to the repair task.
Also refactors the function that handles log collection onto ssh_host
since several non-cros hosts use the same control segments.
TEST=Ran special tasks and collected logs.
BUG=chromium:354944
Change-Id: I88ba114c0e42682e69d7ce8dc544d70e61c02840
Reviewed-on: https://chromium-review.googlesource.com/193324
Tested-by: Prashanth B <[email protected]>
Reviewed-by: Fang Deng <[email protected]>
Reviewed-by: Alex Miller <[email protected]>
Commit-Queue: Prashanth B <[email protected]>
diff --git a/server/control_segments/cleanup b/server/control_segments/cleanup
index 0a98374..6e51c63 100644
--- a/server/control_segments/cleanup
+++ b/server/control_segments/cleanup
@@ -1,4 +1,5 @@
import logging
+import os
from autotest_lib.client.common_lib import error
from autotest_lib.site_utils.graphite import stats
@@ -17,24 +18,19 @@
host = hosts.create_host(machine, initialize=False, auto_monitor=False)
timer = stats.Timer('cleanup_time.%s' % host._get_board_from_afe())
timer.start()
+
# Try to save /var/log files. If the dut is not sshable, try to restart
# with servo. This is a temp fix to collect log for test failed with dut
# not returning from reboot.
# TODO(dshi): This temp fix should be removed after crash collect work
# is completed, crbug.com/336985
try:
- try:
- host.ssh_ping()
- except error.AutoservSshPingHostError:
- # Try to restart dut with servo.
- host._servo_repair_power()
- # Copy log
- log_dir = os.path.join(job.resultdir, machine)
- os.makedirs(log_dir)
- host.get_file('/var/log/', log_dir, preserve_symlinks=True)
- except Exception as e:
- logging.error('Collect /var/log failed with error: %s', e)
- raise
+ host.ssh_ping()
+ except error.AutoservSshPingHostError:
+ # Try to restart dut with servo.
+ host._servo_repair_power()
+ local_log_dir = os.path.join(job.resultdir, machine)
+ host.collect_logs('/var/log', local_log_dir, ignore_errors=True)
host.cleanup()
provision.run_special_task_actions(job, host, labels_list,
diff --git a/server/control_segments/repair b/server/control_segments/repair
index 80382e0..ef79284 100644
--- a/server/control_segments/repair
+++ b/server/control_segments/repair
@@ -1,4 +1,5 @@
import logging
+import os
from autotest_lib.client.common_lib import host_protections
from autotest_lib.server.cros import repair_utils
@@ -33,6 +34,8 @@
logging.debug('Repair with labels list %s', labels_list)
provision.run_special_task_actions(job, host, labels_list,
provision.Repair)
+ local_log_dir = os.path.join(job.resultdir, machine)
+ host.collect_logs('/var/log', local_log_dir, ignore_errors=True)
except Exception as e:
repair_utils.flag_problem_test(machine)
job.record('END FAIL', None, 'repair')
diff --git a/server/hosts/abstract_ssh.py b/server/hosts/abstract_ssh.py
index b731b21..a598ee5 100644
--- a/server/hosts/abstract_ssh.py
+++ b/server/hosts/abstract_ssh.py
@@ -699,3 +699,44 @@
# Clear out the file by opening it for writing and then closing.
fh = open(self.known_hosts_file, "w")
fh.close()
+
+
+ def collect_logs(self, remote_src_dir, local_dest_dir, ignore_errors=True):
+ """Copy log directories from a host to a local directory.
+
+ @param remote_src_dir: A destination directory on the host.
+ @param local_dest_dir: A path to a local destination directory.
+ If it doesn't exist it will be created.
+ @param ignore_errors: If True, ignore exceptions.
+
+ @raises OSError: If there were problems creating the local_dest_dir and
+ ignore_errors is False.
+ @raises AutoservRunError, AutotestRunError: If something goes wrong
+ while copying the directories and ignore_errors is False.
+ """
+ locally_created_dest = False
+ if (not os.path.exists(local_dest_dir)
+ or not os.path.isdir(local_dest_dir)):
+ try:
+ os.makedirs(local_dest_dir)
+ locally_created_dest = True
+ except OSError as e:
+ logging.warning('Unable to collect logs from host '
+ '%s: %s', self.hostname, e)
+ if not ignore_errors:
+ raise
+ return
+ try:
+ self.get_file(
+ remote_src_dir, local_dest_dir, preserve_symlinks=True)
+ except (error.AutotestRunError, error.AutoservRunError,
+ error.AutoservSSHTimeout) as e:
+ logging.warning('Collection of %s to local dir %s from host %s '
+ 'failed: %s', remote_src_dir, local_dest_dir,
+ self.hostname, e)
+ if locally_created_dest:
+ shutil.rmtree(local_dest_dir, ignore_errors=ignore_errors)
+ if not ignore_errors:
+ raise
+
+
