[autotest] Start filing bugs for pool health Currently we don't track and manage a failure to run bvts because of dead duts too well. It usually involves multiple sheriffs and quite a bit of confusion, not to mention multiple bugs across 2 trackers, before the right people are notified. This cl files a bug from run_suite when a suite fails because of insufficient DUTs, and notifies all interested parties. TEST=Filed bugs. BUG=chromium:419573,chromium:423847 Change-Id: I4749d6abfb147e058a491d86d707a651246b5ea2 Reviewed-on: https://chromium-review.googlesource.com/224555 Tested-by: Prashanth B <[email protected]> Reviewed-by: Dan Shi <[email protected]> Commit-Queue: Prashanth B <[email protected]>
diff --git a/site_utils/diagnosis_utils.py b/site_utils/diagnosis_utils.py index 95e5e6b..b55a4b2 100644 --- a/site_utils/diagnosis_utils.py +++ b/site_utils/diagnosis_utils.py
@@ -11,6 +11,7 @@ import common from autotest_lib.server import utils +from autotest_lib.server.cros.dynamic_suite import reporting from autotest_lib.server.cros.dynamic_suite import reporting_utils @@ -210,16 +211,26 @@ # setup_django_environment can't be imported now as paygen server does # not have django package. bad_statuses = ('Repair Failed', 'Repairing', 'Verifying') - available_hosts = [host for host in hosts - if not host.status in bad_statuses and - not host.locked] + unusable_hosts = [] + available_hosts = [] + for host in hosts: + if host.status in bad_statuses or host.locked: + unusable_hosts.append(host.hostname) + else: + available_hosts.append(host) logging.debug('%d of %d DUTs are available for board %s pool %s.', len(available_hosts), len(hosts), board, pool) if len(available_hosts) < minimum_duts: + bug_id = '' + if unusable_hosts: + pool_health_bug = reporting.PoolHealthBug( + pool, board, unusable_hosts) + bug_id = reporting.Reporter().report(pool_health_bug)[0] raise NotEnoughDutsError( 'Number of available DUTs for board %s pool %s is %d, which' - ' is less than the minimum value %d.' % - (board, pool, len(available_hosts), minimum_duts)) + ' is less than the minimum value %d. ' + 'Filed https://crbug.com/%s' % + (board, pool, len(available_hosts), minimum_duts, bug_id)) def diagnose_job(self, job_id, instance_server):