[autotest] Start filing bugs for pool health
Currently we don't track and manage a failure to run bvts
because of dead duts too well. It usually involves multiple
sheriffs and quite a bit of confusion, not to mention multiple
bugs across 2 trackers, before the right people are notified.
This cl files a bug from run_suite when a suite fails because
of insufficient DUTs, and notifies all interested parties.
TEST=Filed bugs.
BUG=chromium:419573,chromium:423847
Change-Id: I4749d6abfb147e058a491d86d707a651246b5ea2
Reviewed-on: https://chromium-review.googlesource.com/224555
Tested-by: Prashanth B <[email protected]>
Reviewed-by: Dan Shi <[email protected]>
Commit-Queue: Prashanth B <[email protected]>
diff --git a/site_utils/diagnosis_utils.py b/site_utils/diagnosis_utils.py
index 95e5e6b..b55a4b2 100644
--- a/site_utils/diagnosis_utils.py
+++ b/site_utils/diagnosis_utils.py
@@ -11,6 +11,7 @@
import common
from autotest_lib.server import utils
+from autotest_lib.server.cros.dynamic_suite import reporting
from autotest_lib.server.cros.dynamic_suite import reporting_utils
@@ -210,16 +211,26 @@
# setup_django_environment can't be imported now as paygen server does
# not have django package.
bad_statuses = ('Repair Failed', 'Repairing', 'Verifying')
- available_hosts = [host for host in hosts
- if not host.status in bad_statuses and
- not host.locked]
+ unusable_hosts = []
+ available_hosts = []
+ for host in hosts:
+ if host.status in bad_statuses or host.locked:
+ unusable_hosts.append(host.hostname)
+ else:
+ available_hosts.append(host)
logging.debug('%d of %d DUTs are available for board %s pool %s.',
len(available_hosts), len(hosts), board, pool)
if len(available_hosts) < minimum_duts:
+ bug_id = ''
+ if unusable_hosts:
+ pool_health_bug = reporting.PoolHealthBug(
+ pool, board, unusable_hosts)
+ bug_id = reporting.Reporter().report(pool_health_bug)[0]
raise NotEnoughDutsError(
'Number of available DUTs for board %s pool %s is %d, which'
- ' is less than the minimum value %d.' %
- (board, pool, len(available_hosts), minimum_duts))
+ ' is less than the minimum value %d. '
+ 'Filed https://crbug.com/%s' %
+ (board, pool, len(available_hosts), minimum_duts, bug_id))
def diagnose_job(self, job_id, instance_server):
