cheets: More improved test counting in case of repeated crash and duped tests. am: bf5a85b055 am: 94b4c97a89 am: 571d41e00e
am: 0dc9a7ed51
Change-Id: I438b7246f4211fbdc21f7ec3031b6dcdc5c460f1
diff --git a/server/cros/tradefed_test.py b/server/cros/tradefed_test.py
index f5d9e95..6c480b1 100644
--- a/server/cros/tradefed_test.py
+++ b/server/cros/tradefed_test.py
@@ -807,16 +807,20 @@
raise error.TestFail('Error: Unexpected test end: ' + line)
npass, nfail, nnotexec = map(int, match.group(2,3,4))
- if accumulative_count:
- total_test[abi] = ntest
- total_pass[abi] = npass
- total_fail[abi] = nfail
- else:
- total_test[abi] = (total_test.get(abi, 0) + ntest -
- last_notexec.get(abi, 0))
- total_pass[abi] = total_pass.get(abi, 0) + npass
- total_fail[abi] = total_fail.get(abi, 0) + nfail
- last_notexec[abi] = nnotexec
+ # When the test crashes too ofen, tradefed seems to finish the
+ # iteration by running "0 tests, 0 passed, ...". Do not count
+ # that in.
+ if ntest > 0:
+ if accumulative_count:
+ total_test[abi] = ntest
+ total_pass[abi] = npass
+ total_fail[abi] = nfail
+ else:
+ total_test[abi] = (total_test.get(abi, 0) + ntest -
+ last_notexec.get(abi, 0))
+ total_pass[abi] = total_pass.get(abi, 0) + npass
+ total_fail[abi] = total_fail.get(abi, 0) + nfail
+ last_notexec[abi] = nnotexec
abi = None
if abi:
diff --git a/server/site_tests/cheets_CTS_N/cheets_CTS_N.py b/server/site_tests/cheets_CTS_N/cheets_CTS_N.py
index 619b1ab..fe3dcf8 100644
--- a/server/site_tests/cheets_CTS_N/cheets_CTS_N.py
+++ b/server/site_tests/cheets_CTS_N/cheets_CTS_N.py
@@ -390,12 +390,20 @@
# Consistency check, did we really run as many as we
# thought initially?
if expected_tests != tests:
- retry_inconsistency_error = ('Retry inconsistency - '
+ msg = ('Retry inconsistency - '
'initially saw %d failed+notexecuted, ran %d tests. '
'passed=%d, failed=%d, notexecuted=%d, waived=%d.' %
(expected_tests, tests, passed, failed, notexecuted,
waived))
- logging.warning(retry_inconsistency_error)
+ logging.warning(msg)
+ if expected_tests > tests:
+ # See b/36523200#comment8. Due to the existence of
+ # the multiple tests having the same ID, more cases
+ # may be run than previous fail count. As a
+ # workaround, making it an error only when the tests
+ # run were less than expected.
+ # TODO(kinaba): Find a way to handle this dup.
+ retry_inconsistency_error = msg
if not self._consistent(tests, passed, failed, notexecuted):
logging.warning('Tradefed inconsistency - retrying.')
session_id, counts = self._tradefed_retry(test_name,
