Revert "Revert "[autotest] TKO parser mark original tests as invalid""
This reverts commit dbd9037df0cbe1232fcb016a2c261f5994ae03a2.
Commit this cl again. Once this cl lands, deploy the db changes together with
https://chrome-internal-review.googlesource.com/#/c/161494/
https://chrome-internal-review.googlesource.com/#/c/161426/
DEPLOY=apache, migrate, wmatrix db changes should be pushed together (see commit messeage)
Change-Id: I47916708e7b49bbc2064370262e3f9e231f08efe
Reviewed-on: https://chromium-review.googlesource.com/197306
Reviewed-by: Dan Shi <[email protected]>
Tested-by: Fang Deng <[email protected]>
Commit-Queue: Fang Deng <[email protected]>
diff --git a/frontend/migrations/086_add_invalidates_test_idx_to_tko_tests.py b/frontend/migrations/086_add_invalidates_test_idx_to_tko_tests.py
new file mode 100644
index 0000000..ffded8c
--- /dev/null
+++ b/frontend/migrations/086_add_invalidates_test_idx_to_tko_tests.py
@@ -0,0 +1,35 @@
+ADD_COLUMN = """
+ALTER TABLE tko_tests
+ADD COLUMN `invalidates_test_idx` int(10) unsigned DEFAULT NULL;
+"""
+ADD_INDEX = """ALTER TABLE tko_tests ADD INDEX(invalidates_test_idx);"""
+ADD_FOREIGN_KEY = """
+ALTER TABLE tko_tests
+ADD CONSTRAINT invalidates_test_idx_fk FOREIGN KEY
+(`invalidates_test_idx`) REFERENCES `tko_tests`(`test_idx`)
+ON DELETE NO ACTION;
+"""
+DROP_FOREIGN_KEY = """
+ALTER TABLE tko_tests DROP FOREIGN KEY `invalidates_test_idx_fk`;
+"""
+DROP_COLUMN = """ALTER TABLE tko_tests DROP `invalidates_test_idx`; """
+
+def migrate_up(manager):
+ """Pick up the changes.
+
+ @param manager: A MigrationManager object.
+
+ """
+ manager.execute(ADD_COLUMN)
+ manager.execute(ADD_INDEX)
+ manager.execute(ADD_FOREIGN_KEY)
+
+
+def migrate_down(manager):
+ """Drop the changes.
+
+ @param manager: A MigrationManager object.
+
+ """
+ manager.execute(DROP_FOREIGN_KEY)
+ manager.execute(DROP_COLUMN)
diff --git a/frontend/tko/models.py b/frontend/tko/models.py
index a9c6bf0..286279b 100644
--- a/frontend/tko/models.py
+++ b/frontend/tko/models.py
@@ -210,6 +210,10 @@
machine = dbmodels.ForeignKey(Machine, db_column='machine_idx')
finished_time = dbmodels.DateTimeField(null=True, blank=True)
started_time = dbmodels.DateTimeField(null=True, blank=True)
+ invalid = dbmodels.BooleanField(default=False)
+ invalidates_test = dbmodels.ForeignKey(
+ 'self', null=True, db_column='invalidates_test_idx',
+ related_name='invalidates_test_set')
objects = model_logic.ExtendedManager()
diff --git a/tko/parse.py b/tko/parse.py
index ef7971a..0105a28 100755
--- a/tko/parse.py
+++ b/tko/parse.py
@@ -3,11 +3,14 @@
import os, sys, optparse, fcntl, errno, traceback, socket
import common
+from autotest_lib.frontend import setup_django_environment
from autotest_lib.client.common_lib import mail, pidfile
+from autotest_lib.frontend.tko import models as tko_models
from autotest_lib.tko import db as tko_db, utils as tko_utils
from autotest_lib.tko import models, status_lib
from autotest_lib.tko.perf_upload import perf_uploader
from autotest_lib.client.common_lib import utils
+from autotest_lib.server.cros.dynamic_suite import constants
def parse_args():
@@ -71,6 +74,85 @@
mail.send("", job.user, "", subject, message_header + message)
+def _invalidate_original_tests(orig_job_idx, retry_job_idx):
+ """Retry tests invalidates original tests.
+
+ Whenever a retry job is complete, we want to invalidate the original
+ job's test results, such that the consumers of the tko database
+ (e.g. tko frontend, wmatrix) could figure out which results are the latest.
+
+ When a retry job is parsed, we retrieve the original job's afe_job_id
+ from the retry job's keyvals, which is then converted to tko job_idx and
+ passed into this method as |orig_job_idx|.
+
+ In this method, we are going to invalidate the rows in tko_tests that are
+ associated with the original job by flipping their 'invalid' bit to True.
+ In addition, in tko_tests, we also maintain a pointer from the retry results
+ to the original results, so that later we can always know which rows in
+ tko_tests are retries and which are the corresponding original results.
+ This is done by setting the field 'invalidates_test_idx' of the tests
+ associated with the retry job.
+
+ For example, assume Job(job_idx=105) are retried by Job(job_idx=108), after
+ this method is run, their tko_tests rows will look like:
+ __________________________________________________________________________
+ test_idx| job_idx | test | ... | invalid | invalidates_test_idx
+ 10 | 105 | dummy_Fail.Error| ... | 1 | NULL
+ 11 | 105 | dummy_Fail.Fail | ... | 1 | NULL
+ ...
+ 20 | 108 | dummy_Fail.Error| ... | 0 | 10
+ 21 | 108 | dummy_Fail.Fail | ... | 0 | 11
+ __________________________________________________________________________
+ Note the invalid bits of the rows for Job(job_idx=105) are set to '1'.
+ And the 'invalidates_test_idx' fields of the rows for Job(job_idx=108)
+ are set to 10 and 11 (the test_idx of the rows for the original job).
+
+ @param orig_job_idx: An integer representing the original job's
+ tko job_idx. Tests associated with this job will
+ be marked as 'invalid'.
+ @param retry_job_idx: An integer representing the retry job's
+ tko job_idx. The field 'invalidates_test_idx'
+ of the tests associated with this job will be updated.
+
+ """
+ msg = 'orig_job_idx: %s, retry_job_idx: %s' % (orig_job_idx, retry_job_idx)
+ if not orig_job_idx or not retry_job_idx:
+ tko_utils.dprint('ERROR: Could not invalidate tests: ' + msg)
+ # Using django models here makes things easier, but make sure that
+ # before this method is called, all other relevant transactions have been
+ # committed to avoid race condition. In the long run, we might consider
+ # to make the rest of parser use django models.
+ orig_tests = tko_models.Test.objects.filter(job__job_idx=orig_job_idx)
+ retry_tests = tko_models.Test.objects.filter(job__job_idx=retry_job_idx)
+
+ # Invalidate original tests.
+ orig_tests.update(invalid=True)
+
+ # Maintain a dictionary that maps (test, subdir) to original tests.
+ # Note that within the scope of a job, (test, subdir) uniquelly
+ # identifies a test run, but 'test' does not.
+ # In a control file, one could run the same test with different
+ # 'subdir_tag', for example,
+ # job.run_test('dummy_Fail', tag='Error', subdir_tag='subdir_1')
+ # job.run_test('dummy_Fail', tag='Error', subdir_tag='subdir_2')
+ # In tko, we will get
+ # (test='dummy_Fail.Error', subdir='dummy_Fail.Error.subdir_1')
+ # (test='dummy_Fail.Error', subdir='dummy_Fail.Error.subdir_2')
+ invalidated_tests = {(orig_test.test, orig_test.subdir): orig_test
+ for orig_test in orig_tests}
+ for retry in retry_tests:
+ # It is possible that (retry.test, retry.subdir) doesn't exist
+ # in invalidated_tests. This could happen when the original job
+ # didn't run some of its tests. For example, a dut goes offline
+ # since the beginning of the job, in which case invalidated_tests
+ # will only have one entry for 'SERVER_JOB'.
+ orig_test = invalidated_tests.get((retry.test, retry.subdir), None)
+ if orig_test:
+ retry.invalidates_test = orig_test
+ retry.save()
+ tko_utils.dprint('DEBUG: Invalidated tests associated to job: ' + msg)
+
+
def parse_one(db, jobname, path, reparse, mail_on_failure):
"""
Parse a single job. Optionally send email on failure.
@@ -158,13 +240,26 @@
% (jobname, job.user))
mailfailure(jobname, job, message)
- # write the job into the database
+ # write the job into the database.
db.insert_job(jobname, job)
# Upload perf values to the perf dashboard, if applicable.
for test in job.tests:
perf_uploader.upload_test(job, test)
+ # Although the cursor has autocommit, we still need to force it to commit
+ # existing changes before we can use django models, otherwise it
+ # will go into deadlock when django models try to start a new trasaction
+ # while the current one has not finished yet.
+ db.commit()
+
+ # Handle retry job.
+ orig_afe_job_id = job_keyval.get(constants.RETRY_ORIGINAL_JOB_ID, None)
+ if orig_afe_job_id:
+ orig_job_idx = tko_models.Job.objects.get(
+ afe_job_id=orig_afe_job_id).job_idx
+ _invalidate_original_tests(orig_job_idx, job.index)
+
# Serializing job into a binary file
try:
from autotest_lib.tko import tko_pb2
diff --git a/tko/site_parse_unittest.py b/tko/site_parse_unittest.py
index 4a2a485..f54b772 100755
--- a/tko/site_parse_unittest.py
+++ b/tko/site_parse_unittest.py
@@ -11,6 +11,8 @@
import common
from autotest_lib.client.common_lib import global_config
+from autotest_lib.frontend import setup_django_environment
+from autotest_lib.frontend import setup_test_environment
from autotest_lib.tko.site_parse import StackTrace
@@ -18,6 +20,7 @@
def setUp(self):
+ setup_test_environment.set_up()
self._fake_results = tempfile.mkdtemp()
self._cros_src_dir = global_config.global_config.get_config_value(
'CROS', 'source_tree', default=None)
@@ -39,6 +42,7 @@
def tearDown(self):
+ setup_test_environment.tear_down()
shutil.rmtree(self._fake_results)
if os.path.exists(self._cache_dir):
shutil.rmtree(self._cache_dir)
