Revert "[autotest] Fix constraints in hqe_by_insert_time"
This reverts commit e1207af9ea77dfe241c14639a773449f6d07ecf5.
Reason for revert: Speculative fix for crbug.com/781302
Original change's description:
> [autotest] Fix constraints in hqe_by_insert_time
>
> get_host_queue_entries_by_insert_time was showing some strange behavior because
> it was making a wrong assumptions about the gap between insert-time and
> start-time being small.
>
> - Renamed to "get_host_queue_entries_by_start_time"
> - Changed the API to be based on started_on constraints.
> - Changed the function to infer only correct insert-time constraints. This
> means we cannot assume any lower bound on the insert-time, because the gap
> between insert and start times may be arbitrarily large.
> - Added more unit test coverage.
>
> BUG=chromium:748209
> TEST=unit tests pass.
>
> Change-Id: I60133e7a5f6431f35349c17f0644bc04e04cfcbd
> Reviewed-on: https://chromium-review.googlesource.com/685735
> Commit-Ready: Paul Hobbs <[email protected]>
> Tested-by: Paul Hobbs <[email protected]>
> Reviewed-by: Paul Hobbs <[email protected]>
Bug: chromium:748209
Change-Id: I6a4e3c030527aafdd43f1218f514fbd3bbe6bbc6
Reviewed-on: https://chromium-review.googlesource.com/753998
Commit-Ready: Paul Hobbs <[email protected]>
Tested-by: Paul Hobbs <[email protected]>
Reviewed-by: Allen Li <[email protected]>
diff --git a/client/common_lib/time_utils.py b/client/common_lib/time_utils.py
index 7835ab7..1e014ff 100644
--- a/client/common_lib/time_utils.py
+++ b/client/common_lib/time_utils.py
@@ -9,11 +9,13 @@
from autotest_lib.client.common_lib import decorators
+
try:
import pytz
except ImportError:
pytz = None
+
try:
import tzlocal
except ImportError:
diff --git a/frontend/afe/rpc_interface.py b/frontend/afe/rpc_interface.py
index 7c67954..e4cad35 100644
--- a/frontend/afe/rpc_interface.py
+++ b/frontend/afe/rpc_interface.py
@@ -1227,35 +1227,35 @@
return image
-
-def get_host_queue_entries_by_start_time(
- insert_time_after=None, insert_time_before=None,
- started_on__gte=None, started_on__lte=None, **filter_data):
+def get_host_queue_entries_by_insert_time(
+ insert_time_after=None, insert_time_before=None, **filter_data):
"""Like get_host_queue_entries, but using the insert index table.
- TODO(phobbs) remove these dead arguments after the API change has finished
- rolling out:
-
- @param insert_time_after: (ignored)
- @param insert_time_before: (ignored)
- @param started_on__gte: (optional) A datetime string lower bound for the
- hqe's started_on column.
- @param started_on__lte: (optional) A datetime string upper bound for the
- hqe's started_on column.
+ @param insert_time_after: A lower bound on insert_time
+ @param insert_time_before: An upper bound on insert_time
@returns A sequence of nested dictionaries of host and job information.
"""
- assert started_on__gte is not None or started_on__lte is not None, \
- ('Caller to get_host_queue_entries_by_start_time must provide '
- ' either started_on__gte or started_on__lte.')
-
- # We know a upper bound on when the job was started, started_on__lte.
- # We know insert_time < started_on.
- # If the started_on < t, then insert_time < t by transitivity.
- if started_on__lte:
- # The HQE start times table is inserted periodically, so to be safe
- # we need to get the next HQE-start-time row after our constraint.
+ assert insert_time_after is not None or insert_time_before is not None, \
+ ('Caller to get_host_queue_entries_by_insert_time must provide either'
+ ' insert_time_after or insert_time_before.')
+ # Get insert bounds on the index of the host queue entries.
+ if insert_time_after:
query = models.HostQueueEntryStartTimes.objects.filter(
- insert_time__gte=started_on__lte).order_by('-insert_time')
+ # Note: '-insert_time' means descending. We want the largest
+ # insert time smaller than the insert time.
+ insert_time__lte=insert_time_after).order_by('-insert_time')
+ try:
+ constraint = query[0].highest_hqe_id
+ if 'id__gte' in filter_data:
+ constraint = max(constraint, filter_data['id__gte'])
+ filter_data['id__gte'] = constraint
+ except IndexError:
+ pass
+
+ # Get end bounds.
+ if insert_time_before:
+ query = models.HostQueueEntryStartTimes.objects.filter(
+ insert_time__gte=insert_time_before).order_by('insert_time')
try:
constraint = query[0].highest_hqe_id
if 'id__lte' in filter_data:
@@ -1264,25 +1264,11 @@
except IndexError:
pass
- # We can't provide a lower bound on the when the job was inserted because
- # there may be an arbitrarily large gap between insert and start time (even
- # days long).
-
- # We still need to provide the started_on constraints - the id constraint
- # from insert time isn't sufficient because the index table is approximate.
- if started_on__gte:
- filter_data['started_on__gte'] = started_on__gte
- if started_on__lte:
- filter_data['started_on__lte'] = started_on__lte
return rpc_utils.prepare_rows_as_nested_dicts(
models.HostQueueEntry.query_objects(filter_data),
('host', 'job'))
-# TODO(phobbs) remove the below once this has finished rolling out.
-get_host_queue_entries_by_insert_time = get_host_queue_entries_by_start_time
-
-
def get_host_queue_entries(start_time=None, end_time=None, **filter_data):
"""\
@returns A sequence of nested dictionaries of host and job information.
diff --git a/frontend/afe/rpc_interface_unittest.py b/frontend/afe/rpc_interface_unittest.py
index f9e1bf2..635c7f1 100755
--- a/frontend/afe/rpc_interface_unittest.py
+++ b/frontend/afe/rpc_interface_unittest.py
@@ -332,67 +332,83 @@
job = self._create_job(shard=shard, control_file='foo')
HqeStatus = models.HostQueueEntry.Status
- for i, day in enumerate(['2017-01-01', '2017-01-02', '2017-01-03']):
- models.HostQueueEntryStartTimes(
- insert_time=day, highest_hqe_id=i*2).save()
- models.HostQueueEntry(
- id=i*2, job=job, started_on=day,
- status=HqeStatus.QUEUED).save()
- models.HostQueueEntry(
- id=i*2 + 1, job=job, started_on=day + ' 12:00:00',
- status=HqeStatus.QUEUED).save()
+ models.HostQueueEntry(
+ id=1, job=job, started_on='2017-01-01',
+ status=HqeStatus.QUEUED).save()
+ models.HostQueueEntry(
+ id=2, job=job, started_on='2017-01-02',
+ status=HqeStatus.QUEUED).save()
+ models.HostQueueEntry(
+ id=3, job=job, started_on='2017-01-03',
+ status=HqeStatus.QUEUED).save()
- def test_get_hqe_by_insert_time_lower_bounds(self):
- """Check the started_on__gte constraints."""
+ models.HostQueueEntryStartTimes(
+ insert_time='2017-01-03', highest_hqe_id=3).save()
+ models.HostQueueEntryStartTimes(
+ insert_time='2017-01-02', highest_hqe_id=2).save()
+ models.HostQueueEntryStartTimes(
+ insert_time='2017-01-01', highest_hqe_id=1).save()
+
+ def test_get_host_queue_entries_by_insert_time(self):
+ """Check the insert_time_after and insert_time_before constraints."""
self._create_hqes_and_start_time_index_entries()
hqes = rpc_interface.get_host_queue_entries_by_insert_time(
- started_on__gte='2017-01-01')
- self.assertEquals(len(hqes), 6)
-
- hqes = rpc_interface.get_host_queue_entries_by_insert_time(
- started_on__gte='2017-01-02')
- self.assertEquals(len(hqes), 4)
-
- hqes = rpc_interface.get_host_queue_entries_by_insert_time(
- started_on__gte='2017-01-03')
- self.assertEquals(len(hqes), 2)
-
- def test_get_hqe_by_insert_time_upper_bounds(self):
- """Check the started_on__lte constraints."""
- self._create_hqes_and_start_time_index_entries()
- hqes = rpc_interface.get_host_queue_entries_by_insert_time(
- started_on__lte='2017-01-01')
- self.assertEquals(len(hqes), 1)
-
- hqes = rpc_interface.get_host_queue_entries_by_insert_time(
- started_on__lte='2017-01-02')
+ insert_time_after='2017-01-01')
self.assertEquals(len(hqes), 3)
hqes = rpc_interface.get_host_queue_entries_by_insert_time(
- started_on__lte='2017-01-03')
- self.assertEquals(len(hqes), 5)
+ insert_time_after='2017-01-02')
+ self.assertEquals(len(hqes), 2)
+
+ hqes = rpc_interface.get_host_queue_entries_by_insert_time(
+ insert_time_after='2017-01-03')
+ self.assertEquals(len(hqes), 1)
+
+ hqes = rpc_interface.get_host_queue_entries_by_insert_time(
+ insert_time_before='2017-01-01')
+ self.assertEquals(len(hqes), 1)
+
+ hqes = rpc_interface.get_host_queue_entries_by_insert_time(
+ insert_time_before='2017-01-02')
+ self.assertEquals(len(hqes), 2)
+
+ hqes = rpc_interface.get_host_queue_entries_by_insert_time(
+ insert_time_before='2017-01-03')
+ self.assertEquals(len(hqes), 3)
+
+
+ def test_get_host_queue_entries_by_insert_time_with_missing_index_row(self):
+ """Shows that the constraints are approximate.
+
+ The query may return rows which are actually outside of the bounds
+ given, if the index table does not have an entry for the specific time.
+ """
+ self._create_hqes_and_start_time_index_entries()
+ hqes = rpc_interface.get_host_queue_entries_by_insert_time(
+ insert_time_before='2016-12-01')
+ self.assertEquals(len(hqes), 1)
def test_get_hqe_by_insert_time_with_before_and_after(self):
self._create_hqes_and_start_time_index_entries()
hqes = rpc_interface.get_host_queue_entries_by_insert_time(
- started_on__lte='2017-01-02 13:00:00',
- started_on__gte='2017-01-02')
- self.assertEquals(len(hqes), 2)
+ insert_time_before='2017-01-02',
+ insert_time_after='2017-01-02')
+ self.assertEquals(len(hqes), 1)
def test_get_hqe_by_insert_time_and_id_constraint(self):
self._create_hqes_and_start_time_index_entries()
- # The time constraint is looser than the id constraint, so the id
+ # The time constraint is looser than the id constraint, so the time
# constraint should take precedence.
hqes = rpc_interface.get_host_queue_entries_by_insert_time(
- started_on__lte='2017-01-02',
+ insert_time_before='2017-01-02',
id__lte=1)
- self.assertEquals(len(hqes), 2)
+ self.assertEquals(len(hqes), 1)
# Now make the time constraint tighter than the id constraint.
hqes = rpc_interface.get_host_queue_entries_by_insert_time(
- started_on__lte='2017-01-02',
+ insert_time_before='2017-01-01',
id__lte=42)
- self.assertEquals(len(hqes), 3)
+ self.assertEquals(len(hqes), 1)
def test_view_invalid_host(self):
# RPCs used by View Host page should work for invalid hosts
diff --git a/server/lib/status_history.py b/server/lib/status_history.py
index c810c4c..fb7afd0 100755
--- a/server/lib/status_history.py
+++ b/server/lib/status_history.py
@@ -375,6 +375,8 @@
query_end = time_utils.epoch_time_to_date_string(end_time)
hqelist = afe.get_host_queue_entries_by_insert_time(
host_id=host_id,
+ insert_time_after=query_start,
+ insert_time_before=query_end,
started_on__gte=query_start,
started_on__lte=query_end,
complete=1)
