From e6530d11d058e50c41872f3c2c9ac286b57ed70e Mon Sep 17 00:00:00 2001 From: "James E. Blair" Date: Sat, 2 Jul 2022 10:29:28 -0700 Subject: Reduce redundant Gerrit queries Sometimes Gerrit events may arrive in batches (for example, an automated process modifies several related changes nearly simultaneously). Because of our inbuilt delay (10 seconds by default), it's possible that in these cases, many or all of the updates represented by these events will have settled on the Gerrit server before we even start processing the first event. In these cases, we don't need to query the same changes multiple times. Take for example a stack of 10 changes. Someone approves all 10 simultaneously. That would produce (at least) 10 events for Zuul to process. Each event would cause Zuul to query all 10 changes in the series (since they are related). That's 100 change queries (and each change query requires 2 or 3 HTTP queries). But if we know that all the event arrived before our first set of change queries, we can reduce that set of 100 queries to 10 by suppressing any queries after the first. This change generates a logical timestamp (ltime) immediately before querying Gerrit for a change, and stores that ltime in the change cache. Whenever an event arrives for processing with an ltime later than the query ltime, we assume the change is already up to date with that event and do not perform another query. Change-Id: Ib1b9245cc84ab3f5a0624697f4e3fc73bc8e03bd --- zuul/zk/event_queues.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'zuul/zk') diff --git a/zuul/zk/event_queues.py b/zuul/zk/event_queues.py index 8718f609c..fa4e06bd0 100644 --- a/zuul/zk/event_queues.py +++ b/zuul/zk/event_queues.py @@ -884,7 +884,7 @@ class ConnectionEventQueue(ZooKeeperEventQueue): self._put({'event_data': data}) def __iter__(self): - for data, ack_ref, _ in self._iterEvents(): + for data, ack_ref, zstat in self._iterEvents(): if not data: self.log.warning("Malformed event found: %s", data) self._remove(ack_ref.path) @@ -893,6 +893,7 @@ class ConnectionEventQueue(ZooKeeperEventQueue): event = model.ConnectionEvent.fromDict( data.get('event_data', data)) event.ack_ref = ack_ref + event.zuul_event_ltime = zstat.creation_transaction_id yield event -- cgit v1.2.1