Hopefully fix rare case when sync would get stuck indefinitely

This commit is contained in:
croneter 2021-01-31 17:39:44 +01:00
parent 82e38366f5
commit e21f4c143d

View file

@ -136,6 +136,11 @@ class ProcessingQueue(queue.Queue, object):
return self._current_queue._qsize() if self._current_queue else 0 return self._current_queue._qsize() if self._current_queue else 0
def _total_qsize(self): def _total_qsize(self):
"""
This method is BROKEN as it can lead to a deadlock when a single item
from the current section takes longer to download then any new items
coming in
"""
return sum(q._qsize() for q in self._queues) if self._queues else 0 return sum(q._qsize() for q in self._queues) if self._queues else 0
def put(self, item, block=True, timeout=None): def put(self, item, block=True, timeout=None):
@ -147,16 +152,16 @@ class ProcessingQueue(queue.Queue, object):
try: try:
if self.maxsize > 0: if self.maxsize > 0:
if not block: if not block:
if self._total_qsize() == self.maxsize: if self._qsize() == self.maxsize:
raise queue.Full raise queue.Full
elif timeout is None: elif timeout is None:
while self._total_qsize() == self.maxsize: while self._qsize() == self.maxsize:
self.not_full.wait() self.not_full.wait()
elif timeout < 0: elif timeout < 0:
raise ValueError("'timeout' must be a non-negative number") raise ValueError("'timeout' must be a non-negative number")
else: else:
endtime = _time() + timeout endtime = _time() + timeout
while self._total_qsize() == self.maxsize: while self._qsize() == self.maxsize:
remaining = endtime - _time() remaining = endtime - _time()
if remaining <= 0.0: if remaining <= 0.0:
raise queue.Full raise queue.Full