Hopefully fix rare case when sync would get stuck indefinitely
This commit is contained in:
parent
82e38366f5
commit
e21f4c143d
1 changed files with 8 additions and 3 deletions
|
@ -136,6 +136,11 @@ class ProcessingQueue(queue.Queue, object):
|
|||
return self._current_queue._qsize() if self._current_queue else 0
|
||||
|
||||
def _total_qsize(self):
|
||||
"""
|
||||
This method is BROKEN as it can lead to a deadlock when a single item
|
||||
from the current section takes longer to download then any new items
|
||||
coming in
|
||||
"""
|
||||
return sum(q._qsize() for q in self._queues) if self._queues else 0
|
||||
|
||||
def put(self, item, block=True, timeout=None):
|
||||
|
@ -147,16 +152,16 @@ class ProcessingQueue(queue.Queue, object):
|
|||
try:
|
||||
if self.maxsize > 0:
|
||||
if not block:
|
||||
if self._total_qsize() == self.maxsize:
|
||||
if self._qsize() == self.maxsize:
|
||||
raise queue.Full
|
||||
elif timeout is None:
|
||||
while self._total_qsize() == self.maxsize:
|
||||
while self._qsize() == self.maxsize:
|
||||
self.not_full.wait()
|
||||
elif timeout < 0:
|
||||
raise ValueError("'timeout' must be a non-negative number")
|
||||
else:
|
||||
endtime = _time() + timeout
|
||||
while self._total_qsize() == self.maxsize:
|
||||
while self._qsize() == self.maxsize:
|
||||
remaining = endtime - _time()
|
||||
if remaining <= 0.0:
|
||||
raise queue.Full
|
||||
|
|
Loading…
Reference in a new issue