Hopefully fix rare case when sync would get stuck indefinitely
This commit is contained in:
parent
82e38366f5
commit
e21f4c143d
1 changed files with 8 additions and 3 deletions
|
@ -136,6 +136,11 @@ class ProcessingQueue(queue.Queue, object):
|
||||||
return self._current_queue._qsize() if self._current_queue else 0
|
return self._current_queue._qsize() if self._current_queue else 0
|
||||||
|
|
||||||
def _total_qsize(self):
|
def _total_qsize(self):
|
||||||
|
"""
|
||||||
|
This method is BROKEN as it can lead to a deadlock when a single item
|
||||||
|
from the current section takes longer to download then any new items
|
||||||
|
coming in
|
||||||
|
"""
|
||||||
return sum(q._qsize() for q in self._queues) if self._queues else 0
|
return sum(q._qsize() for q in self._queues) if self._queues else 0
|
||||||
|
|
||||||
def put(self, item, block=True, timeout=None):
|
def put(self, item, block=True, timeout=None):
|
||||||
|
@ -147,16 +152,16 @@ class ProcessingQueue(queue.Queue, object):
|
||||||
try:
|
try:
|
||||||
if self.maxsize > 0:
|
if self.maxsize > 0:
|
||||||
if not block:
|
if not block:
|
||||||
if self._total_qsize() == self.maxsize:
|
if self._qsize() == self.maxsize:
|
||||||
raise queue.Full
|
raise queue.Full
|
||||||
elif timeout is None:
|
elif timeout is None:
|
||||||
while self._total_qsize() == self.maxsize:
|
while self._qsize() == self.maxsize:
|
||||||
self.not_full.wait()
|
self.not_full.wait()
|
||||||
elif timeout < 0:
|
elif timeout < 0:
|
||||||
raise ValueError("'timeout' must be a non-negative number")
|
raise ValueError("'timeout' must be a non-negative number")
|
||||||
else:
|
else:
|
||||||
endtime = _time() + timeout
|
endtime = _time() + timeout
|
||||||
while self._total_qsize() == self.maxsize:
|
while self._qsize() == self.maxsize:
|
||||||
remaining = endtime - _time()
|
remaining = endtime - _time()
|
||||||
if remaining <= 0.0:
|
if remaining <= 0.0:
|
||||||
raise queue.Full
|
raise queue.Full
|
||||||
|
|
Loading…
Reference in a new issue