Troplo/PlexKodiConnect
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Download entire xmls asynchronously, not just a piece

Browse source
This commit is contained in:
croneter 2018-11-11 10:47:18 +01:00
parent 14359dd5f4
commit 18891a67fc
1 changed files with 23 additions and 36 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

59
resources/lib/plex_functions.py
View file

@ -556,8 +556,7 @@ class DownloadGen(object):
def __init__(self, url, plex_type=None, last_viewed_at=None, def __init__(self, url, plex_type=None, last_viewed_at=None,
updated_at=None, args=None): updated_at=None, args=None):
self.lock = backgroundthread.threading.Lock() self.lock = backgroundthread.threading.Lock()
self.pending_lock = backgroundthread.threading.Lock() self.args = args or {}
self._args = args or {}
url += '?' url += '?'
if plex_type: if plex_type:
url = '%stype=%s&' % (url, v.PLEX_TYPE_NUMBER_FROM_PLEX_TYPE[plex_type]) url = '%stype=%s&' % (url, v.PLEX_TYPE_NUMBER_FROM_PLEX_TYPE[plex_type])
@ -565,48 +564,46 @@ class DownloadGen(object):
url = '%slastViewedAt>=%s&' % (url, last_viewed_at) url = '%slastViewedAt>=%s&' % (url, last_viewed_at)
if updated_at: if updated_at:
url = '%supdatedAt>=%s&' % (url, updated_at) url = '%supdatedAt>=%s&' % (url, updated_at)
self._url = url[:-1] self.url = url[:-1]
self._pos = 0 self._download_chunk(start=0)
self._current = 0
self._exhausted = False
# Start download the next chunk once we've reached 10% of the current
# xml's content
self._threshold = int(CONTAINERSIZE / 10)
self._download_chunk()
self.attrib = deepcopy(self.xml.attrib) self.attrib = deepcopy(self.xml.attrib)
self.total = int(self.attrib['totalSize'])
# Will keep track whether we still have results incoming
self.pending_counter = []
for i in range(CONTAINERSIZE,
self.total + (CONTAINERSIZE - self.total % CONTAINERSIZE),
CONTAINERSIZE):
self._download_chunk(start=i)
self.pending_counter.append(None)
def _download_chunk(self): def _download_chunk(self, start):
self._args.update({ self.args.update({
'X-Plex-Container-Size': CONTAINERSIZE, 'X-Plex-Container-Size': CONTAINERSIZE,
'X-Plex-Container-Start': self._pos, 'X-Plex-Container-Start': start,
'sort': 'id', # Entries are sorted by plex_id 'sort': 'id', # Entries are sorted by plex_id
'excludeAllLeaves': 1 # PMS wont attach a first summary child 'excludeAllLeaves': 1 # PMS wont attach a first summary child
}) })
if self._pos == 0: if start == 0:
self.xml = DU().downloadUrl(self._url, parameters=self._args) # We need the result NOW
self.xml = DU().downloadUrl(self.url, parameters=self.args)
try: try:
self.xml.attrib self.xml.attrib
except AttributeError: except AttributeError:
LOG.error('Error while downloading chunks: %s, args: %s', LOG.error('Error while downloading chunks: %s, args: %s',
self._url, self._args) self.url, self.args)
raise RuntimeError('Error while downloading chunks for %s' raise RuntimeError('Error while downloading chunks for %s'
% self._url) % self.url)
if len(self.xml) < CONTAINERSIZE:
self._exhausted = True
else: else:
self.pending_lock.acquire()
task = DownloadChunk() task = DownloadChunk()
task.setup(self._url, self._args, self.on_chunk_downloaded) task.setup(self.url, self.args, self.on_chunk_downloaded)
backgroundthread.BGThreader.addTask(task) backgroundthread.BGThreader.addTask(task)
def on_chunk_downloaded(self, xml): def on_chunk_downloaded(self, xml):
if xml: if xml:
if len(xml) < CONTAINERSIZE:
self._exhausted = True
with self.lock: with self.lock:
for child in xml: for child in xml:
self.xml.append(child) self.xml.append(child)
self.pending_lock.release() self.pending_counter.pop()
def __iter__(self): def __iter__(self):
return self return self
@ -615,24 +612,14 @@ class DownloadGen(object):
return self.__next__() return self.__next__()
def __next__(self): def __next__(self):
if (not self._exhausted and
self._current % CONTAINERSIZE == self._threshold):
# Kick off download of next chunk
self._pos += CONTAINERSIZE
self._download_chunk()
while True: while True:
if len(self.xml): if len(self.xml):
child = self.xml[0] child = self.xml[0]
with self.lock: with self.lock:
self.xml.remove(child) self.xml.remove(child)
self._current += 1
return child return child
# Currently no items left, BUT some could still be downloading sleep(20)
self.pending_lock.acquire() if not len(self.pending_counter) and not len(self.xml):
# Above will block until download finished - release immediately
# again
self.pending_lock.release()
if not len(self.xml):
raise StopIteration raise StopIteration
def get(self, key, default=None): def get(self, key, default=None):