Troplo/PlexKodiConnect
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Merge pull request #1056 from croneter/optimize

Browse source 
Optimize some code
This commit is contained in:
croneter 2019-11-14 17:20:06 +01:00 committed by GitHub
commit 1654c3175e
No known key found for this signature in database
GPG key ID: 4AEE18F83AFDEB23
2 changed files with 88 additions and 121 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

17
resources/lib/library_sync/full_sync.py
View file

@ -192,12 +192,10 @@ class FullSync(common.fullsync_mixin):
self.process_item(xml_item)
if self.item_count == BATCH_SIZE:
break
# Make sure Plex DB above is closed before adding/updating
if self.item_count == BATCH_SIZE:
self.update_library()
# Make sure Plex DB above is closed before adding/updating!
self.update_library()
if last:
break
self.update_library()
reset_collections()
return True
except RuntimeError:
@ -252,7 +250,7 @@ class FullSync(common.fullsync_mixin):
def threaded_get_iterators(self, kinds, queue, all_items=False):
"""
PF.SectionItems is costly, so let's do it asynchronous
Getting iterators is costly, so let's do it asynchronously
"""
try:
for kind in kinds:
@ -276,10 +274,11 @@ class FullSync(common.fullsync_mixin):
updated_at = section.last_sync - UPDATED_AT_SAFETY \
if section.last_sync else None
try:
element.iterator = PF.SectionItems(section.section_id,
plex_type=element.plex_type,
updated_at=updated_at,
last_viewed_at=None)
element.iterator = PF.get_section_iterator(
section.section_id,
plex_type=element.plex_type,
updated_at=updated_at,
last_viewed_at=None)
except RuntimeError:
LOG.warn('Sync at least partially unsuccessful')
self.successful = False

192
resources/lib/plex_functions.py
View file

@ -557,23 +557,7 @@ def GetAllPlexChildren(key):
return DownloadChunks("{server}/library/metadata/%s/children" % key)
def GetPlexSectionResults(viewId, args=None):
"""
Returns a list (XML API dump) of all Plex items in the Plex
section with key = viewId.
Input:
args: optional dict to be urlencoded
Returns None if something went wrong
"""
url = "{server}/library/sections/%s/all" % viewId
if args:
url = utils.extend_url(url, args)
return DownloadChunks(url)
class DownloadChunk(backgroundthread.Task):
class ThreadedDownloadChunk(backgroundthread.Task):
"""
This task will also be executed while library sync is suspended!
"""
@ -581,7 +565,7 @@ class DownloadChunk(backgroundthread.Task):
self.url = url
self.args = args
self.callback = callback
super(DownloadChunk, self).__init__()
super(ThreadedDownloadChunk, self).__init__()
def run(self):
xml = DU().downloadUrl(self.url, parameters=self.args)
@ -601,14 +585,15 @@ class DownloadGen(object):
Yields XML etree children or raises RuntimeError at the end
"""
def __init__(self, url, plex_type=None, last_viewed_at=None,
updated_at=None, args=None):
def __init__(self, url, plex_type, last_viewed_at, updated_at, args,
downloader):
self._downloader = downloader
self.successful = True
self.args = args or {}
self.xml = None
self.args = args
self.args.update({
'X-Plex-Container-Size': CONTAINERSIZE,
'sort': 'id', # Entries are sorted by plex_id
'excludeAllLeaves': 1 # PMS wont attach a first summary child
'X-Plex-Container-Start': 0,
'X-Plex-Container-Size': CONTAINERSIZE
})
url += '?'
if plex_type:
@ -618,8 +603,8 @@ class DownloadGen(object):
if updated_at:
url = '%supdatedAt>=%s&' % (url, updated_at)
self.url = url[:-1]
self._download_chunk(start=0)
self.attrib = deepcopy(self.xml.attrib)
_blocking_download_chunk(self.url, self.args, 0, self.set_xml)
self.attrib = self.xml.attrib
self.current = 0
self.total = int(self.attrib['totalSize'])
self.cache_factor = 10
@ -629,34 +614,24 @@ class DownloadGen(object):
self.total + CONTAINERSIZE - self.total % CONTAINERSIZE)
for pos in range(CONTAINERSIZE, end, CONTAINERSIZE):
self.pending_counter.append(None)
self._download_chunk(start=pos)
self._downloader(self.url, self.args, pos, self.on_chunk_downloaded)
def _download_chunk(self, start):
self.args['X-Plex-Container-Start'] = start
if start == 0:
# We need the result NOW
self.xml = DU().downloadUrl(self.url, parameters=self.args)
try:
self.xml.attrib
except AttributeError:
LOG.error('Error while downloading chunks: %s, args: %s',
self.url, self.args)
raise RuntimeError('Error while downloading chunks for %s'
% self.url)
else:
task = DownloadChunk(self.url,
deepcopy(self.args), # Beware!
self.on_chunk_downloaded)
backgroundthread.BGThreader.addTask(task)
def set_xml(self, xml):
self.xml = xml
def on_chunk_downloaded(self, xml):
if xml is not None:
for child in xml:
self.xml.append(child)
self.xml.extend(xml)
else:
self.successful = False
self.pending_counter.pop()
def get(self, key, default=None):
"""
Mimick etree xml's way to access xml.attrib via xml.get(key, default)
"""
return self.attrib.get(key, default)
def __iter__(self):
return self
@ -669,8 +644,11 @@ class DownloadGen(object):
if (self.current % CONTAINERSIZE == 0 and
self.current <= self.total - (self.cache_factor - 1) * CONTAINERSIZE):
self.pending_counter.append(None)
self._download_chunk(
start=self.current + (self.cache_factor - 1) * CONTAINERSIZE)
self._downloader(
self.url,
self.args,
self.current + (self.cache_factor - 1) * CONTAINERSIZE,
self.on_chunk_downloaded)
return child
except IndexError:
if not self.pending_counter and not len(self.xml):
@ -679,46 +657,67 @@ class DownloadGen(object):
else:
raise StopIteration()
LOG.debug('Waiting for download to finish')
app.APP.monitor.waitForAbort(0.1)
if app.APP.monitor.waitForAbort(0.1):
raise StopIteration('PKC needs to exit now')
next = __next__
def get(self, key, default=None):
return self.attrib.get(key, default)
def _blocking_download_chunk(url, args, start, callback):
"""
callback will be called with the downloaded xml (fragment)
"""
args['X-Plex-Container-Start'] = start
xml = DU().downloadUrl(url, parameters=args)
try:
xml.attrib
except AttributeError:
LOG.error('Error while downloading chunks: %s, args: %s',
url, args)
raise RuntimeError('Error while downloading chunks for %s'
% url)
callback(xml)
class SectionItems(DownloadGen):
"""
Iterator object to get all items of a Plex library section
"""
def __init__(self, section_id, plex_type=None, last_viewed_at=None,
updated_at=None, args=None):
if plex_type in (v.PLEX_TYPE_EPISODE, v.PLEX_TYPE_SONG):
# Annoying Plex bug. You won't get all episodes otherwise
url = '{server}/library/sections/%s/allLeaves' % section_id
plex_type = None
else:
url = '{server}/library/sections/%s/all' % section_id
super(SectionItems, self).__init__(url, plex_type, last_viewed_at,
updated_at, args)
def _async_download_chunk(url, args, start, callback):
args['X-Plex-Container-Start'] = start
task = ThreadedDownloadChunk(url,
deepcopy(args), # Beware!
callback)
backgroundthread.BGThreader.addTask(task)
class Children(DownloadGen):
"""
Iterator object to get all items of a Plex library section
"""
def __init__(self, plex_id):
super(Children, self).__init__(
'{server}/library/metadata/%s/children' % plex_id)
class Leaves(DownloadGen):
"""
Iterator object to get all items of a Plex library section
"""
def __init__(self, section_id):
super(Leaves, self).__init__(
'{server}/library/sections/%s/allLeaves' % section_id)
def get_section_iterator(section_id, plex_type=None, last_viewed_at=None,
updated_at=None, args=None):
args = args or {}
args.update({
'checkFiles': 0,
'includeExtras': 0, # Trailers and Extras => Extras
'includeReviews': 0,
'includeRelated': 0, # Similar movies => Video -> Related
'skipRefresh': 1, # don't scan
'excludeAllLeaves': 1 # PMS wont attach a first summary child
})
if plex_type == v.PLEX_TYPE_ALBUM:
# Kodi sorts Newest Albums by their position within the Kodi music
# database - great...
downloader = _blocking_download_chunk
args['sort'] = 'addedAt:asc'
else:
downloader = _async_download_chunk
args['sort'] = 'id' # Entries are sorted by plex_id
if plex_type in (v.PLEX_TYPE_EPISODE, v.PLEX_TYPE_SONG):
# Annoying Plex bug. You won't get all episodes otherwise
url = '{server}/library/sections/%s/allLeaves' % section_id
plex_type = None
else:
url = '{server}/library/sections/%s/all' % section_id
return DownloadGen(url,
plex_type,
last_viewed_at,
updated_at,
args,
downloader)
def DownloadChunks(url):
@ -767,37 +766,6 @@ def DownloadChunks(url):
return xml
def GetAllPlexLeaves(viewId, lastViewedAt=None, updatedAt=None):
"""
Returns a list (raw XML API dump) of all Plex subitems for the key.
(e.g. /library/sections/2/allLeaves pointing to all TV shows)
Input:
viewId Id of Plex library, e.g. '2'
lastViewedAt Unix timestamp; only retrieves PMS items viewed
since that point of time until now.
updatedAt Unix timestamp; only retrieves PMS items updated
by the PMS since that point of time until now.
If lastViewedAt and updatedAt=None, ALL PMS items are returned.
Warning: lastViewedAt and updatedAt are combined with AND by the PMS!
Relevant "master time": PMS server. I guess this COULD lead to problems,
e.g. when server and client are in different time zones.
"""
args = []
url = "{server}/library/sections/%s/allLeaves" % viewId
if lastViewedAt:
args.append('lastViewedAt>=%s' % lastViewedAt)
if updatedAt:
args.append('updatedAt>=%s' % updatedAt)
if args:
url += '?' + '&'.join(args)
return DownloadChunks(url)
def GetPlexOnDeck(viewId):
"""
"""