Troplo/PlexKodiConnect
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Code refactoring of download in chunks

Browse source
This commit is contained in:
tomkat83 2017-04-01 18:28:02 +02:00
parent a3201f8a30
commit 95fd016bd7
3 changed files with 37 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
resources/lib/PlexFunctions.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*- # -*- coding: utf-8 -*-
import logging from logging import getLogger
from urllib import urlencode from urllib import urlencode
from ast import literal_eval from ast import literal_eval
from urlparse import urlparse, parse_qsl from urlparse import urlparse, parse_qsl
@ -12,7 +12,9 @@ from variables import PLEX_TO_KODI_TIMEFACTOR
############################################################################### ###############################################################################
log = logging.getLogger("PLEX."+__name__) log = getLogger("PLEX."+__name__)
CONTAINERSIZE = int(settings('limitindex'))
############################################################################### ###############################################################################
@ -141,7 +143,7 @@ def GetPlexMetadata(key):
return xml return xml
def GetAllPlexChildren(key, containerSize=None): def GetAllPlexChildren(key):
""" """
Returns a list (raw xml API dump) of all Plex children for the key. Returns a list (raw xml API dump) of all Plex children for the key.
(e.g. /library/metadata/194853/children pointing to a season) (e.g. /library/metadata/194853/children pointing to a season)
@ -149,11 +151,10 @@ def GetAllPlexChildren(key, containerSize=None):
Input: Input:
key Key to a Plex item, e.g. 12345 key Key to a Plex item, e.g. 12345
""" """
url = "{server}/library/metadata/%s/children?" % key return DownloadChunks("{server}/library/metadata/%s/children?" % key)
return DownloadChunks(url, containerSize)
def GetPlexSectionResults(viewId, args=None, containerSize=None): def GetPlexSectionResults(viewId, args=None):
""" """
Returns a list (XML API dump) of all Plex items in the Plex Returns a list (XML API dump) of all Plex items in the Plex
section with key = viewId. section with key = viewId.
@ -166,38 +167,23 @@ def GetPlexSectionResults(viewId, args=None, containerSize=None):
url = "{server}/library/sections/%s/all?" % viewId url = "{server}/library/sections/%s/all?" % viewId
if args: if args:
url += urlencode(args) + '&' url += urlencode(args) + '&'
return DownloadChunks(url, containerSize) return DownloadChunks(url)
def DownloadChunks(url, containerSize): def DownloadChunks(url):
""" """
Downloads PMS url in chunks of containerSize (int). Downloads PMS url in chunks of CONTAINERSIZE.
If containerSize is None: ONE xml is fetched directly
url MUST end with '?' (if no other url encoded args are present) or '&' url MUST end with '?' (if no other url encoded args are present) or '&'
Returns a stitched-together xml or None. Returns a stitched-together xml or None.
""" """
if containerSize is None:
# Get rid of '?' or '&' at the end of url
xml = downloadutils.DownloadUtils().downloadUrl(url[:-1])
if xml == 401:
return 401
try:
xml.attrib
except AttributeError:
# Nope, not an XML, abort
log.error("Error getting url %s" % url[:-1])
return None
else:
return xml
xml = None xml = None
pos = 0 pos = 0
errorCounter = 0 errorCounter = 0
while errorCounter < 10: while errorCounter < 10:
args = { args = {
'X-Plex-Container-Size': containerSize, 'X-Plex-Container-Size': CONTAINERSIZE,
'X-Plex-Container-Start': pos 'X-Plex-Container-Start': pos
} }
xmlpart = downloadutils.DownloadUtils().downloadUrl( xmlpart = downloadutils.DownloadUtils().downloadUrl(
@ -208,33 +194,32 @@ def DownloadChunks(url, containerSize):
except AttributeError: except AttributeError:
log.error('Error while downloading chunks: %s' log.error('Error while downloading chunks: %s'
% (url + urlencode(args))) % (url + urlencode(args)))
pos += containerSize pos += CONTAINERSIZE
errorCounter += 1 errorCounter += 1
continue continue
# Very first run: starting xml (to retain data in xml's root!) # Very first run: starting xml (to retain data in xml's root!)
if xml is None: if xml is None:
xml = deepcopy(xmlpart) xml = deepcopy(xmlpart)
if len(xmlpart) < containerSize: if len(xmlpart) < CONTAINERSIZE:
break break
else: else:
pos += containerSize pos += CONTAINERSIZE
continue continue
# Build answer xml - containing the entire library # Build answer xml - containing the entire library
for child in xmlpart: for child in xmlpart:
xml.append(child) xml.append(child)
# Done as soon as we don't receive a full complement of items # Done as soon as we don't receive a full complement of items
if len(xmlpart) < containerSize: if len(xmlpart) < CONTAINERSIZE:
break break
pos += containerSize pos += CONTAINERSIZE
if errorCounter == 10: if errorCounter == 10:
log.error('Fatal error while downloading chunks for %s' % url) log.error('Fatal error while downloading chunks for %s' % url)
return None return None
return xml return xml
def GetAllPlexLeaves(viewId, lastViewedAt=None, updatedAt=None, def GetAllPlexLeaves(viewId, lastViewedAt=None, updatedAt=None):
containerSize=None):
""" """
Returns a list (raw XML API dump) of all Plex subitems for the key. Returns a list (raw XML API dump) of all Plex subitems for the key.
(e.g. /library/sections/2/allLeaves pointing to all TV shows) (e.g. /library/sections/2/allLeaves pointing to all TV shows)
@ -245,7 +230,6 @@ def GetAllPlexLeaves(viewId, lastViewedAt=None, updatedAt=None,
since that point of time until now. since that point of time until now.
updatedAt Unix timestamp; only retrieves PMS items updated updatedAt Unix timestamp; only retrieves PMS items updated
by the PMS since that point of time until now. by the PMS since that point of time until now.
containerSize Number of items simultaneously fetched from PMS
If lastViewedAt and updatedAt=None, ALL PMS items are returned. If lastViewedAt and updatedAt=None, ALL PMS items are returned.
@ -265,14 +249,13 @@ def GetAllPlexLeaves(viewId, lastViewedAt=None, updatedAt=None,
url += '?' + '&'.join(args) + '&' url += '?' + '&'.join(args) + '&'
else: else:
url += '?' url += '?'
return DownloadChunks(url, containerSize) return DownloadChunks(url)
def GetPlexOnDeck(viewId, containerSize=None): def GetPlexOnDeck(viewId):
""" """
""" """
url = "{server}/library/sections/%s/onDeck?" % viewId return DownloadChunks("{server}/library/sections/%s/onDeck?" % viewId)
return DownloadChunks(url, containerSize)
def GetPlexCollections(mediatype): def GetPlexCollections(mediatype):

4
resources/lib/entrypoint.py
View file

@ -791,9 +791,7 @@ def browse_plex(key=None, plex_section_id=None):
if key: if key:
xml = downloadutils.DownloadUtils().downloadUrl('{server}%s' % key) xml = downloadutils.DownloadUtils().downloadUrl('{server}%s' % key)
else: else:
xml = GetPlexSectionResults( xml = GetPlexSectionResults(plex_section_id)
plex_section_id,
containerSize=int(settings('limitindex')))
try: try:
xml[0].attrib xml[0].attrib
except (ValueError, AttributeError, IndexError, TypeError): except (ValueError, AttributeError, IndexError, TypeError):

45
resources/lib/librarysync.py
View file

@ -346,7 +346,6 @@ class LibrarySync(Thread):
self.enableMusic = settings('enableMusic') == "true" self.enableMusic = settings('enableMusic') == "true"
self.enableBackgroundSync = settings( self.enableBackgroundSync = settings(
'enableBackgroundSync') == "true" 'enableBackgroundSync') == "true"
self.limitindex = int(settings('limitindex'))
# Init for replacing paths # Init for replacing paths
window('remapSMB', value=settings('remapSMB')) window('remapSMB', value=settings('remapSMB'))
@ -422,8 +421,7 @@ class LibrarySync(Thread):
if not view.attrib['type'] == mediatype: if not view.attrib['type'] == mediatype:
continue continue
libraryId = view.attrib['key'] libraryId = view.attrib['key']
items = GetAllPlexLeaves(libraryId, items = GetAllPlexLeaves(libraryId)
containerSize=self.limitindex)
if items in (None, 401): if items in (None, 401):
log.error("Could not download section %s" log.error("Could not download section %s"
% view.attrib['key']) % view.attrib['key'])
@ -468,9 +466,7 @@ class LibrarySync(Thread):
# Let the PMS process this first! # Let the PMS process this first!
xbmc.sleep(1000) xbmc.sleep(1000)
# Get PMS items to find the item we just changed # Get PMS items to find the item we just changed
items = GetAllPlexLeaves(libraryId, items = GetAllPlexLeaves(libraryId, lastViewedAt=timestamp)
lastViewedAt=timestamp,
containerSize=self.limitindex)
# Toggle watched state back # Toggle watched state back
scrobble(plexId, 'unwatched') scrobble(plexId, 'unwatched')
if items in (None, 401): if items in (None, 401):
@ -1083,8 +1079,7 @@ class LibrarySync(Thread):
# Get items per view # Get items per view
viewId = view['id'] viewId = view['id']
viewName = view['name'] viewName = view['name']
all_plexmovies = GetPlexSectionResults( all_plexmovies = GetPlexSectionResults(viewId, args=None)
viewId, args=None, containerSize=self.limitindex)
if all_plexmovies is None: if all_plexmovies is None:
log.info("Couldnt get section items, aborting for view.") log.info("Couldnt get section items, aborting for view.")
continue continue
@ -1127,8 +1122,7 @@ class LibrarySync(Thread):
return return
xml = GetAllPlexLeaves(viewId, xml = GetAllPlexLeaves(viewId,
lastViewedAt=lastViewedAt, lastViewedAt=lastViewedAt,
updatedAt=updatedAt, updatedAt=updatedAt)
containerSize=self.limitindex)
# Return if there are no items in PMS reply - it's faster # Return if there are no items in PMS reply - it's faster
try: try:
xml[0].attrib xml[0].attrib
@ -1178,8 +1172,7 @@ class LibrarySync(Thread):
# Get items per view # Get items per view
viewId = view['id'] viewId = view['id']
viewName = view['name'] viewName = view['name']
allPlexTvShows = GetPlexSectionResults( allPlexTvShows = GetPlexSectionResults(viewId)
viewId, containerSize=self.limitindex)
if allPlexTvShows is None: if allPlexTvShows is None:
log.error("Error downloading show xml for view %s" % viewId) log.error("Error downloading show xml for view %s" % viewId)
continue continue
@ -1206,8 +1199,7 @@ class LibrarySync(Thread):
if self.threadStopped(): if self.threadStopped():
return False return False
# Grab all seasons to tvshow from PMS # Grab all seasons to tvshow from PMS
seasons = GetAllPlexChildren( seasons = GetAllPlexChildren(tvShowId)
tvShowId, containerSize=self.limitindex)
if seasons is None: if seasons is None:
log.error("Error download season xml for show %s" % tvShowId) log.error("Error download season xml for show %s" % tvShowId)
continue continue
@ -1232,8 +1224,7 @@ class LibrarySync(Thread):
if self.threadStopped(): if self.threadStopped():
return False return False
# Grab all episodes to tvshow from PMS # Grab all episodes to tvshow from PMS
episodes = GetAllPlexLeaves( episodes = GetAllPlexLeaves(view['id'])
view['id'], containerSize=self.limitindex)
if episodes is None: if episodes is None:
log.error("Error downloading episod xml for view %s" log.error("Error downloading episod xml for view %s"
% view.get('name')) % view.get('name'))
@ -1297,12 +1288,17 @@ class LibrarySync(Thread):
} }
# Process artist, then album and tracks last to minimize overhead # Process artist, then album and tracks last to minimize overhead
# Each album needs to be processed directly with its songs
# Remaining songs without album will be processed last
for kind in (v.PLEX_TYPE_ARTIST, for kind in (v.PLEX_TYPE_ARTIST,
v.PLEX_TYPE_ALBUM, v.PLEX_TYPE_ALBUM,
v.PLEX_TYPE_SONG): v.PLEX_TYPE_SONG):
if self.threadStopped(): if self.threadStopped():
return False return False
log.debug("Start processing music %s" % kind) log.debug("Start processing music %s" % kind)
self.allKodiElementsId = {}
self.allPlexElementsId = {}
self.updatelist = []
if self.ProcessMusic(views, if self.ProcessMusic(views,
kind, kind,
urlArgs[kind], urlArgs[kind],
@ -1326,10 +1322,6 @@ class LibrarySync(Thread):
return True return True
def ProcessMusic(self, views, kind, urlArgs, method): def ProcessMusic(self, views, kind, urlArgs, method):
self.allKodiElementsId = {}
self.allPlexElementsId = {}
self.updatelist = []
# Get a list of items already existing in Kodi db # Get a list of items already existing in Kodi db
if self.compare: if self.compare:
with plexdb.Get_Plex_DB() as plex_db: with plexdb.Get_Plex_DB() as plex_db:
@ -1340,17 +1332,13 @@ class LibrarySync(Thread):
# Yet empty/nothing yet synched # Yet empty/nothing yet synched
except ValueError: except ValueError:
pass pass
for view in views: for view in views:
if self.threadStopped(): if self.threadStopped():
return False return False
# Get items per view # Get items per view
viewId = view['id'] itemsXML = GetPlexSectionResults(view['id'], args=urlArgs)
viewName = view['name']
itemsXML = GetPlexSectionResults(
viewId, args=urlArgs, containerSize=self.limitindex)
if itemsXML is None: if itemsXML is None:
log.error("Error downloading xml for view %s" % viewId) log.error("Error downloading xml for view %s" % view['id'])
continue continue
elif itemsXML == 401: elif itemsXML == 401:
return False return False
@ -1358,9 +1346,8 @@ class LibrarySync(Thread):
self.GetUpdatelist(itemsXML, self.GetUpdatelist(itemsXML,
'Music', 'Music',
method, method,
viewName, view['name'],
viewId) view['id'])
if self.compare: if self.compare:
# Manual sync, process deletes # Manual sync, process deletes
with itemtypes.Music() as Music: with itemtypes.Music() as Music: