From d8378584a2134260dbe3151513e9e93ae019fafb Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Mon, 14 Mar 2016 14:51:49 +0100 Subject: [PATCH] Limit max number of items requested from PMS Otherwise, slow NAS PMS may lead to timeouts - and thus empty libraries --- resources/language/English/strings.xml | 2 +- resources/language/German/strings.xml | 2 + resources/lib/PlexFunctions.py | 120 ++++++++++++++++--------- resources/lib/librarysync.py | 31 ++++--- resources/settings.xml | 2 +- 5 files changed, 105 insertions(+), 52 deletions(-) diff --git a/resources/language/English/strings.xml b/resources/language/English/strings.xml index 9d6697c2..d7e81a30 100644 --- a/resources/language/English/strings.xml +++ b/resources/language/English/strings.xml @@ -277,7 +277,7 @@ Force artwork caching Limit artwork cache threads (recommended for rpi) Enable fast startup (requires server plugin) - Maximum items to request from the server at once + Maximum items to request from the server at once (restart!) Playback [COLOR yellow]Enter network credentials[/COLOR] Enable Plex Trailers (Plexpass is needed) diff --git a/resources/language/German/strings.xml b/resources/language/German/strings.xml index c2f49228..352beb52 100644 --- a/resources/language/German/strings.xml +++ b/resources/language/German/strings.xml @@ -252,6 +252,8 @@ Unterdrücke Server-Verbindungsmeldungen beim Starten Benutze lokale Pfade anstelle von Addon-Umleitungen beim Abspielen + Max. Anzahl gleichzeitig nachgefragter PMS Einträge (Neustart!) + Plex Media Server Authorisierung ist zu häufig fehlgeschlagen. In den Einstellungen können die Anzahl erfolgloser Versuche zurückgesetzt werden. diff --git a/resources/lib/PlexFunctions.py b/resources/lib/PlexFunctions.py index d8e7e835..7d05172d 100644 --- a/resources/lib/PlexFunctions.py +++ b/resources/lib/PlexFunctions.py @@ -3,6 +3,7 @@ from urllib import urlencode from ast import literal_eval from urlparse import urlparse, parse_qs import re +from copy import deepcopy from xbmcaddon import Addon @@ -184,7 +185,7 @@ def GetPlexMetadata(key): return xml -def GetAllPlexChildren(key): +def GetAllPlexChildren(key, containerSize=None): """ Returns a list (raw xml API dump) of all Plex children for the key. (e.g. /library/metadata/194853/children pointing to a season) @@ -192,18 +193,11 @@ def GetAllPlexChildren(key): Input: key Key to a Plex item, e.g. 12345 """ - xml = downloadutils.DownloadUtils().downloadUrl( - "{server}/library/metadata/%s/children" % key) - try: - xml.attrib - except AttributeError: - logMsg( - title, "Error retrieving all children for Plex item %s" % key, -1) - xml = None - return xml + url = "{server}/library/metadata/%s/children?" % key + return DownloadChunks(url, containerSize) -def GetPlexSectionResults(viewId, args=None): +def GetPlexSectionResults(viewId, args=None, containerSize=None): """ Returns a list (XML API dump) of all Plex items in the Plex section with key = viewId. @@ -213,26 +207,79 @@ def GetPlexSectionResults(viewId, args=None): Returns None if something went wrong """ - result = [] - url = "{server}/library/sections/%s/all" % viewId + url = "{server}/library/sections/%s/all?" % viewId if args: - url += "?" + urlencode(args) - - result = downloadutils.DownloadUtils().downloadUrl(url) - - try: - result.tag - # Nope, not an XML, abort - except AttributeError: - logMsg(title, - "Error retrieving all items for Plex section %s" - % viewId, -1) - result = None - - return result + url += urlencode(args) + '&' + return DownloadChunks(url, containerSize) -def GetAllPlexLeaves(viewId, lastViewedAt=None, updatedAt=None): +def DownloadChunks(url, containerSize): + """ + Downloads PMS url in chunks of containerSize (int). + If containerSize is None: ONE xml is fetched directly + + url MUST end with '?' (if no other url encoded args are present) or '&' + + Returns a stitched-together xml or None. + """ + if containerSize is None: + # Get rid of '?' or '&' at the end of url + xml = downloadutils.DownloadUtils().downloadUrl(url[:-1]) + try: + xml.attrib + except AttributeError: + # Nope, not an XML, abort + logMsg(title, "Error getting url %s" % url[:-1], -1) + return None + else: + return xml + + xml = None + pos = 0 + errorCounter = 0 + logMsg(title, 'Downloading allLeaves of %s in chunks of %s' + % (url, str(containerSize)), 1) + while errorCounter < 10: + args = { + 'X-Plex-Container-Size': containerSize, + 'X-Plex-Container-Start': pos + } + xmlpart = downloadutils.DownloadUtils().downloadUrl( + url + urlencode(args)) + # If something went wrong - skip in the hope that it works next time + try: + xmlpart.attrib + except AttributeError: + logMsg(title, 'Error while downloading chunks: %s' + % (url + urlencode(args)), -1) + pos += containerSize + errorCounter += 1 + continue + + # Very first run: starting xml (to retain data in xml's root!) + if xml is None: + xml = deepcopy(xmlpart) + if len(xmlpart) < containerSize: + break + else: + pos += containerSize + continue + # Build answer xml - containing the entire library + for child in xmlpart: + xml.append(child) + # Done as soon as we don't receive a full complement of items + if len(xmlpart) < containerSize: + break + pos += containerSize + if errorCounter == 10: + logMsg(title, 'Fatal error while downloading chunks for %s' % url, -1) + return None + logMsg(title, 'Done downloading chunks', 1) + return xml + + +def GetAllPlexLeaves(viewId, lastViewedAt=None, updatedAt=None, + containerSize=None): """ Returns a list (raw XML API dump) of all Plex subitems for the key. (e.g. /library/sections/2/allLeaves pointing to all TV shows) @@ -243,6 +290,7 @@ def GetAllPlexLeaves(viewId, lastViewedAt=None, updatedAt=None): since that point of time until now. updatedAt Unix timestamp; only retrieves PMS items updated by the PMS since that point of time until now. + containerSize Number of items simultaneously fetched from PMS If lastViewedAt and updatedAt=None, ALL PMS items are returned. @@ -259,19 +307,11 @@ def GetAllPlexLeaves(viewId, lastViewedAt=None, updatedAt=None): if updatedAt: args.append('updatedAt>=%s' % updatedAt) if args: - url += '?' + '&'.join(args) + url += '?' + '&'.join(args) + '&' + else: + url += '?' - xml = downloadutils.DownloadUtils().downloadUrl(url) - - try: - xml.attrib - # Nope, not an XML, abort - except AttributeError: - logMsg(title, - "Error retrieving all leaves for Plex section %s" - % viewId, -1) - xml = None - return xml + return DownloadChunks(url, containerSize) def GetPlexCollections(mediatype): diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 5e030133..e1c3b1e1 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -237,6 +237,7 @@ class LibrarySync(Thread): else False self.enableBackgroundSync = True if utils.settings( 'enableBackgroundSync') == "true" else False + self.limitindex = int(utils.settings('limitindex')) # Time offset between Kodi and PMS in seconds (=Koditime - PMStime) self.timeoffset = 0 @@ -340,7 +341,8 @@ class LibrarySync(Thread): xbmc.sleep(2000) # Get all PMS items to find the item we changed items = PlexFunctions.GetAllPlexLeaves(libraryId, - lastViewedAt=timestamp) + lastViewedAt=timestamp, + containerSize=self.limitindex) # Toggle watched state back PlexFunctions.scrobble(plexId, 'unwatched') # Get server timestamp for this change @@ -482,7 +484,9 @@ class LibrarySync(Thread): self.updatelist = [] # Get items per view items = PlexFunctions.GetAllPlexLeaves( - view['id'], updatedAt=self.getPMSfromKodiTime(lastSync)) + view['id'], + updatedAt=self.getPMSfromKodiTime(lastSync), + containerSize=self.limitindex) # Just skip if something went wrong if not items: continue @@ -516,7 +520,9 @@ class LibrarySync(Thread): songupdate = False for view in self.views: items = PlexFunctions.GetAllPlexLeaves( - view['id'], lastViewedAt=self.getPMSfromKodiTime(lastSync)) + view['id'], + lastViewedAt=self.getPMSfromKodiTime(lastSync), + containerSize=self.limitindex) for item in items: itemId = item.attrib.get('ratingKey') # Skipping items 'title=All episodes' without a 'ratingKey' @@ -1070,7 +1076,8 @@ class LibrarySync(Thread): # Get items per view viewId = view['id'] viewName = view['name'] - all_plexmovies = PlexFunctions.GetPlexSectionResults(viewId) + all_plexmovies = PlexFunctions.GetPlexSectionResults( + viewId, args=None, containerSize=self.limitindex) if not all_plexmovies: self.logMsg("Couldnt get section items, aborting for view.", 1) continue @@ -1107,7 +1114,8 @@ class LibrarySync(Thread): """ xml = PlexFunctions.GetAllPlexLeaves(viewId, lastViewedAt=lastViewedAt, - updatedAt=updatedAt) + updatedAt=updatedAt, + containerSize=self.limitindex) # Return if there are no items in PMS reply - it's faster try: xml[0].attrib @@ -1202,7 +1210,8 @@ class LibrarySync(Thread): # Get items per view viewId = view['id'] viewName = view['name'] - allPlexTvShows = PlexFunctions.GetPlexSectionResults(viewId) + allPlexTvShows = PlexFunctions.GetPlexSectionResults( + viewId, containerSize=self.limitindex) if not allPlexTvShows: self.logMsg( "Error downloading show view xml for view %s" % viewId, -1) @@ -1228,7 +1237,8 @@ class LibrarySync(Thread): if self.threadStopped(): return False # Grab all seasons to tvshow from PMS - seasons = PlexFunctions.GetAllPlexChildren(tvShowId) + seasons = PlexFunctions.GetAllPlexChildren( + tvShowId, containerSize=self.limitindex) if not seasons: self.logMsg( "Error downloading season xml for show %s" % tvShowId, -1) @@ -1252,7 +1262,8 @@ class LibrarySync(Thread): if self.threadStopped(): return False # Grab all episodes to tvshow from PMS - episodes = PlexFunctions.GetAllPlexLeaves(view['id']) + episodes = PlexFunctions.GetAllPlexLeaves( + view['id'], containerSize=self.limitindex) if not episodes: self.logMsg( "Error downloading episod xml for view %s" @@ -1265,7 +1276,7 @@ class LibrarySync(Thread): None, None) self.logMsg("Analyzed all episodes of TV show with Plex Id %s" - % tvShowId, 1) + % view['id'], 1) # Process self.updatelist self.GetAndProcessXMLs(itemType) @@ -1350,7 +1361,7 @@ class LibrarySync(Thread): viewId = view['id'] viewName = view['name'] itemsXML = PlexFunctions.GetPlexSectionResults( - viewId, args=urlArgs) + viewId, args=urlArgs, containerSize=self.limitindex) if not itemsXML: self.logMsg("Error downloading xml for view %s" % viewId, -1) diff --git a/resources/settings.xml b/resources/settings.xml index 28ac97b1..5c2ac6f8 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -49,7 +49,7 @@ - +