Troplo/PlexKodiConnect
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Code refactoring of download in chunks

Browse source
This commit is contained in:
tomkat83 2017-04-01 18:28:02 +02:00
parent a3201f8a30
commit 95fd016bd7
3 changed files with 37 additions and 69 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

57
resources/lib/PlexFunctions.py
View file

@ -1,5 +1,5 @@
# -*- coding: utf-8 -*-
import logging
from logging import getLogger
from urllib import urlencode
from ast import literal_eval
from urlparse import urlparse, parse_qsl
@ -12,7 +12,9 @@ from variables import PLEX_TO_KODI_TIMEFACTOR
###############################################################################
log = logging.getLogger("PLEX."+__name__)
log = getLogger("PLEX."+__name__)
CONTAINERSIZE = int(settings('limitindex'))
###############################################################################
@ -141,7 +143,7 @@ def GetPlexMetadata(key):
return xml
def GetAllPlexChildren(key, containerSize=None):
def GetAllPlexChildren(key):
"""
Returns a list (raw xml API dump) of all Plex children for the key.
(e.g. /library/metadata/194853/children pointing to a season)
@ -149,11 +151,10 @@ def GetAllPlexChildren(key, containerSize=None):
Input:
key Key to a Plex item, e.g. 12345
"""
url = "{server}/library/metadata/%s/children?" % key
return DownloadChunks(url, containerSize)
return DownloadChunks("{server}/library/metadata/%s/children?" % key)
def GetPlexSectionResults(viewId, args=None, containerSize=None):
def GetPlexSectionResults(viewId, args=None):
"""
Returns a list (XML API dump) of all Plex items in the Plex
section with key = viewId.
@ -166,38 +167,23 @@ def GetPlexSectionResults(viewId, args=None, containerSize=None):
url = "{server}/library/sections/%s/all?" % viewId
if args:
url += urlencode(args) + '&'
return DownloadChunks(url, containerSize)
return DownloadChunks(url)
def DownloadChunks(url, containerSize):
def DownloadChunks(url):
"""
Downloads PMS url in chunks of containerSize (int).
If containerSize is None: ONE xml is fetched directly
Downloads PMS url in chunks of CONTAINERSIZE.
url MUST end with '?' (if no other url encoded args are present) or '&'
Returns a stitched-together xml or None.
"""
if containerSize is None:
# Get rid of '?' or '&' at the end of url
xml = downloadutils.DownloadUtils().downloadUrl(url[:-1])
if xml == 401:
return 401
try:
xml.attrib
except AttributeError:
# Nope, not an XML, abort
log.error("Error getting url %s" % url[:-1])
return None
else:
return xml
xml = None
pos = 0
errorCounter = 0
while errorCounter < 10:
args = {
'X-Plex-Container-Size': containerSize,
'X-Plex-Container-Size': CONTAINERSIZE,
'X-Plex-Container-Start': pos
}
xmlpart = downloadutils.DownloadUtils().downloadUrl(
@ -208,33 +194,32 @@ def DownloadChunks(url, containerSize):
except AttributeError:
log.error('Error while downloading chunks: %s'
% (url + urlencode(args)))
pos += containerSize
pos += CONTAINERSIZE
errorCounter += 1
continue
# Very first run: starting xml (to retain data in xml's root!)
if xml is None:
xml = deepcopy(xmlpart)
if len(xmlpart) < containerSize:
if len(xmlpart) < CONTAINERSIZE:
break
else:
pos += containerSize
pos += CONTAINERSIZE
continue
# Build answer xml - containing the entire library
for child in xmlpart:
xml.append(child)
# Done as soon as we don't receive a full complement of items
if len(xmlpart) < containerSize:
if len(xmlpart) < CONTAINERSIZE:
break
pos += containerSize
pos += CONTAINERSIZE
if errorCounter == 10:
log.error('Fatal error while downloading chunks for %s' % url)
return None
return xml
def GetAllPlexLeaves(viewId, lastViewedAt=None, updatedAt=None,
containerSize=None):
def GetAllPlexLeaves(viewId, lastViewedAt=None, updatedAt=None):
"""
Returns a list (raw XML API dump) of all Plex subitems for the key.
(e.g. /library/sections/2/allLeaves pointing to all TV shows)
@ -245,7 +230,6 @@ def GetAllPlexLeaves(viewId, lastViewedAt=None, updatedAt=None,
since that point of time until now.
updatedAt Unix timestamp; only retrieves PMS items updated
by the PMS since that point of time until now.
containerSize Number of items simultaneously fetched from PMS
If lastViewedAt and updatedAt=None, ALL PMS items are returned.
@ -265,14 +249,13 @@ def GetAllPlexLeaves(viewId, lastViewedAt=None, updatedAt=None,
url += '?' + '&'.join(args) + '&'
else:
url += '?'
return DownloadChunks(url, containerSize)
return DownloadChunks(url)
def GetPlexOnDeck(viewId, containerSize=None):
def GetPlexOnDeck(viewId):
"""
"""
url = "{server}/library/sections/%s/onDeck?" % viewId
return DownloadChunks(url, containerSize)
return DownloadChunks("{server}/library/sections/%s/onDeck?" % viewId)
def GetPlexCollections(mediatype):

4
resources/lib/entrypoint.py
View file

@ -791,9 +791,7 @@ def browse_plex(key=None, plex_section_id=None):
if key:
xml = downloadutils.DownloadUtils().downloadUrl('{server}%s' % key)
else:
xml = GetPlexSectionResults(
plex_section_id,
containerSize=int(settings('limitindex')))
xml = GetPlexSectionResults(plex_section_id)
try:
xml[0].attrib
except (ValueError, AttributeError, IndexError, TypeError):

45
resources/lib/librarysync.py
View file

@ -346,7 +346,6 @@ class LibrarySync(Thread):
self.enableMusic = settings('enableMusic') == "true"
self.enableBackgroundSync = settings(
'enableBackgroundSync') == "true"
self.limitindex = int(settings('limitindex'))
# Init for replacing paths
window('remapSMB', value=settings('remapSMB'))
@ -422,8 +421,7 @@ class LibrarySync(Thread):
if not view.attrib['type'] == mediatype:
continue
libraryId = view.attrib['key']
items = GetAllPlexLeaves(libraryId,
containerSize=self.limitindex)
items = GetAllPlexLeaves(libraryId)
if items in (None, 401):
log.error("Could not download section %s"
% view.attrib['key'])
@ -468,9 +466,7 @@ class LibrarySync(Thread):
# Let the PMS process this first!
xbmc.sleep(1000)
# Get PMS items to find the item we just changed
items = GetAllPlexLeaves(libraryId,
lastViewedAt=timestamp,
containerSize=self.limitindex)
items = GetAllPlexLeaves(libraryId, lastViewedAt=timestamp)
# Toggle watched state back
scrobble(plexId, 'unwatched')
if items in (None, 401):
@ -1083,8 +1079,7 @@ class LibrarySync(Thread):
# Get items per view
viewId = view['id']
viewName = view['name']
all_plexmovies = GetPlexSectionResults(
viewId, args=None, containerSize=self.limitindex)
all_plexmovies = GetPlexSectionResults(viewId, args=None)
if all_plexmovies is None:
log.info("Couldnt get section items, aborting for view.")
continue
@ -1127,8 +1122,7 @@ class LibrarySync(Thread):
return
xml = GetAllPlexLeaves(viewId,
lastViewedAt=lastViewedAt,
updatedAt=updatedAt,
containerSize=self.limitindex)
updatedAt=updatedAt)
# Return if there are no items in PMS reply - it's faster
try:
xml[0].attrib
@ -1178,8 +1172,7 @@ class LibrarySync(Thread):
# Get items per view
viewId = view['id']
viewName = view['name']
allPlexTvShows = GetPlexSectionResults(
viewId, containerSize=self.limitindex)
allPlexTvShows = GetPlexSectionResults(viewId)
if allPlexTvShows is None:
log.error("Error downloading show xml for view %s" % viewId)
continue
@ -1206,8 +1199,7 @@ class LibrarySync(Thread):
if self.threadStopped():
return False
# Grab all seasons to tvshow from PMS
seasons = GetAllPlexChildren(
tvShowId, containerSize=self.limitindex)
seasons = GetAllPlexChildren(tvShowId)
if seasons is None:
log.error("Error download season xml for show %s" % tvShowId)
continue
@ -1232,8 +1224,7 @@ class LibrarySync(Thread):
if self.threadStopped():
return False
# Grab all episodes to tvshow from PMS
episodes = GetAllPlexLeaves(
view['id'], containerSize=self.limitindex)
episodes = GetAllPlexLeaves(view['id'])
if episodes is None:
log.error("Error downloading episod xml for view %s"
% view.get('name'))
@ -1297,12 +1288,17 @@ class LibrarySync(Thread):
}
# Process artist, then album and tracks last to minimize overhead
# Each album needs to be processed directly with its songs
# Remaining songs without album will be processed last
for kind in (v.PLEX_TYPE_ARTIST,
v.PLEX_TYPE_ALBUM,
v.PLEX_TYPE_SONG):
if self.threadStopped():
return False
log.debug("Start processing music %s" % kind)
self.allKodiElementsId = {}
self.allPlexElementsId = {}
self.updatelist = []
if self.ProcessMusic(views,
kind,
urlArgs[kind],
@ -1326,10 +1322,6 @@ class LibrarySync(Thread):
return True
def ProcessMusic(self, views, kind, urlArgs, method):
self.allKodiElementsId = {}
self.allPlexElementsId = {}
self.updatelist = []
# Get a list of items already existing in Kodi db
if self.compare:
with plexdb.Get_Plex_DB() as plex_db:
@ -1340,17 +1332,13 @@ class LibrarySync(Thread):
# Yet empty/nothing yet synched
except ValueError:
pass
for view in views:
if self.threadStopped():
return False
# Get items per view
viewId = view['id']
viewName = view['name']
itemsXML = GetPlexSectionResults(
viewId, args=urlArgs, containerSize=self.limitindex)
itemsXML = GetPlexSectionResults(view['id'], args=urlArgs)
if itemsXML is None:
log.error("Error downloading xml for view %s" % viewId)
log.error("Error downloading xml for view %s" % view['id'])
continue
elif itemsXML == 401:
return False
@ -1358,9 +1346,8 @@ class LibrarySync(Thread):
self.GetUpdatelist(itemsXML,
'Music',
method,
viewName,
viewId)
view['name'],
view['id'])
if self.compare:
# Manual sync, process deletes
with itemtypes.Music() as Music: