Troplo/PlexKodiConnect
1
0
Fork 0
Code Issues Pull requests Projects Releases Wiki Activity

Sped up initial sync

Browse source
This commit is contained in:
tomkat83 2016-01-11 07:55:22 +01:00
parent 9f3db90a7c
commit de3a058463
2 changed files with 134 additions and 111 deletions
Show stats Download patch file Download diff file Expand all files Collapse all files

10
resources/lib/PlexAPI.py
View file

@ -1599,13 +1599,14 @@ class API():
provider = regex.findall(item['guid']) provider = regex.findall(item['guid'])
try: try:
provider = provider[0] provider = provider[0]
except: except KeyError:
provider = None provider = None
return provider return provider
def getTitle(self): def getTitle(self):
""" """
Returns an item's name/title or "Missing Title Name" Returns an item's name/title or "Missing Title Name" for both XML and
JSON PMS replies
Output: Output:
title, sorttitle title, sorttitle
@ -1613,7 +1614,12 @@ class API():
sorttitle = title, if no sorttitle is found sorttitle = title, if no sorttitle is found
""" """
item = self.item item = self.item
# XML
try:
item = item[self.child].attrib item = item[self.child].attrib
# JSON
except KeyError:
pass
try: try:
title = item['title'] title = item['title']
except: except:

213
resources/lib/librarysync.py
View file

@ -55,14 +55,15 @@ class ThreadedGetMetadata(threading.Thread):
while self.stopped() is False: while self.stopped() is False:
# grabs Plex item from queue # grabs Plex item from queue
try: try:
itemId = self.queue.get(block=False) updateItem = self.queue.get(block=False)
# Empty queue # Empty queue
except: except:
continue continue
# Download Metadata # Download Metadata
plexElement = plx.GetPlexMetadata(itemId) plexElement = plx.GetPlexMetadata(updateItem['itemId'])
updateItem['XML'] = plexElement
# place item into out queue # place item into out queue
self.out_queue.put(plexElement) self.out_queue.put(updateItem)
# Keep track of where we are at # Keep track of where we are at
with self.lock: with self.lock:
getMetadataCount += 1 getMetadataCount += 1
@ -84,41 +85,40 @@ class ThreadedProcessMetadata(threading.Thread):
Input: Input:
queue: Queue.Queue() object that you'll need to fill up with queue: Queue.Queue() object that you'll need to fill up with
the downloaded XML eTree objects the downloaded XML eTree objects
data = { itemType: as used to call functions in itemtypes.py
'itemType': as used to call functions in itemtypes.py
e.g. 'Movies' => itemtypes.Movies() e.g. 'Movies' => itemtypes.Movies()
'method' Name of method in itemtypes.py
e.g. 'add_updateSeason'
'viewName' Plex str for library view (e.g. 'Movies')
'viewId' Plex Id to identifiy the library view
}
lock: threading.Lock(), used for counting where we are lock: threading.Lock(), used for counting where we are
userStop Handle to a function where True is used to stop this Thread userStop Handle to a function where True is used to stop this Thread
""" """
def __init__(self, queue, data, lock, userStop): def __init__(self, queue, itemType, lock, userStop):
self.queue = queue self.queue = queue
self.lock = lock self.lock = lock
self.data = data self.itemType = itemType
self.userStop = userStop self.userStop = userStop
self._shouldstop = threading.Event() self._shouldstop = threading.Event()
threading.Thread.__init__(self) threading.Thread.__init__(self)
def run(self): def run(self):
# Constructs the method name, e.g. itemtypes.Movies # Constructs the method name, e.g. itemtypes.Movies
itemFkt = getattr(itemtypes, self.data['itemType']) itemFkt = getattr(itemtypes, self.itemType)
viewName = self.data['viewName']
viewId = self.data['viewId']
global processMetadataCount global processMetadataCount
global processingViewName
with itemFkt() as item: with itemFkt() as item:
itemSubFkt = getattr(item, self.data['method'])
while self.stopped() is False: while self.stopped() is False:
# grabs item from queue # grabs item from queue
try: try:
plexitem = self.queue.get(block=False) updateItem = self.queue.get(block=False)
# Empty queue # Empty queue
except: except:
continue continue
# Do the work; lock to be sure we've only got 1 Thread # Do the work; lock to be sure we've only got 1 Thread
plexitem = updateItem['XML']
method = updateItem['method']
viewName = updateItem['viewName']
viewId = updateItem['viewId']
title = updateItem['title']
itemSubFkt = getattr(item, method)
with self.lock: with self.lock:
itemSubFkt( itemSubFkt(
plexitem, plexitem,
@ -127,6 +127,7 @@ class ThreadedProcessMetadata(threading.Thread):
) )
# Keep track of where we are at # Keep track of where we are at
processMetadataCount += 1 processMetadataCount += 1
processingViewName = title
# signals to queue job is done # signals to queue job is done
self.queue.task_done() self.queue.task_done()
@ -145,16 +146,16 @@ class ThreadedShowSyncInfo(threading.Thread):
dialog xbmcgui.DialogProgressBG() object to show progress dialog xbmcgui.DialogProgressBG() object to show progress
locks = [downloadLock, processLock] Locks() to the other threads locks = [downloadLock, processLock] Locks() to the other threads
total: Total number of items to get total: Total number of items to get
viewName: Name of library we're getting userStop: function handle to stop thread
""" """
def __init__(self, dialog, locks, total, viewName, userStop): def __init__(self, dialog, locks, total, itemType, userStop):
self.locks = locks self.locks = locks
self.total = total self.total = total
self.viewName = viewName
self.userStop = userStop self.userStop = userStop
self.addonName = clientinfo.ClientInfo().getAddonName() self.addonName = clientinfo.ClientInfo().getAddonName()
self._shouldstop = threading.Event() self._shouldstop = threading.Event()
self.dialog = dialog self.dialog = dialog
self.itemType = itemType
threading.Thread.__init__(self) threading.Thread.__init__(self)
def run(self): def run(self):
@ -163,18 +164,20 @@ class ThreadedShowSyncInfo(threading.Thread):
processLock = self.locks[1] processLock = self.locks[1]
self.dialog.create( self.dialog.create(
"%s: Sync %s: %s items" % (self.addonName, "%s: Sync %s: %s items" % (self.addonName,
self.viewName, self.itemType,
str(total)), str(total)),
"Starting") "Starting")
global getMetadataCount global getMetadataCount
global processMetadataCount global processMetadataCount
global processingViewName
total = 2 * total total = 2 * total
totalProgress = 0 totalProgress = 0
while self.stopped() is False: while self.stopped() is False:
with downloadLock: with downloadLock:
getMetadataProgress = getMetadataCount getMetadataProgress = getMetadataCount
#with processLock: with processLock:
processMetadataProgress = processMetadataCount processMetadataProgress = processMetadataCount
viewName = processingViewName
totalProgress = getMetadataProgress + processMetadataProgress totalProgress = getMetadataProgress + processMetadataProgress
try: try:
percentage = int(float(totalProgress) / float(total)*100.0) percentage = int(float(totalProgress) / float(total)*100.0)
@ -182,10 +185,11 @@ class ThreadedShowSyncInfo(threading.Thread):
percentage = 0 percentage = 0
self.dialog.update( self.dialog.update(
percentage, percentage,
message="Downloaded: %s, Processed: %s" % message="Downloaded: %s, Processed: %s: %s" %
(getMetadataProgress, processMetadataProgress) (getMetadataProgress, processMetadataProgress, viewName)
) )
time.sleep(1) time.sleep(1)
self.dialog.close()
def stopThread(self): def stopThread(self):
self._shouldstop.set() self._shouldstop.set()
@ -573,7 +577,7 @@ class LibrarySync(threading.Thread):
# Save total # Save total
utils.window('Emby.nodes.total', str(totalnodes)) utils.window('Emby.nodes.total', str(totalnodes))
def GetUpdatelist(self, elementList): def GetUpdatelist(self, elementList, itemType, method, viewName, viewId):
""" """
Adds items to self.updatelist as well as self.allPlexElementsId dict Adds items to self.updatelist as well as self.allPlexElementsId dict
@ -584,56 +588,71 @@ class LibrarySync(threading.Thread):
Output: self.updatelist, self.allPlexElementsId Output: self.updatelist, self.allPlexElementsId
self.updatelist APPENDED(!!) list itemids (Plex Keys as self.updatelist APPENDED(!!) list itemids (Plex Keys as
as received from API.getKey()) as received from API.getKey())
= [
{
'itemId': xxx,
'itemType': 'Movies'/'TVShows'/...,
'method': 'add_update', 'add_updateSeason', ...
'viewName': xxx,
'viewId': xxx
}, ...
]
self.allPlexElementsId APPENDED(!!) dic self.allPlexElementsId APPENDED(!!) dic
= {itemid: checksum} = {itemid: checksum}
""" """
if self.compare: if self.compare:
# Manual sync # Manual sync
for item in elementList: for item in elementList:
# Skipping XML item 'title=All episodes' without ratingKey # Skipping XML item 'title=All episodes' without a 'ratingKey'
if item.get('ratingKey', False): if item.get('ratingKey', False):
if self.shouldStop(): if self.shouldStop():
return False return False
API = PlexAPI.API(item) API = PlexAPI.API(item)
plex_checksum = API.getChecksum() plex_checksum = API.getChecksum()
itemid = API.getKey() itemId = API.getKey()
self.allPlexElementsId[itemid] = plex_checksum title, sorttitle = API.getTitle()
kodi_checksum = self.allKodiElementsId.get(itemid) self.allPlexElementsId[itemId] = plex_checksum
kodi_checksum = self.allKodiElementsId.get(itemId)
if kodi_checksum != plex_checksum: if kodi_checksum != plex_checksum:
# Only update if movie is not in Kodi or checksum is # Only update if movie is not in Kodi or checksum is
# different # different
self.updatelist.append(itemid) self.updatelist.append({
'itemId': itemId,
'itemType': itemType,
'method': method,
'viewName': viewName,
'viewId': viewId,
'title': title})
else: else:
# Initial or repair sync: get all Plex movies # Initial or repair sync: get all Plex movies
for item in elementList: for item in elementList:
# Only look at valid items = Plex library items # Only look at valid items = Plex library items
if item.get('ratingKey', False): if item.get('ratingKey', False):
if self.shouldStop():
return False
API = PlexAPI.API(item) API = PlexAPI.API(item)
itemid = API.getKey() itemId = API.getKey()
plex_checksum = API.getChecksum() plex_checksum = API.getChecksum()
self.allPlexElementsId[itemid] = plex_checksum self.allPlexElementsId[itemId] = plex_checksum
self.updatelist.append(itemid) self.updatelist.append({
'itemId': itemId,
'itemType': itemType,
'method': method,
'viewName': viewName,
'viewId': viewId})
# Update the Kodi popup info # Update the Kodi popup info
return self.updatelist, self.allPlexElementsId return self.updatelist, self.allPlexElementsId
def GetAndProcessXMLs(self, itemType, viewName, viewId, method): def GetAndProcessXMLs(self, itemType):
""" """
Downloads all XMLs for itemType (e.g. Movies, TV-Shows). Processes them Downloads all XMLs for itemType (e.g. Movies, TV-Shows). Processes them
by then calling itemtypes.<itemType>() by then calling itemtypes.<itemType>()
Input: Input:
itemType: to construct itemtypes.py function name: 'Movies' self.updatelist
viewName: Plex name of library, e.g. 'My Movies'
viewId: Plex Id for that library
method: name of itemtypes.py method,
e.g. 'add_updateSeason'
Returns True if/when done
""" """
# Some logging, just in case. # Some logging, just in case.
self.logMsg("self.updatelist: %s" % self.updatelist, 2) self.logMsg("self.updatelist: %s" % self.updatelist, 2)
self.logMsg("self.allPlexElementsId: %s" % self.allPlexElementsId, 2)
self.logMsg("self.allKodiElementsId: %s" % self.allKodiElementsId, 2)
# Run through self.updatelist, get XML metadata per item # Run through self.updatelist, get XML metadata per item
# Initiate threads # Initiate threads
@ -649,36 +668,28 @@ class LibrarySync(threading.Thread):
getMetadataCount = 0 getMetadataCount = 0
global processMetadataCount global processMetadataCount
processMetadataCount = 0 processMetadataCount = 0
global processingViewName
processingViewName = ''
# Populate queue: GetMetadata # Populate queue: GetMetadata
for itemId in self.updatelist: for updateItem in self.updatelist:
getMetadataQueue.put(itemId) getMetadataQueue.put(updateItem)
# Spawn GetMetadata threads # Spawn GetMetadata threads for downloading
threads = [] threads = []
for i in range(self.syncThreadNumber): for i in range(self.syncThreadNumber):
t = ThreadedGetMetadata( thread = ThreadedGetMetadata(getMetadataQueue,
getMetadataQueue,
processMetadataQueue, processMetadataQueue,
getMetadataLock, getMetadataLock,
self.shouldStop self.shouldStop)
) thread.setDaemon(True)
t.setDaemon(True) thread.start()
t.start() threads.append(thread)
threads.append(t)
self.logMsg("%s download threads spawned" % self.syncThreadNumber, 1) self.logMsg("%s download threads spawned" % self.syncThreadNumber, 1)
self.logMsg("Queue populated", 1) self.logMsg("Queue populated", 1)
# Spawn one more thread to process Metadata, once downloaded # Spawn one more thread to process Metadata, once downloaded
data = { thread = ThreadedProcessMetadata(processMetadataQueue,
'itemType': itemType, itemType,
'method': method,
'viewName': viewName,
'viewId': viewId
}
thread = ThreadedProcessMetadata(
processMetadataQueue,
data,
processMetadataLock, processMetadataLock,
self.shouldStop self.shouldStop)
)
thread.setDaemon(True) thread.setDaemon(True)
thread.start() thread.start()
threads.append(thread) threads.append(thread)
@ -691,7 +702,7 @@ class LibrarySync(threading.Thread):
dialog, dialog,
[getMetadataLock, processMetadataLock], [getMetadataLock, processMetadataLock],
total, total,
viewName, itemType,
self.shouldStop self.shouldStop
) )
thread.setDaemon(True) thread.setDaemon(True)
@ -709,6 +720,7 @@ class LibrarySync(threading.Thread):
# Wait till threads are indeed dead # Wait till threads are indeed dead
for thread in threads: for thread in threads:
thread.join() thread.join()
if dialog:
dialog.close() dialog.close()
self.logMsg("=====================", 1) self.logMsg("=====================", 1)
self.logMsg("Sync threads finished", 1) self.logMsg("Sync threads finished", 1)
@ -718,11 +730,11 @@ class LibrarySync(threading.Thread):
def PlexMovies(self): def PlexMovies(self):
# Initialize # Initialize
plx = PlexAPI.PlexAPI() plx = PlexAPI.PlexAPI()
self.updatelist = []
self.allPlexElementsId = {} self.allPlexElementsId = {}
itemType = 'Movies'
views = plx.GetPlexCollections('movie') views = plx.GetPlexCollections('movie')
self.logMsg("Media folders: %s" % views, 1) self.logMsg("Processing Plex %s. Libraries: %s" % (itemType, views), 1)
if self.compare: if self.compare:
# Get movies from Plex server # Get movies from Plex server
@ -740,8 +752,8 @@ class LibrarySync(threading.Thread):
self.allKodiElementsId = {} self.allKodiElementsId = {}
##### PROCESS MOVIES ##### ##### PROCESS MOVIES #####
for view in views:
self.updatelist = [] self.updatelist = []
for view in views:
if self.shouldStop(): if self.shouldStop():
return False return False
# Get items per view # Get items per view
@ -749,14 +761,13 @@ class LibrarySync(threading.Thread):
viewName = view['name'] viewName = view['name']
all_plexmovies = plx.GetPlexSectionResults(viewId) all_plexmovies = plx.GetPlexSectionResults(viewId)
# Populate self.updatelist and self.allPlexElementsId # Populate self.updatelist and self.allPlexElementsId
self.GetUpdatelist(all_plexmovies) self.GetUpdatelist(all_plexmovies,
self.logMsg("Processed view %s with ID %s" % (viewName, viewId), 1) itemType,
self.GetAndProcessXMLs( 'add_update',
'Movies',
viewName, viewName,
viewId, viewId)
'add_update' self.GetAndProcessXMLs(itemType)
) self.logMsg("Processed view %s with ID %s" % (viewName, viewId), 1)
##### PROCESS DELETES ##### ##### PROCESS DELETES #####
if self.compare: if self.compare:
# Manual sync, process deletes # Manual sync, process deletes
@ -765,7 +776,7 @@ class LibrarySync(threading.Thread):
if kodimovie not in self.allPlexElementsId: if kodimovie not in self.allPlexElementsId:
Movie.remove(kodimovie) Movie.remove(kodimovie)
else: else:
self.logMsg("%s compare finished." % 'Movies', 1) self.logMsg("%s compare finished." % itemType, 1)
return True return True
def musicvideos(self, embycursor, kodicursor, pdialog, compare=False): def musicvideos(self, embycursor, kodicursor, pdialog, compare=False):
@ -959,8 +970,8 @@ class LibrarySync(threading.Thread):
def PlexTVShows(self): def PlexTVShows(self):
# Initialize # Initialize
plx = PlexAPI.PlexAPI() plx = PlexAPI.PlexAPI()
self.updatelist = []
self.allPlexElementsId = {} self.allPlexElementsId = {}
itemType = 'TVShows'
views = plx.GetPlexCollections('show') views = plx.GetPlexCollections('show')
self.logMsg("Media folders: %s" % views, 1) self.logMsg("Media folders: %s" % views, 1)
@ -986,8 +997,8 @@ class LibrarySync(threading.Thread):
embyconn.close() embyconn.close()
##### PROCESS TV Shows ##### ##### PROCESS TV Shows #####
for view in views:
self.updatelist = [] self.updatelist = []
for view in views:
if self.shouldStop(): if self.shouldStop():
return False return False
# Get items per view # Get items per view
@ -995,12 +1006,11 @@ class LibrarySync(threading.Thread):
viewName = view['name'] viewName = view['name']
allPlexTvShows = plx.GetPlexSectionResults(viewId) allPlexTvShows = plx.GetPlexSectionResults(viewId)
# Populate self.updatelist and self.allPlexElementsId # Populate self.updatelist and self.allPlexElementsId
self.GetUpdatelist(allPlexTvShows) self.GetUpdatelist(allPlexTvShows,
# Process self.updatelist itemType,
self.GetAndProcessXMLs('TVShows', 'add_update',
viewName, viewName,
viewId, viewId)
'add_update')
self.logMsg("Processed view %s with ID %s" % (viewName, viewId), 1) self.logMsg("Processed view %s with ID %s" % (viewName, viewId), 1)
# COPY for later use # COPY for later use
@ -1009,35 +1019,42 @@ class LibrarySync(threading.Thread):
##### PROCESS TV Seasons ##### ##### PROCESS TV Seasons #####
# Cycle through tv shows # Cycle through tv shows
for tvShowId in allPlexTvShowsId: for tvShowId in allPlexTvShowsId:
self.updatelist = [] if self.shouldStop():
return False
# Grab all seasons to tvshow from PMS # Grab all seasons to tvshow from PMS
seasons = plx.GetAllPlexChildren(tvShowId) seasons = plx.GetAllPlexChildren(tvShowId)
# Populate self.updatelist and self.allPlexElementsId # Populate self.updatelist and self.allPlexElementsId
self.GetUpdatelist(seasons) self.GetUpdatelist(seasons,
# Process self.updatelist itemType,
self.GetAndProcessXMLs('TVShows', 'add_updateSeason',
None, None,
tvShowId, # send showId instead of viewid tvShowId) # send showId instead of viewid
'add_updateSeason')
XMLtvshow = plx.GetPlexMetadata(tvShowId)
with itemtypes.TVShows() as TVshow:
TVshow.refreshSeasonEntry(XMLtvshow, tvShowId)
self.logMsg("Processed all seasons of TV show with Plex Id %s" % tvShowId, 1) self.logMsg("Processed all seasons of TV show with Plex Id %s" % tvShowId, 1)
##### PROCESS TV Episodes ##### ##### PROCESS TV Episodes #####
# Cycle through tv shows # Cycle through tv shows
for tvShowId in allPlexTvShowsId: for tvShowId in allPlexTvShowsId:
self.updatelist = [] if self.shouldStop():
return False
# Grab all episodes to tvshow from PMS # Grab all episodes to tvshow from PMS
episodes = plx.GetAllPlexLeaves(tvShowId) episodes = plx.GetAllPlexLeaves(tvShowId)
# Populate self.updatelist and self.allPlexElementsId # Populate self.updatelist and self.allPlexElementsId
self.GetUpdatelist(episodes) self.GetUpdatelist(episodes,
# Process self.updatelist itemType,
self.GetAndProcessXMLs('TVShows', 'add_updateEpisode',
None, None,
None, None)
'add_updateEpisode')
self.logMsg("Processed all episodes of TV show with Plex Id %s" % tvShowId, 1) self.logMsg("Processed all episodes of TV show with Plex Id %s" % tvShowId, 1)
# Process self.updatelist
self.GetAndProcessXMLs(itemType)
# Refresh season info
# Cycle through tv shows
with itemtypes.TVShows() as TVshow:
for tvShowId in allPlexTvShowsId:
XMLtvshow = plx.GetPlexMetadata(tvShowId)
TVshow.refreshSeasonEntry(XMLtvshow, tvShowId)
##### PROCESS DELETES ##### ##### PROCESS DELETES #####
if self.compare: if self.compare:
# Manual sync, process deletes # Manual sync, process deletes