Speed up sync - download art in the background

resources/lib/PlexAPI.py

@@ -1841,16 +1841,14 @@ class API():
             'Backdrop' : LIST with the first entry xml key "art"
         }
         """
-        item = self.item.attrib
-
         allartworks = {
-            'Primary': "",
+            'Primary': "",  # corresponds to Plex poster ('thumb')
             'Art': "",
-            'Banner': "",
+            'Banner': "",   # corresponds to Plex banner ('banner') for series
             'Logo': "",
-            'Thumb': "",
+            'Thumb': "",    # corresponds to Plex (grand)parent posters (thumb)
             'Disc': "",
-            'Backdrop': []
+            'Backdrop': []  # Corresponds to Plex fanart ('art')
         }
         # Process backdrops
         # Get background artwork URL
@@ -1870,11 +1868,23 @@ class API():
                     self.__getOneArtwork('parentArt'))
             if not allartworks['Primary']:
                 allartworks['Primary'] = self.__getOneArtwork('parentThumb')
+        return allartworks
 
-        # Plex does not get much artwork - go ahead and get the rest from
+    def getFanartArtwork(self, allartworks, parentInfo=False):
-        # fanart tv only for movie or tv show
+        """
-        if settings('FanartTV') == 'true':
+        Downloads additional fanart from third party sources (well, link to
-            if item.get('type') in ('movie', 'show'):
+        fanart only).
+
+        allartworks = {
+            'Primary': "",
+            'Art': "",
+            'Banner': "",
+            'Logo': "",
+            'Thumb': "",
+            'Disc': "",
+            'Backdrop': []
+        }
+        """
         externalId = self.getExternalItemId()
         if externalId is not None:
             allartworks = self.getFanartTVArt(externalId, allartworks)

resources/lib/embydb_functions.py

@@ -373,3 +373,30 @@ class Embydb_Functions():
         query = "DELETE FROM emby WHERE emby_id LIKE ?"
         self.embycursor.execute(query, (plexid+"%",))
 
+    def itemsByType(self, plextype):
+        """
+        Returns a list of dictionaries for all Kodi DB items present for
+        plextype. One dict is of the type
+
+        {
+            'plexId':       the Plex id
+            'kodiId':       the Kodi id
+            'kodi_type':    e.g. 'movie', 'tvshow'
+            'plex_type':    e.g. 'Movie', 'Series', the input plextype
+        }
+        """
+        query = ' '.join((
+            "SELECT emby_id, kodi_id, media_type",
+            "FROM emby",
+            "WHERE emby_type = ?",
+        ))
+        self.embycursor.execute(query, (plextype, ))
+        result = []
+        for row in self.embycursor.fetchall():
+            result.append({
+                'plexId': row[0],
+                'kodiId': row[1],
+                'kodi_type': row[2],
+                'plex_type': plextype
+            })
+        return result

resources/lib/itemtypes.py

@@ -65,6 +65,28 @@ class Items(object):
         self.kodiconn.close()
         return self
 
+    @CatchExceptions(warnuser=True)
+    def getfanart(self, item, kodiId, mediaType, allartworks=None):
+        """
+        """
+        API = PlexAPI.API(item)
+        if allartworks is None:
+            allartworks = API.getAllArtwork()
+        self.artwork.addArtwork(API.getFanartArtwork(allartworks),
+                                kodiId,
+                                mediaType,
+                                self.kodicursor)
+        # Also get artwork for collections/movie sets
+        if mediaType == 'movie':
+            for setname in API.getCollections():
+                log.debug('Getting artwork for movie set %s' % setname)
+                setid = self.kodi_db.createBoxset(setname)
+                self.artwork.addArtwork(API.getSetArtwork(),
+                                        setid,
+                                        "set",
+                                        self.kodicursor)
+                self.kodi_db.assignBoxset(setid, kodiId)
+
     def itemsbyId(self, items, process, pdialog=None):
         # Process items by itemid. Process can be added, update, userdata, remove
         embycursor = self.embycursor
@@ -485,7 +507,7 @@ class Movies(Items):
             tags.append("Favorite movies")
         self.kodi_db.addTags(movieid, tags, "movie")
         # Add any sets from Plex collection tags
-        self.kodi_db.addSets(movieid, collections, kodicursor, API)
+        self.kodi_db.addSets(movieid, collections, kodicursor)
         # Process playstates
         self.kodi_db.addPlaystate(fileid, resume, runtime, playcount, dateplayed)
 

resources/lib/kodidb_functions.py

@@ -508,6 +508,48 @@ class Kodidb_Functions():
 
                 self.artwork.addOrUpdateArt(thumb, actorid, arttype, "thumb", self.cursor)
 
+    def existingArt(self, kodiId, mediaType, refresh=False):
+        """
+        For kodiId, returns an artwork dict with already existing art from
+        the Kodi db
+        """
+        # Only get EITHER poster OR thumb (should have same URL)
+        kodiToPKC = {
+            'banner': 'Banner',
+            'clearart': 'Art',
+            'clearlogo': 'Logo',
+            'discart': 'Disc',
+            'landscape': 'Thumb',
+            'thumb': 'Primary'
+        }
+        # BoxRear yet unused
+        result = {'BoxRear': ''}
+        for art in kodiToPKC:
+            query = ' '.join((
+                "SELECT url",
+                "FROM art",
+                "WHERE media_id = ?",
+                "AND media_type = ?",
+                "AND type = ?"
+            ))
+            self.cursor.execute(query, (kodiId, mediaType, art,))
+            try:
+                url = self.cursor.fetchone()[0]
+            except TypeError:
+                url = ""
+            result[kodiToPKC[art]] = url
+        # There may be several fanart URLs saved
+        query = ' '.join((
+            "SELECT url",
+            "FROM art",
+            "WHERE media_id = ?",
+            "AND media_type = ?",
+            "AND type LIKE ?"
+        ))
+        data = self.cursor.execute(query, (kodiId, mediaType, "fanart%",))
+        result['Backdrop'] = [d[0] for d in data]
+        return result
+
     def addGenres(self, kodiid, genres, mediatype):
 
         
@@ -1180,15 +1222,9 @@ class Kodidb_Functions():
                 ))
                 self.cursor.execute(query, (kodiid, mediatype, tag_id,))
 
-    def addSets(self, movieid, collections, kodicursor, API):
+    def addSets(self, movieid, collections, kodicursor):
         for setname in collections:
             setid = self.createBoxset(setname)
-            # Process artwork
-            if settings('setFanartTV') == 'true':
-                self.artwork.addArtwork(API.getSetArtwork(),
-                                        setid,
-                                        "set",
-                                        kodicursor)
             self.assignBoxset(setid, movieid)
 
     def createBoxset(self, boxsetname):

resources/lib/librarysync.py

@@ -5,6 +5,7 @@
 import logging
 from threading import Thread, Lock
 import Queue
+from random import shuffle
 
 import xbmc
 import xbmcgui
@@ -255,6 +256,100 @@ class ThreadedShowSyncInfo(Thread):
         log.debug('Dialog Infobox thread terminated')
 
 
+@ThreadMethodsAdditionalSuspend('suspend_LibraryThread')
+@ThreadMethodsAdditionalStop('plex_shouldStop')
+@ThreadMethods
+class ProcessFanartThread(Thread):
+    """
+    Threaded download of additional fanart in the background
+
+    Input:
+        queue           Queue.Queue() object that you will need to fill with
+                        dicts of the following form:
+            {
+              'itemId':                 the Plex id as a string
+              'class':                  the itemtypes class, e.g. 'Movies'
+              'mediaType':              the kodi media type, e.g. 'movie'
+              'refresh': True/False     if true, will overwrite any 3rd party
+                                        fanart. If False, will only get missing
+            }
+    """
+    def __init__(self, queue):
+        self.queue = queue
+        Thread.__init__(self)
+
+    def run(self):
+        threadStopped = self.threadStopped
+        threadSuspended = self.threadSuspended
+        queue = self.queue
+        log.debug('Started Fanart thread')
+        while not threadStopped():
+            # In the event the server goes offline
+            while threadSuspended():
+                # Set in service.py
+                if threadStopped():
+                    # Abort was requested while waiting. We should exit
+                    log.debug('Fanart thread terminated while suspended')
+                    break
+                xbmc.sleep(1000)
+            # grabs Plex item from queue
+            try:
+                item = queue.get(block=False)
+            except Queue.Empty:
+                xbmc.sleep(50)
+                continue
+            if item['refresh'] is True:
+                # Leave the Plex art untouched
+                allartworks = None
+            else:
+                with embydb.GetEmbyDB() as emby_db:
+                    try:
+                        kodiId = emby_db.getItem_byId(item['itemId'])[0]
+                    except TypeError:
+                        log.error('Could not get Kodi id for plex id %s'
+                                  % item['itemId'])
+                        queue.task_done()
+                        continue
+                with kodidb.GetKodiDB('video') as kodi_db:
+                    allartworks = kodi_db.existingArt(kodiId,
+                                                      item['mediaType'])
+                # Check if we even need to get additional art
+                needsupdate = False
+                for key, value in allartworks.iteritems():
+                    if not value and not key == 'BoxRear':
+                        needsupdate = True
+                if needsupdate is False:
+                    log.debug('Already got all art for Plex id %s'
+                              % item['itemId'])
+                    queue.task_done()
+                    continue
+
+            log.debug('Getting additional fanart for Plex id %s'
+                      % item['itemId'])
+            # Download Metadata
+            xml = PF.GetPlexMetadata(item['itemId'])
+            if xml is None:
+                # Did not receive a valid XML - skip that item for now
+                log.warn("Could not get metadata for %s. Skipping that item "
+                         "for now" % item['itemId'])
+                queue.task_done()
+                continue
+            elif xml == 401:
+                log.warn('HTTP 401 returned by PMS. Too much strain? '
+                         'Cancelling sync for now')
+                # Kill remaining items in queue (for main thread to cont.)
+                queue.task_done()
+                continue
+
+            # Do the work
+            with getattr(itemtypes, item['class'])() as cls:
+                cls.getfanart(xml[0], kodiId, item['mediaType'], allartworks)
+            # signals to queue job is done
+            log.debug('Done getting fanart for Plex id %s' % item['itemId'])
+            queue.task_done()
+        log.debug('Fanart thread terminated')
+
+
 @ThreadMethodsAdditionalSuspend('suspend_LibraryThread')
 @ThreadMethodsAdditionalStop('plex_shouldStop')
 @ThreadMethods
@@ -275,6 +370,9 @@ class LibrarySync(Thread):
         self.queue = queue
         self.itemsToProcess = []
         self.sessionKeys = []
+        if settings('FanartTV') == 'true':
+            self.fanartqueue = Queue.Queue()
+            self.fanartthread = ProcessFanartThread(self.fanartqueue)
         # How long should we wait at least to process new/changed PMS items?
         self.saftyMargin = int(settings('saftyMargin'))
 
@@ -887,6 +985,16 @@ class LibrarySync(Thread):
             except:
                 pass
         log.info("Sync threads finished")
+        if settings('FanartTV') == 'true':
+            # Save to queue for later processing
+            typus = {'Movies': 'movie', 'TVShows': 'tvshow'}[itemType]
+            for item in self.updatelist:
+                self.fanartqueue.put({
+                    'itemId': item['itemId'],
+                    'class': itemType,
+                    'mediaType': typus,
+                    'refresh': False
+                })
         self.updatelist = []
 
     @LogTime
@@ -1484,6 +1592,30 @@ class LibrarySync(Thread):
             with itemFkt() as Fkt:
                 Fkt.updatePlaystate(item)
 
+    def fanartSync(self, refresh=False):
+        """
+        Checks all Plex movies and TV shows whether they still need fanart
+
+        refresh=True        Force refresh all external fanart
+        """
+        items = []
+        typus = {
+            'Movie': 'Movies',
+            'Series': 'TVShows'
+        }
+        with embydb.GetEmbyDB() as emby_db:
+            for plextype in typus:
+                items.extend(emby_db.itemsByType(plextype))
+        # Shuffle the list to not always start out identically
+        items = shuffle(items)
+        for item in items:
+            self.fanartqueue.put({
+                'itemId': item['plexId'],
+                'mediaType': item['kodi_type'],
+                'class': typus[item['plex_type']],
+                'refresh': refresh
+            })
+
     def run(self):
         try:
             self.run_internal()
@@ -1508,6 +1640,7 @@ class LibrarySync(Thread):
         fullSyncInterval = self.fullSyncInterval
         lastSync = 0
         lastTimeSync = 0
+        lastFanartSync = 0
         lastProcessing = 0
         oneDay = 60*60*24
 
@@ -1527,6 +1660,9 @@ class LibrarySync(Thread):
         if self.enableMusic:
             advancedSettingsXML()
 
+        if settings('FanartTV') == 'true':
+            self.fanartthread.start()
+
         while not threadStopped():
 
             # In the event the server goes offline
@@ -1661,6 +1797,12 @@ class LibrarySync(Thread):
                         window('plex_dbScan', value="true")
                         self.syncPMStime()
                         window('plex_dbScan', clear=True)
+                    elif (now - lastFanartSync > oneDay and
+                            settings('FanartTV') == 'true'):
+                        lastFanartSync = now
+                        log.info('Starting daily fanart sync')
+                        self.fanartSync()
+                        log.info('Finished init of daily fanart sync')
                     elif enableBackgroundSync:
                         # Check back whether we should process something
                         # Only do this once every 10 seconds