Rewire fanart sync

resources/lib/library_sync/__init__.py

@@ -6,4 +6,4 @@ from .time import sync_pms_time
 from .websocket import store_websocket_message, process_websocket_messages, \
     WEBSOCKET_MESSAGES, PLAYSTATE_SESSIONS
 from .common import update_kodi_library
-from .fanart import ThreadedProcessFanart, FANART_QUEUE
+from .fanart import FanartThread, FanartTask

resources/lib/library_sync/common.py

@@ -8,7 +8,7 @@ from .. import state
 
 class libsync_mixin(object):
     def isCanceled(self):
-        return (self._canceled or xbmc.abortRequested or
+        return (self._canceled or state.STOP_PKC or
                 state.SUSPEND_LIBRARY_THREAD or state.SUSPEND_SYNC)
 
 

resources/lib/library_sync/fanart.py

@@ -3,107 +3,88 @@ from __future__ import absolute_import, division, unicode_literals
 from logging import getLogger
 import xbmc
 
+from . import common
 from ..plex_api import API
 from ..plex_db import PlexDB
-from .. import backgroundthread
+from .. import backgroundthread, utils, kodidb_functions as kodidb
-from .. import utils, kodidb_functions as kodidb
+from .. import itemtypes, plex_functions as PF, variables as v, state
-from .. import itemtypes, artwork, plex_functions as PF, variables as v, state
 
 
 LOG = getLogger('PLEX.sync.fanart')
 
+SUPPORTED_TYPES = (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW)
 SYNC_FANART = utils.settings('FanartTV') == 'true'
-FANART_QUEUE = backgroundthread.Queue.Queue()
+PREFER_KODI_COLLECTION_ART = utils.settings('PreferKodiCollectionArt') == 'false'
 
 
-class ThreadedProcessFanart(backgroundthread.KillableThread):
+class FanartThread(backgroundthread.KillableThread):
     """
-    Threaded download of additional fanart in the background
+    This will potentially take hours!
-
-    Input:
-        queue           Queue.Queue() object that you will need to fill with
-                        dicts of the following form:
-            {
-              'plex_id':                the Plex id as a string
-              'plex_type':              the Plex media type, e.g. 'movie'
-              'refresh': True/False     if True, will overwrite any 3rd party
-                                        fanart. If False, will only get missing
-            }
     """
+    def __init__(self, callback, refresh=False):
+        self.callback = callback
+        self.refresh = refresh
+        super(FanartThread, self).__init__()
+
     def isCanceled(self):
-        return xbmc.abortRequested or state.STOP_PKC
+        return state.STOP_PKC
 
     def isSuspended(self):
-        return (state.SUSPEND_LIBRARY_THREAD or
+        return state.SUSPEND_LIBRARY_THREAD or state.STOP_SYNC
-                state.DB_SCAN or
-                state.STOP_SYNC or
-                state.SUSPEND_SYNC)
 
     def run(self):
-        LOG.info('---===### Starting FanartSync ###===---')
         try:
-            self._run()
+            self._run_internal()
         except:
-            utils.ERROR(txt='FanartSync crashed', notify=True)
+            utils.ERROR()
-            raise
-        LOG.info('---===### Stopping FanartSync ###===---')
 
-    def _run(self):
+    def _run_internal(self):
-        """
+        LOG.info('Starting FanartThread')
-        Do the work
+        with PlexDB() as plexdb:
-        """
+            func = plexdb.every_plex_id if self.refresh else plexdb.missing_fanart
-        # First run through our already synced items in the Plex DB
+            for typus in SUPPORTED_TYPES:
-        for plex_type in (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW):
+                for plex_id in func(typus):
-            with PlexDB() as plexdb:
-                for plex_id in plexdb.fanart(plex_type):
                     if self.isCanceled():
-                        break
+                        return
-                    while self.isSuspended():
+                    if self.isSuspended():
                         if self.isCanceled():
-                            break
+                            return
                         xbmc.sleep(1000)
-                    process_item(plexdb, {'plex_id': plex_id,
+                    process_fanart(plex_id, typus, self.refresh)
-                                          'plex_type': plex_type,
+        LOG.info('FanartThread finished')
-                                          'refresh': False})
+        self.callback()
-
-        # Then keep checking the queue for new items
-        while not self.isCanceled():
-            # In the event the server goes offline
-            while self.isSuspended():
-                # Set in service.py
-                if self.isCanceled():
-                    return
-                xbmc.sleep(1000)
-            # grabs Plex item from queue
-            try:
-                item = FANART_QUEUE.get(block=False)
-            except backgroundthread.Empty:
-                xbmc.sleep(1000)
-                continue
-            FANART_QUEUE.task_done()
-            if isinstance(item, artwork.ArtworkSyncMessage):
-                if state.IMAGE_SYNC_NOTIFICATIONS:
-                    utils.dialog('notification',
-                                 heading=utils.lang(29999),
-                                 message=item.message,
-                                 icon='{plex}',
-                                 sound=False)
-                continue
-            LOG.debug('Get additional fanart for Plex id %s', item['plex_id'])
-            with PlexDB() as plexdb:
-                process_item(plexdb, item)
 
 
-def process_item(plexdb, item):
+class FanartTask(backgroundthread.Task, common.libsync_mixin):
+    """
+    This task will also be executed while library sync is suspended!
+    """
+    def setup(self, plex_id, plex_type, refresh=False):
+        self.plex_id = plex_id
+        self.plex_type = plex_type
+        self.refresh = refresh
+
+    def run(self):
+        process_fanart(self.plex_id, self.plex_type, self.refresh)
+
+
+def process_fanart(plex_id, plex_type, refresh=False):
+    """
+    Will look for additional fanart for the plex_type item with plex_id.
+    Will check if we already got all artwork and only look if some are indeed
+    missing.
+    Will set the fanart_synced flag in the Plex DB if successful.
+    """
     done = False
     try:
         artworks = None
-        db_item = plexdb.item_by_id(item['plex_id'], item['plex_type'])
+        with PlexDB() as plexdb:
+            db_item = plexdb.item_by_id(plex_id,
+                                        plex_type)
         if not db_item:
-            LOG.error('Could not get Kodi id for plex id %s, abort getfanart',
+            LOG.error('Could not get Kodi id for plex id %s', plex_id)
-                      item['plex_id'])
             return
-        if item['refresh'] is False:
+        if not refresh:
             with kodidb.GetKodiDB('video') as kodi_db:
                 artworks = kodi_db.get_art(db_item['kodi_id'],
                                            db_item['kodi_type'])
@@ -112,37 +93,32 @@ def process_item(plexdb, item):
                 if key not in artworks:
                     break
             else:
-                LOG.debug('Already got all fanart for Plex id %s',
-                          item['plex_id'])
                 done = True
                 return
-        xml = PF.GetPlexMetadata(item['plex_id'])
+        xml = PF.GetPlexMetadata(plex_id)
-        if xml is None:
+        try:
-            LOG.error('Could not get metadata for %s. Skipping that item '
+            xml[0].attrib
-                      'for now', item['plex_id'])
+        except (TypeError, IndexError, AttributeError):
-            return
+            LOG.warn('Could not get metadata for %s. Skipping that item '
-        elif xml == 401:
+                     'for now', plex_id)
-            LOG.error('HTTP 401 returned by PMS. Too much strain? '
-                      'Cancelling sync for now')
             return
         api = API(xml[0])
         if artworks is None:
             artworks = api.artwork()
         # Get additional missing artwork from fanart artwork sites
         artworks = api.fanart_artwork(artworks)
-        with itemtypes.ITEMTYPE_FROM_PLEXTYPE[item['plex_type']] as context:
+        with itemtypes.ITEMTYPE_FROM_PLEXTYPE[plex_type] as context:
             context.set_fanart(artworks,
                                db_item['kodi_id'],
                                db_item['kodi_type'])
         # Additional fanart for sets/collections
-        if api.plex_type() == v.PLEX_TYPE_MOVIE:
+        if plex_type == v.PLEX_TYPE_MOVIE:
             for _, setname in api.collection_list():
                 LOG.debug('Getting artwork for movie set %s', setname)
                 with kodidb.GetKodiDB('video') as kodi_db:
                     setid = kodi_db.create_collection(setname)
                 external_set_artwork = api.set_artwork()
-                if (external_set_artwork and
+                if external_set_artwork and PREFER_KODI_COLLECTION_ART:
-                        utils.settings('PreferKodiCollectionArt') == 'false'):
                     kodi_artwork = api.artwork(kodi_id=setid,
                                                kodi_type=v.KODI_TYPE_SET)
                     for art in kodi_artwork:
@@ -156,5 +132,6 @@ def process_item(plexdb, item):
         done = True
     finally:
         if done is True:
-            LOG.debug('Done getting fanart for Plex id %s', item['plex_id'])
+            with PlexDB() as plexdb:
-            plexdb.set_fanart_synced(item['plex_id'], item['plex_type'])
+                plexdb.set_fanart_synced(plex_id,
+                                         plex_type)

resources/lib/library_sync/websocket.py

@@ -5,10 +5,10 @@ from logging import getLogger
 
 from .common import update_kodi_library
 from .full_sync import PLAYLIST_SYNC_ENABLED
-from .fanart import FANART_QUEUE, SYNC_FANART
+from .fanart import SYNC_FANART, FanartTask
 from ..plex_api import API
 from ..plex_db import PlexDB
-from .. import playlists, plex_functions as PF, itemtypes
+from .. import backgroundthread, playlists, plex_functions as PF, itemtypes
 from .. import utils, variables as v, state
 
 LOG = getLogger('PLEX.sync.websocket')
@@ -89,11 +89,11 @@ def process_websocket_messages():
             successful, video, music = process_new_item_message(message)
             if (successful and SYNC_FANART and
                     message['type'] in (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW)):
-                FANART_QUEUE.put({
+                task = FanartTask()
-                    'plex_id': utils.cast(int, message['ratingKey']),
+                task.setup(utils.cast(int, message['ratingKey']),
-                    'plex_type': message['type'],
+                           message['type'],
-                    'refresh': False
+                           refresh=False)
-                })
+                backgroundthread.BGThreader.addTask(task)
         if successful is True:
             delete_list.append(i)
             update_kodi_video_library = True if video else update_kodi_video_library

resources/lib/plex_db/common.py

@@ -119,7 +119,15 @@ class PlexDBBase(object):
         query = 'DELETE FROM ? WHERE plex_id = ?' % plex_type
         self.cursor.execute(query, (plex_id, ))
 
-    def fanart(self, plex_type):
+    def every_plex_id(self, plex_type):
+        """
+        Returns an iterator for plex_type for every single plex_id
+        """
+        query = 'SELECT plex_id from %s' % plex_type
+        self.cursor.execute(query)
+        return (x[0] for x in self.cursor)
+
+    def missing_fanart(self, plex_type):
         """
         Returns an iterator for plex_type for all plex_id, where fanart_synced
         has not yet been set to 1

resources/lib/service_entry.py

@@ -68,6 +68,7 @@ class Service():
         LOG.info('Play playlist prefix: %s',
                  utils.settings('syncSpecificPlexPlaylistsPrefix'))
         LOG.info('XML decoding being used: %s', utils.ETREE)
+        LOG.info("Db version: %s", utils.settings('dbCreatedWithVersion'))
         self.monitor = xbmc.Monitor()
         # Load/Reset PKC entirely - important for user/Kodi profile switch
         initialsetup.reload_pkc()

resources/lib/sync.py

@@ -2,8 +2,6 @@
 # -*- coding: utf-8 -*-
 from __future__ import absolute_import, division, unicode_literals
 from logging import getLogger
-from random import shuffle
-import Queue
 import xbmc
 
 from . import library_sync
@@ -37,12 +35,7 @@ class Sync(backgroundthread.KillableThread):
     def __init__(self):
         self.sync_successful = False
         self.last_full_sync = 0
-        if utils.settings('FanartTV') == 'true':
+        self.fanart = None
-            self.fanartqueue = Queue.Queue()
-            self.fanartthread = library_sync.fanart.ThreadedProcessFanart(self.fanartqueue)
-        else:
-            self.fanartqueue = None
-            self.fanartthread = None
         # How long should we wait at least to process new/changed PMS items?
         # Show sync dialog even if user deactivated?
         self.force_dialog = False
@@ -69,7 +62,7 @@ class Sync(backgroundthread.KillableThread):
             return True
         return False
 
-    def show_kodi_note(self, message, icon="plex"):
+    def show_kodi_note(self, message, icon="plex", force=False):
         """
         Shows a Kodi popup, if user selected to do so. Pass message in unicode
         or string
@@ -77,7 +70,7 @@ class Sync(backgroundthread.KillableThread):
         icon:   "plex": shows Plex icon
                 "error": shows Kodi error icon
         """
-        if state.SYNC_DIALOG is not True and self.force_dialog is not True:
+        if not force and state.SYNC_DIALOG is not True and self.force_dialog is not True:
             return
         if icon == "plex":
             utils.dialog('notification',
@@ -91,43 +84,6 @@ class Sync(backgroundthread.KillableThread):
                          message=message,
                          icon='{error}')
 
-    def sync_fanart(self, missing_only=True, refresh=False):
-        """
-        Throw items to the fanart queue in order to download missing (or all)
-        additional fanart.
-
-        missing_only=True    False will start look-up for EVERY item
-        refresh=False        True will force refresh all external fanart
-        """
-        if utils.settings('FanartTV') == 'false':
-            return
-        with plexdb.Get_Plex_DB() as plex_db:
-            if missing_only:
-                with plexdb.Get_Plex_DB() as plex_db:
-                    items = plex_db.get_missing_fanart()
-                LOG.info('Trying to get %s additional fanart', len(items))
-            else:
-                items = []
-                for plex_type in (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW):
-                    items.extend(plex_db.itemsByType(plex_type))
-                LOG.info('Trying to get ALL additional fanart for %s items',
-                         len(items))
-        if not items:
-            return
-        # Shuffle the list to not always start out identically
-        shuffle(items)
-        # Checking FanartTV for %s items
-        self.fanartqueue.put(artwork.ArtworkSyncMessage(
-            utils.lang(30018) % len(items)))
-        for item in items:
-            self.fanartqueue.put({
-                'plex_id': item['plex_id'],
-                'plex_type': item['plex_type'],
-                'refresh': refresh
-            })
-        # FanartTV lookup completed
-        self.fanartqueue.put(artwork.ArtworkSyncMessage(utils.lang(30019)))
-
     def triage_lib_scans(self):
         """
         Decides what to do if state.RUN_LIB_SCAN has been set. E.g. manually
@@ -148,21 +104,22 @@ class Sync(backgroundthread.KillableThread):
                 self.show_kodi_note(utils.lang(39410), icon='error')
                 self.force_dialog = False
         elif state.RUN_LIB_SCAN == 'fanart':
-            # Only look for missing fanart (No)
+            # Only look for missing fanart (No) or refresh all fanart (Yes)
-            # or refresh all fanart (Yes)
             from .windows import optionsdialog
             refresh = optionsdialog.show(utils.lang(29999),
                                          utils.lang(39223),
                                          utils.lang(39224),  # refresh all
                                          utils.lang(39225)) == 0
-            self.sync_fanart(missing_only=not refresh, refresh=refresh)
+            if not self.start_fanart_download(refresh=refresh):
+                utils.dialog('notification',
+                             heading='{plex}',
+                             message=message,
+                             icon='{plex}',
+                             sound=False)
         elif state.RUN_LIB_SCAN == 'textures':
             artwork.Artwork().fullTextureCacheSync()
-        else:
-            raise NotImplementedError('Library scan not defined: %s'
-                                      % state.RUN_LIB_SCAN)
 
-    def onLibrary_scan_finished(self, successful):
+    def on_library_scan_finished(self, successful):
         """
         Hit this after the full sync has finished
         """
@@ -178,12 +135,35 @@ class Sync(backgroundthread.KillableThread):
         show_dialog = show_dialog if show_dialog is not None else state.SYNC_DIALOG
         if block:
             self.lock.acquire()
-            library_sync.start(show_dialog, repair, self.onLibrary_scan_finished)
+            library_sync.start(show_dialog, repair, self.on_library_scan_finished)
             # Will block until scan is finished
             self.lock.acquire()
             self.lock.release()
         else:
-            library_sync.start(show_dialog, repair, self.onLibrary_scan_finished)
+            library_sync.start(show_dialog, repair, self.on_library_scan_finished)
+
+    def start_fanart_download(self, refresh):
+        if not utils.settings('FanartTV') == 'true':
+            LOG.info('Additional fanart download is deactivated')
+            return False
+        elif self.fanart is None or not self.fanart.is_alive():
+            LOG.info('Start downloading additional fanart with refresh %s',
+                     refresh)
+            self.fanart = library_sync.FanartThread(self.on_fanart_download_finished, refresh)
+            self.fanart.start()
+            return True
+        else:
+            LOG.info('Still downloading fanart')
+            return False
+
+    def on_fanart_download_finished(self):
+        # FanartTV lookup completed
+        if state.SYNC_DIALOG:
+            utils.dialog('notification',
+                         heading='{plex}',
+                         message=utils.lang(30019),
+                         icon='{plex}',
+                         sound=False)
 
     def run(self):
         try:
@@ -191,12 +171,11 @@ class Sync(backgroundthread.KillableThread):
         except:
             state.DB_SCAN = False
             utils.window('plex_dbScan', clear=True)
-            utils.ERROR(txt='Sync.py crashed', notify=True)
+            utils.ERROR(txt='sync.py crashed', notify=True)
             raise
 
     def _run_internal(self):
         LOG.info("---===### Starting Sync ###===---")
-        self.force_dialog = False
         install_sync_done = utils.settings('SyncInstallRunDone') == 'true'
         playlist_monitor = None
         initial_sync_done = False
@@ -239,9 +218,6 @@ class Sync(backgroundthread.KillableThread):
         plex_db.initialize()
         # Hack to speed up look-ups for actors (giant table!)
         utils.create_actor_db_index()
-        # Run start up sync
-        LOG.info("Db version: %s", utils.settings('dbCreatedWithVersion'))
-        LOG.info('Refreshing video nodes and playlists now')
         with kodidb.GetKodiDB('video') as kodi_db:
             # Setup the paths for addon-paths (even when using direct paths)
             kodi_db.setup_path_table()
@@ -276,8 +252,7 @@ class Sync(backgroundthread.KillableThread):
                     if library_sync.PLAYLIST_SYNC_ENABLED:
                         from . import playlists
                         playlist_monitor = playlists.kodi_playlist_monitor()
-                    self.sync_fanart()
+                    self.start_fanart_download(refresh=False)
-                    self.fanartthread.start()
                 else:
                     LOG.error('Initial start-up full sync unsuccessful')
                 self.force_dialog = False
@@ -299,8 +274,7 @@ class Sync(backgroundthread.KillableThread):
                         from . import playlists
                         playlist_monitor = playlists.kodi_playlist_monitor()
                     artwork.Artwork().cache_major_artwork()
-                    self.sync_fanart()
+                    self.start_fanart_download(refresh=False)
-                    self.fanartthread.start()
                 else:
                     LOG.info('Startup sync has not yet been successful')