From 5673abc19b0f9b19077a821d855c8acbfaba9e58 Mon Sep 17 00:00:00 2001 From: croneter Date: Sat, 3 Nov 2018 10:36:37 +0100 Subject: [PATCH] Rewire fanart sync --- resources/lib/library_sync/__init__.py | 2 +- resources/lib/library_sync/common.py | 2 +- resources/lib/library_sync/fanart.py | 153 ++++++++++-------------- resources/lib/library_sync/websocket.py | 14 +-- resources/lib/plex_db/common.py | 10 +- resources/lib/service_entry.py | 1 + resources/lib/sync.py | 104 ++++++---------- 7 files changed, 123 insertions(+), 163 deletions(-) diff --git a/resources/lib/library_sync/__init__.py b/resources/lib/library_sync/__init__.py index 34aa27f6..2d5d4d55 100644 --- a/resources/lib/library_sync/__init__.py +++ b/resources/lib/library_sync/__init__.py @@ -6,4 +6,4 @@ from .time import sync_pms_time from .websocket import store_websocket_message, process_websocket_messages, \ WEBSOCKET_MESSAGES, PLAYSTATE_SESSIONS from .common import update_kodi_library -from .fanart import ThreadedProcessFanart, FANART_QUEUE +from .fanart import FanartThread, FanartTask diff --git a/resources/lib/library_sync/common.py b/resources/lib/library_sync/common.py index 7ace5d9d..bfee2808 100644 --- a/resources/lib/library_sync/common.py +++ b/resources/lib/library_sync/common.py @@ -8,7 +8,7 @@ from .. import state class libsync_mixin(object): def isCanceled(self): - return (self._canceled or xbmc.abortRequested or + return (self._canceled or state.STOP_PKC or state.SUSPEND_LIBRARY_THREAD or state.SUSPEND_SYNC) diff --git a/resources/lib/library_sync/fanart.py b/resources/lib/library_sync/fanart.py index d08ff8ae..72eee725 100644 --- a/resources/lib/library_sync/fanart.py +++ b/resources/lib/library_sync/fanart.py @@ -3,107 +3,88 @@ from __future__ import absolute_import, division, unicode_literals from logging import getLogger import xbmc +from . import common from ..plex_api import API from ..plex_db import PlexDB -from .. import backgroundthread -from .. import utils, kodidb_functions as kodidb -from .. import itemtypes, artwork, plex_functions as PF, variables as v, state +from .. import backgroundthread, utils, kodidb_functions as kodidb +from .. import itemtypes, plex_functions as PF, variables as v, state LOG = getLogger('PLEX.sync.fanart') +SUPPORTED_TYPES = (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW) SYNC_FANART = utils.settings('FanartTV') == 'true' -FANART_QUEUE = backgroundthread.Queue.Queue() +PREFER_KODI_COLLECTION_ART = utils.settings('PreferKodiCollectionArt') == 'false' -class ThreadedProcessFanart(backgroundthread.KillableThread): +class FanartThread(backgroundthread.KillableThread): """ - Threaded download of additional fanart in the background - - Input: - queue Queue.Queue() object that you will need to fill with - dicts of the following form: - { - 'plex_id': the Plex id as a string - 'plex_type': the Plex media type, e.g. 'movie' - 'refresh': True/False if True, will overwrite any 3rd party - fanart. If False, will only get missing - } + This will potentially take hours! """ + def __init__(self, callback, refresh=False): + self.callback = callback + self.refresh = refresh + super(FanartThread, self).__init__() + def isCanceled(self): - return xbmc.abortRequested or state.STOP_PKC + return state.STOP_PKC def isSuspended(self): - return (state.SUSPEND_LIBRARY_THREAD or - state.DB_SCAN or - state.STOP_SYNC or - state.SUSPEND_SYNC) + return state.SUSPEND_LIBRARY_THREAD or state.STOP_SYNC def run(self): - LOG.info('---===### Starting FanartSync ###===---') try: - self._run() + self._run_internal() except: - utils.ERROR(txt='FanartSync crashed', notify=True) - raise - LOG.info('---===### Stopping FanartSync ###===---') + utils.ERROR() - def _run(self): - """ - Do the work - """ - # First run through our already synced items in the Plex DB - for plex_type in (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW): - with PlexDB() as plexdb: - for plex_id in plexdb.fanart(plex_type): + def _run_internal(self): + LOG.info('Starting FanartThread') + with PlexDB() as plexdb: + func = plexdb.every_plex_id if self.refresh else plexdb.missing_fanart + for typus in SUPPORTED_TYPES: + for plex_id in func(typus): if self.isCanceled(): - break - while self.isSuspended(): + return + if self.isSuspended(): if self.isCanceled(): - break + return xbmc.sleep(1000) - process_item(plexdb, {'plex_id': plex_id, - 'plex_type': plex_type, - 'refresh': False}) - - # Then keep checking the queue for new items - while not self.isCanceled(): - # In the event the server goes offline - while self.isSuspended(): - # Set in service.py - if self.isCanceled(): - return - xbmc.sleep(1000) - # grabs Plex item from queue - try: - item = FANART_QUEUE.get(block=False) - except backgroundthread.Empty: - xbmc.sleep(1000) - continue - FANART_QUEUE.task_done() - if isinstance(item, artwork.ArtworkSyncMessage): - if state.IMAGE_SYNC_NOTIFICATIONS: - utils.dialog('notification', - heading=utils.lang(29999), - message=item.message, - icon='{plex}', - sound=False) - continue - LOG.debug('Get additional fanart for Plex id %s', item['plex_id']) - with PlexDB() as plexdb: - process_item(plexdb, item) + process_fanart(plex_id, typus, self.refresh) + LOG.info('FanartThread finished') + self.callback() -def process_item(plexdb, item): +class FanartTask(backgroundthread.Task, common.libsync_mixin): + """ + This task will also be executed while library sync is suspended! + """ + def setup(self, plex_id, plex_type, refresh=False): + self.plex_id = plex_id + self.plex_type = plex_type + self.refresh = refresh + + def run(self): + process_fanart(self.plex_id, self.plex_type, self.refresh) + + +def process_fanart(plex_id, plex_type, refresh=False): + """ + Will look for additional fanart for the plex_type item with plex_id. + Will check if we already got all artwork and only look if some are indeed + missing. + Will set the fanart_synced flag in the Plex DB if successful. + """ done = False try: artworks = None - db_item = plexdb.item_by_id(item['plex_id'], item['plex_type']) + with PlexDB() as plexdb: + db_item = plexdb.item_by_id(plex_id, + plex_type) if not db_item: - LOG.error('Could not get Kodi id for plex id %s, abort getfanart', - item['plex_id']) + LOG.error('Could not get Kodi id for plex id %s', plex_id) return - if item['refresh'] is False: + if not refresh: with kodidb.GetKodiDB('video') as kodi_db: artworks = kodi_db.get_art(db_item['kodi_id'], db_item['kodi_type']) @@ -112,37 +93,32 @@ def process_item(plexdb, item): if key not in artworks: break else: - LOG.debug('Already got all fanart for Plex id %s', - item['plex_id']) done = True return - xml = PF.GetPlexMetadata(item['plex_id']) - if xml is None: - LOG.error('Could not get metadata for %s. Skipping that item ' - 'for now', item['plex_id']) - return - elif xml == 401: - LOG.error('HTTP 401 returned by PMS. Too much strain? ' - 'Cancelling sync for now') + xml = PF.GetPlexMetadata(plex_id) + try: + xml[0].attrib + except (TypeError, IndexError, AttributeError): + LOG.warn('Could not get metadata for %s. Skipping that item ' + 'for now', plex_id) return api = API(xml[0]) if artworks is None: artworks = api.artwork() # Get additional missing artwork from fanart artwork sites artworks = api.fanart_artwork(artworks) - with itemtypes.ITEMTYPE_FROM_PLEXTYPE[item['plex_type']] as context: + with itemtypes.ITEMTYPE_FROM_PLEXTYPE[plex_type] as context: context.set_fanart(artworks, db_item['kodi_id'], db_item['kodi_type']) # Additional fanart for sets/collections - if api.plex_type() == v.PLEX_TYPE_MOVIE: + if plex_type == v.PLEX_TYPE_MOVIE: for _, setname in api.collection_list(): LOG.debug('Getting artwork for movie set %s', setname) with kodidb.GetKodiDB('video') as kodi_db: setid = kodi_db.create_collection(setname) external_set_artwork = api.set_artwork() - if (external_set_artwork and - utils.settings('PreferKodiCollectionArt') == 'false'): + if external_set_artwork and PREFER_KODI_COLLECTION_ART: kodi_artwork = api.artwork(kodi_id=setid, kodi_type=v.KODI_TYPE_SET) for art in kodi_artwork: @@ -156,5 +132,6 @@ def process_item(plexdb, item): done = True finally: if done is True: - LOG.debug('Done getting fanart for Plex id %s', item['plex_id']) - plexdb.set_fanart_synced(item['plex_id'], item['plex_type']) + with PlexDB() as plexdb: + plexdb.set_fanart_synced(plex_id, + plex_type) diff --git a/resources/lib/library_sync/websocket.py b/resources/lib/library_sync/websocket.py index 64000702..7e721616 100644 --- a/resources/lib/library_sync/websocket.py +++ b/resources/lib/library_sync/websocket.py @@ -5,10 +5,10 @@ from logging import getLogger from .common import update_kodi_library from .full_sync import PLAYLIST_SYNC_ENABLED -from .fanart import FANART_QUEUE, SYNC_FANART +from .fanart import SYNC_FANART, FanartTask from ..plex_api import API from ..plex_db import PlexDB -from .. import playlists, plex_functions as PF, itemtypes +from .. import backgroundthread, playlists, plex_functions as PF, itemtypes from .. import utils, variables as v, state LOG = getLogger('PLEX.sync.websocket') @@ -89,11 +89,11 @@ def process_websocket_messages(): successful, video, music = process_new_item_message(message) if (successful and SYNC_FANART and message['type'] in (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW)): - FANART_QUEUE.put({ - 'plex_id': utils.cast(int, message['ratingKey']), - 'plex_type': message['type'], - 'refresh': False - }) + task = FanartTask() + task.setup(utils.cast(int, message['ratingKey']), + message['type'], + refresh=False) + backgroundthread.BGThreader.addTask(task) if successful is True: delete_list.append(i) update_kodi_video_library = True if video else update_kodi_video_library diff --git a/resources/lib/plex_db/common.py b/resources/lib/plex_db/common.py index 5841087d..4d1e9789 100644 --- a/resources/lib/plex_db/common.py +++ b/resources/lib/plex_db/common.py @@ -119,7 +119,15 @@ class PlexDBBase(object): query = 'DELETE FROM ? WHERE plex_id = ?' % plex_type self.cursor.execute(query, (plex_id, )) - def fanart(self, plex_type): + def every_plex_id(self, plex_type): + """ + Returns an iterator for plex_type for every single plex_id + """ + query = 'SELECT plex_id from %s' % plex_type + self.cursor.execute(query) + return (x[0] for x in self.cursor) + + def missing_fanart(self, plex_type): """ Returns an iterator for plex_type for all plex_id, where fanart_synced has not yet been set to 1 diff --git a/resources/lib/service_entry.py b/resources/lib/service_entry.py index 86812c03..7738b0db 100644 --- a/resources/lib/service_entry.py +++ b/resources/lib/service_entry.py @@ -68,6 +68,7 @@ class Service(): LOG.info('Play playlist prefix: %s', utils.settings('syncSpecificPlexPlaylistsPrefix')) LOG.info('XML decoding being used: %s', utils.ETREE) + LOG.info("Db version: %s", utils.settings('dbCreatedWithVersion')) self.monitor = xbmc.Monitor() # Load/Reset PKC entirely - important for user/Kodi profile switch initialsetup.reload_pkc() diff --git a/resources/lib/sync.py b/resources/lib/sync.py index 8f7ae4ae..e03a5670 100644 --- a/resources/lib/sync.py +++ b/resources/lib/sync.py @@ -2,8 +2,6 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, division, unicode_literals from logging import getLogger -from random import shuffle -import Queue import xbmc from . import library_sync @@ -37,12 +35,7 @@ class Sync(backgroundthread.KillableThread): def __init__(self): self.sync_successful = False self.last_full_sync = 0 - if utils.settings('FanartTV') == 'true': - self.fanartqueue = Queue.Queue() - self.fanartthread = library_sync.fanart.ThreadedProcessFanart(self.fanartqueue) - else: - self.fanartqueue = None - self.fanartthread = None + self.fanart = None # How long should we wait at least to process new/changed PMS items? # Show sync dialog even if user deactivated? self.force_dialog = False @@ -69,7 +62,7 @@ class Sync(backgroundthread.KillableThread): return True return False - def show_kodi_note(self, message, icon="plex"): + def show_kodi_note(self, message, icon="plex", force=False): """ Shows a Kodi popup, if user selected to do so. Pass message in unicode or string @@ -77,7 +70,7 @@ class Sync(backgroundthread.KillableThread): icon: "plex": shows Plex icon "error": shows Kodi error icon """ - if state.SYNC_DIALOG is not True and self.force_dialog is not True: + if not force and state.SYNC_DIALOG is not True and self.force_dialog is not True: return if icon == "plex": utils.dialog('notification', @@ -91,43 +84,6 @@ class Sync(backgroundthread.KillableThread): message=message, icon='{error}') - def sync_fanart(self, missing_only=True, refresh=False): - """ - Throw items to the fanart queue in order to download missing (or all) - additional fanart. - - missing_only=True False will start look-up for EVERY item - refresh=False True will force refresh all external fanart - """ - if utils.settings('FanartTV') == 'false': - return - with plexdb.Get_Plex_DB() as plex_db: - if missing_only: - with plexdb.Get_Plex_DB() as plex_db: - items = plex_db.get_missing_fanart() - LOG.info('Trying to get %s additional fanart', len(items)) - else: - items = [] - for plex_type in (v.PLEX_TYPE_MOVIE, v.PLEX_TYPE_SHOW): - items.extend(plex_db.itemsByType(plex_type)) - LOG.info('Trying to get ALL additional fanart for %s items', - len(items)) - if not items: - return - # Shuffle the list to not always start out identically - shuffle(items) - # Checking FanartTV for %s items - self.fanartqueue.put(artwork.ArtworkSyncMessage( - utils.lang(30018) % len(items))) - for item in items: - self.fanartqueue.put({ - 'plex_id': item['plex_id'], - 'plex_type': item['plex_type'], - 'refresh': refresh - }) - # FanartTV lookup completed - self.fanartqueue.put(artwork.ArtworkSyncMessage(utils.lang(30019))) - def triage_lib_scans(self): """ Decides what to do if state.RUN_LIB_SCAN has been set. E.g. manually @@ -148,21 +104,22 @@ class Sync(backgroundthread.KillableThread): self.show_kodi_note(utils.lang(39410), icon='error') self.force_dialog = False elif state.RUN_LIB_SCAN == 'fanart': - # Only look for missing fanart (No) - # or refresh all fanart (Yes) + # Only look for missing fanart (No) or refresh all fanart (Yes) from .windows import optionsdialog refresh = optionsdialog.show(utils.lang(29999), utils.lang(39223), utils.lang(39224), # refresh all utils.lang(39225)) == 0 - self.sync_fanart(missing_only=not refresh, refresh=refresh) + if not self.start_fanart_download(refresh=refresh): + utils.dialog('notification', + heading='{plex}', + message=message, + icon='{plex}', + sound=False) elif state.RUN_LIB_SCAN == 'textures': artwork.Artwork().fullTextureCacheSync() - else: - raise NotImplementedError('Library scan not defined: %s' - % state.RUN_LIB_SCAN) - def onLibrary_scan_finished(self, successful): + def on_library_scan_finished(self, successful): """ Hit this after the full sync has finished """ @@ -178,12 +135,35 @@ class Sync(backgroundthread.KillableThread): show_dialog = show_dialog if show_dialog is not None else state.SYNC_DIALOG if block: self.lock.acquire() - library_sync.start(show_dialog, repair, self.onLibrary_scan_finished) + library_sync.start(show_dialog, repair, self.on_library_scan_finished) # Will block until scan is finished self.lock.acquire() self.lock.release() else: - library_sync.start(show_dialog, repair, self.onLibrary_scan_finished) + library_sync.start(show_dialog, repair, self.on_library_scan_finished) + + def start_fanart_download(self, refresh): + if not utils.settings('FanartTV') == 'true': + LOG.info('Additional fanart download is deactivated') + return False + elif self.fanart is None or not self.fanart.is_alive(): + LOG.info('Start downloading additional fanart with refresh %s', + refresh) + self.fanart = library_sync.FanartThread(self.on_fanart_download_finished, refresh) + self.fanart.start() + return True + else: + LOG.info('Still downloading fanart') + return False + + def on_fanart_download_finished(self): + # FanartTV lookup completed + if state.SYNC_DIALOG: + utils.dialog('notification', + heading='{plex}', + message=utils.lang(30019), + icon='{plex}', + sound=False) def run(self): try: @@ -191,12 +171,11 @@ class Sync(backgroundthread.KillableThread): except: state.DB_SCAN = False utils.window('plex_dbScan', clear=True) - utils.ERROR(txt='Sync.py crashed', notify=True) + utils.ERROR(txt='sync.py crashed', notify=True) raise def _run_internal(self): LOG.info("---===### Starting Sync ###===---") - self.force_dialog = False install_sync_done = utils.settings('SyncInstallRunDone') == 'true' playlist_monitor = None initial_sync_done = False @@ -239,9 +218,6 @@ class Sync(backgroundthread.KillableThread): plex_db.initialize() # Hack to speed up look-ups for actors (giant table!) utils.create_actor_db_index() - # Run start up sync - LOG.info("Db version: %s", utils.settings('dbCreatedWithVersion')) - LOG.info('Refreshing video nodes and playlists now') with kodidb.GetKodiDB('video') as kodi_db: # Setup the paths for addon-paths (even when using direct paths) kodi_db.setup_path_table() @@ -276,8 +252,7 @@ class Sync(backgroundthread.KillableThread): if library_sync.PLAYLIST_SYNC_ENABLED: from . import playlists playlist_monitor = playlists.kodi_playlist_monitor() - self.sync_fanart() - self.fanartthread.start() + self.start_fanart_download(refresh=False) else: LOG.error('Initial start-up full sync unsuccessful') self.force_dialog = False @@ -299,8 +274,7 @@ class Sync(backgroundthread.KillableThread): from . import playlists playlist_monitor = playlists.kodi_playlist_monitor() artwork.Artwork().cache_major_artwork() - self.sync_fanart() - self.fanartthread.start() + self.start_fanart_download(refresh=False) else: LOG.info('Startup sync has not yet been successful')