From e76156759218f65beaf66470886ee14eb21b825b Mon Sep 17 00:00:00 2001 From: croneter Date: Mon, 5 Nov 2018 15:23:51 +0100 Subject: [PATCH] Rewire image caching thread --- resources/lib/artwork.py | 195 ++++++++++++++---------------- resources/lib/kodidb_functions.py | 15 +++ resources/lib/sync.py | 15 ++- 3 files changed, 119 insertions(+), 106 deletions(-) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index f2f6b92f..ccdc1cfc 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -9,8 +9,7 @@ import requests import xbmc -from . import path_ops -from . import utils +from . import backgroundthread, path_ops, utils from . import state ############################################################################### @@ -19,9 +18,12 @@ LOG = getLogger('PLEX.artwork') # Disable annoying requests warnings requests.packages.urllib3.disable_warnings() ARTWORK_QUEUE = Queue() -IMAGE_CACHING_SUSPENDS = ['SUSPEND_LIBRARY_THREAD', 'DB_SCAN', 'STOP_SYNC'] +IMAGE_CACHING_SUSPENDS = [ + state.SUSPEND_LIBRARY_THREAD, + state.DB_SCAN +] if not utils.settings('imageSyncDuringPlayback') == 'true': - IMAGE_CACHING_SUSPENDS.append('SUSPEND_SYNC') + IMAGE_CACHING_SUSPENDS.append(state.SUSPEND_SYNC) ############################################################################### @@ -34,9 +36,7 @@ def double_urldecode(text): return unquote(unquote(text)) -@utils.thread_methods(add_suspends=IMAGE_CACHING_SUSPENDS) -class ImageCachingThread(Thread): - sleep_between = 50 +class ImageCachingThread(backgroundthread.KillableThread): # Potentially issues with limited number of threads # Hence let Kodi wait till download is successful timeout = (35.1, 35.1) @@ -45,24 +45,98 @@ class ImageCachingThread(Thread): self.queue = ARTWORK_QUEUE Thread.__init__(self) + def isCanceled(self): + return state.STOP_PKC + + def isSuspended(self): + return any(IMAGE_CACHING_SUSPENDS) + + @staticmethod + def _art_url_generator(): + from . import kodidb_functions as kodidb + for kind in ('video', 'music'): + with kodidb.GetKodiDB(kind) as kodi_db: + for kodi_type in ('poster', 'fanart'): + for url in kodi_db.artwork_generator(kodi_type): + yield url + + def missing_art_cache_generator(self): + from . import kodidb_functions as kodidb + with kodidb.GetKodiDB('texture') as kodi_db: + for url in self._art_url_generator(): + if kodi_db.url_not_yet_cached(url): + yield url + + def _cache_url(self, url): + url = double_urlencode(utils.try_encode(url)) + sleeptime = 0 + while True: + try: + requests.head( + url="http://%s:%s/image/image://%s" + % (state.WEBSERVER_HOST, + state.WEBSERVER_PORT, + url), + auth=(state.WEBSERVER_USERNAME, + state.WEBSERVER_PASSWORD), + timeout=self.timeout) + except requests.Timeout: + # We don't need the result, only trigger Kodi to start the + # download. All is well + break + except requests.ConnectionError: + if self.isCanceled(): + # Kodi terminated + break + # Server thinks its a DOS attack, ('error 10053') + # Wait before trying again + if sleeptime > 5: + LOG.error('Repeatedly got ConnectionError for url %s', + double_urldecode(url)) + break + LOG.debug('Were trying too hard to download art, server ' + 'over-loaded. Sleep %s seconds before trying ' + 'again to download %s', + 2**sleeptime, double_urldecode(url)) + xbmc.sleep((2**sleeptime) * 1000) + sleeptime += 1 + continue + except Exception as err: + LOG.error('Unknown exception for url %s: %s'. + double_urldecode(url), err) + import traceback + LOG.error("Traceback:\n%s", traceback.format_exc()) + break + # We did not even get a timeout + break + def run(self): LOG.info("---===### Starting ImageCachingThread ###===---") - stopped = self.stopped - suspended = self.suspended - queue = self.queue - sleep_between = self.sleep_between - while not stopped(): - # In the event the server goes offline - while suspended(): + # Cache already synced artwork first + for url in self.missing_art_cache_generator(): + if self.isCanceled(): + return + while self.isSuspended(): # Set in service.py - if stopped(): + if self.isCanceled(): # Abort was requested while waiting. We should exit LOG.info("---===### Stopped ImageCachingThread ###===---") return xbmc.sleep(1000) + self._cache_url(url) + # Now wait for more stuff to cache - via the queue + while not self.isCanceled(): + # In the event the server goes offline + while self.isSuspended(): + # Set in service.py + if self.isCanceled(): + # Abort was requested while waiting. We should exit + LOG.info("---===### Stopped ImageCachingThread ###===---") + return + xbmc.sleep(1000) try: - url = queue.get(block=False) + url = self.queue.get(block=False) except Empty: xbmc.sleep(1000) continue @@ -73,52 +147,12 @@ class ImageCachingThread(Thread): message=url.message, icon='{plex}', sound=False) - queue.task_done() + self.queue.task_done() continue - url = double_urlencode(utils.try_encode(url)) - sleeptime = 0 - while True: - try: - requests.head( - url="http://%s:%s/image/image://%s" - % (state.WEBSERVER_HOST, - state.WEBSERVER_PORT, - url), - auth=(state.WEBSERVER_USERNAME, - state.WEBSERVER_PASSWORD), - timeout=self.timeout) - except requests.Timeout: - # We don't need the result, only trigger Kodi to start the - # download. All is well - break - except requests.ConnectionError: - if stopped(): - # Kodi terminated - break - # Server thinks its a DOS attack, ('error 10053') - # Wait before trying again - if sleeptime > 5: - LOG.error('Repeatedly got ConnectionError for url %s', - double_urldecode(url)) - break - LOG.debug('Were trying too hard to download art, server ' - 'over-loaded. Sleep %s seconds before trying ' - 'again to download %s', - 2**sleeptime, double_urldecode(url)) - xbmc.sleep((2**sleeptime) * 1000) - sleeptime += 1 - continue - except Exception as err: - LOG.error('Unknown exception for url %s: %s'. - double_urldecode(url), err) - import traceback - LOG.error("Traceback:\n%s", traceback.format_exc()) - break - # We did not even get a timeout - break - queue.task_done() + self._cache_url(url) + self.queue.task_done() # Sleep for a bit to reduce CPU strain - xbmc.sleep(sleep_between) + xbmc.sleep(100) LOG.info("---===### Stopped ImageCachingThread ###===---") @@ -127,47 +161,6 @@ class Artwork(): if enableTextureCache: queue = ARTWORK_QUEUE - def cache_major_artwork(self): - """ - Takes the existing Kodi library and caches posters and fanart. - Necessary because otherwise PKC caches artwork e.g. from fanart.tv - which basically blocks Kodi from getting needed artwork fast (e.g. - while browsing the library) - """ - if not self.enableTextureCache: - return - artworks = list() - # Get all posters and fanart/background for video and music - for kind in ('video', 'music'): - connection = utils.kodi_sql(kind) - cursor = connection.cursor() - for typus in ('poster', 'fanart'): - cursor.execute('SELECT url FROM art WHERE type == ?', - (typus, )) - artworks.extend(cursor.fetchall()) - connection.close() - artworks_to_cache = list() - connection = utils.kodi_sql('texture') - cursor = connection.cursor() - for url in artworks: - query = 'SELECT url FROM texture WHERE url == ? LIMIT 1' - cursor.execute(query, (url[0], )) - if not cursor.fetchone(): - artworks_to_cache.append(url) - connection.close() - if not artworks_to_cache: - LOG.info('Caching of major images to Kodi texture cache done') - return - length = len(artworks_to_cache) - LOG.info('Caching has not been completed - caching %s major images', - length) - # Caching %s Plex images - self.queue.put(ArtworkSyncMessage(utils.lang(30006) % length)) - for url in artworks_to_cache: - self.queue.put(url[0]) - # Plex image caching done - self.queue.put(ArtworkSyncMessage(utils.lang(30007))) - def fullTextureCacheSync(self): """ This method will sync all Kodi artwork to textures13.db diff --git a/resources/lib/kodidb_functions.py b/resources/lib/kodidb_functions.py index 13fd5512..7dd9c7cc 100644 --- a/resources/lib/kodidb_functions.py +++ b/resources/lib/kodidb_functions.py @@ -1199,6 +1199,21 @@ class KodiDBMethods(object): ''' self.cursor.execute(query, (kodi_id, kodi_type)) + def artwork_generator(self, kodi_type): + """ + """ + self.cursor.execute('SELECT url FROM art WHERE type == ?', + (kodi_type, )) + return (x[0] for x in self.cursor) + + def url_not_yet_cached(self, url): + """ + Returns True if url has not yet been cached to the Kodi texture cache + """ + self.cursor.execute('SELECT url FROM texture WHERE url == ? LIMIT 1', + (url, )) + return self.cursor.fetchone() is None + def kodiid_from_filename(path, kodi_type=None, db_type=None): """ diff --git a/resources/lib/sync.py b/resources/lib/sync.py index 010ed103..10e54c7f 100644 --- a/resources/lib/sync.py +++ b/resources/lib/sync.py @@ -35,8 +35,7 @@ class Sync(backgroundthread.KillableThread): self.fanart = None # Show sync dialog even if user deactivated? self.force_dialog = False - if utils.settings('enableTextureCache') == "true": - self.image_cache_thread = artwork.ImageCachingThread() + self.image_cache_thread = None # Lock used to wait on a full sync, e.g. on initial sync # self.lock = backgroundthread.threading.Lock() super(Sync, self).__init__() @@ -162,6 +161,13 @@ class Sync(backgroundthread.KillableThread): icon='{plex}', sound=False) + def start_image_cache_thread(self): + if utils.settings('enableTextureCache') == "true": + self.image_cache_thread = artwork.ImageCachingThread() + self.image_cache_thread.start() + else: + LOG.info('Image caching has been deactivated') + def run(self): try: self._run_internal() @@ -250,7 +256,7 @@ class Sync(backgroundthread.KillableThread): from . import playlists playlist_monitor = playlists.kodi_playlist_monitor() self.start_fanart_download(refresh=False) - self.image_cache_thread.start() + self.start_image_cache_thread() else: LOG.error('Initial start-up full sync unsuccessful') self.force_dialog = False @@ -271,9 +277,8 @@ class Sync(backgroundthread.KillableThread): if library_sync.PLAYLIST_SYNC_ENABLED: from . import playlists playlist_monitor = playlists.kodi_playlist_monitor() - artwork.Artwork().cache_major_artwork() self.start_fanart_download(refresh=False) - self.image_cache_thread.start() + self.start_image_cache_thread() else: LOG.info('Startup sync has not yet been successful')