From 9e2f789e53c3f2f777c5d89c9ac1ad683fa778bf Mon Sep 17 00:00:00 2001 From: Shaun Date: Sat, 16 Jan 2016 14:07:36 +1100 Subject: [PATCH] add a thread pool option to the image cache --- addon.xml | 2 +- resources/lib/artwork.py | 147 +++++++++++++++++++++------- resources/lib/entrypoint.py | 30 ++---- resources/lib/image_cache_thread.py | 52 ++++++++++ resources/settings.xml | 1 + 5 files changed, 173 insertions(+), 59 deletions(-) create mode 100644 resources/lib/image_cache_thread.py diff --git a/addon.xml b/addon.xml index d0f562d3..1eb161ed 100644 --- a/addon.xml +++ b/addon.xml @@ -1,7 +1,7 @@ diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index 5952c204..139ebdae 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -8,10 +8,12 @@ import os import urllib import xbmc +import xbmcgui import xbmcvfs import utils import clientinfo +import image_cache_thread ################################################################################################# @@ -23,13 +25,18 @@ class Artwork(): xbmc_username = None xbmc_password = None + imageCacheThreads = [] + imageCacheLimitThreads = 0 def __init__(self): - self.clientinfo = clientinfo.ClientInfo() self.addonName = self.clientinfo.getAddonName() self.enableTextureCache = utils.settings('enableTextureCache') == "true" + self.imageCacheLimitThreads = int(utils.settings("imageCacheLimit")) + self.imageCacheLimitThreads = int(self.imageCacheLimitThreads * 5); + utils.logMsg("Using Image Cache Thread Count: " + str(self.imageCacheLimitThreads), 1) + if not self.xbmc_port and self.enableTextureCache: self.setKodiWebServerDetails() @@ -37,7 +44,6 @@ class Artwork(): self.server = utils.window('emby_server%s' % self.userId) def logMsg(self, msg, lvl=1): - className = self.__class__.__name__ utils.logMsg("%s %s" % (self.addonName, className), msg, lvl) @@ -159,63 +165,130 @@ class Artwork(): def FullTextureCacheSync(self): # This method will sync all Kodi artwork to textures13.db # and cache them locally. This takes diskspace! - - # Remove all existing textures first - path = xbmc.translatePath("special://thumbnails/").decode('utf-8') - if xbmcvfs.exists(path): - allDirs, allFiles = xbmcvfs.listdir(path) - for dir in allDirs: - allDirs, allFiles = xbmcvfs.listdir(path+dir) - for file in allFiles: - xbmcvfs.delete(os.path.join(path+dir,file)) - textureconnection = utils.KodiSQL('texture') - texturecursor = textureconnection.cursor() - texturecursor.execute('SELECT tbl_name FROM sqlite_master WHERE type="table"') - rows = texturecursor.fetchall() - for row in rows: - tableName = row[0] - if(tableName != "version"): - texturecursor.execute("DELETE FROM " + tableName) - textureconnection.commit() - texturecursor.close() + if not xbmcgui.Dialog().yesno("Image Texture Cache", "Running the image cache process can take some time.", "Are you sure you want continue?"): + return + + self.logMsg("Doing Image Cache Sync", 1) + dialog = xbmcgui.DialogProgressBG() + dialog.create("Emby for Kodi", "Image Cache Sync") + + # ask to rest all existing or not + if xbmcgui.Dialog().yesno("Image Texture Cache", "Reset all existing cache data first?", ""): + self.logMsg("Resetting all cache data first", 1) + # Remove all existing textures first + path = xbmc.translatePath("special://thumbnails/").decode('utf-8') + if xbmcvfs.exists(path): + allDirs, allFiles = xbmcvfs.listdir(path) + for dir in allDirs: + allDirs, allFiles = xbmcvfs.listdir(path+dir) + for file in allFiles: + xbmcvfs.delete(os.path.join(path+dir,file)) + + # remove all existing data from texture DB + textureconnection = utils.kodiSQL('texture') + texturecursor = textureconnection.cursor() + texturecursor.execute('SELECT tbl_name FROM sqlite_master WHERE type="table"') + rows = texturecursor.fetchall() + for row in rows: + tableName = row[0] + if(tableName != "version"): + texturecursor.execute("DELETE FROM " + tableName) + textureconnection.commit() + texturecursor.close() + # Cache all entries in video DB - connection = utils.KodiSQL('video') + connection = utils.kodiSQL('video') cursor = connection.cursor() - cursor.execute("SELECT url FROM art") + cursor.execute("SELECT url FROM art WHERE media_type != 'actor'") # dont include actors result = cursor.fetchall() + total = len(result) + count = 1 + percentage = 0.0 + self.logMsg("Image cache sync about to process " + str(total) + " images", 1) for url in result: + percentage = int((float(count) / float(total))*100) + textMessage = str(count) + " of " + str(total) + " (" + str(len(self.imageCacheThreads)) + ")" + dialog.update(percentage, message="Updating Image Cache: " + textMessage) self.CacheTexture(url[0]) - cursor.close() + count += 1 + cursor.close() # Cache all entries in music DB - connection = utils.KodiSQL('music') + connection = utils.kodiSQL('music') cursor = connection.cursor() cursor.execute("SELECT url FROM art") result = cursor.fetchall() + total = len(result) + count = 1 + percentage = 0.0 + self.logMsg("Image cache sync about to process " + str(total) + " images", 1) for url in result: + percentage = int((float(count) / float(total))*100) + textMessage = str(count) + " of " + str(total) + dialog.update(percentage, message="Updating Image Cache: " + textMessage) self.CacheTexture(url[0]) + count += 1 cursor.close() + + dialog.update(percentage, message="Waiting for all threads to exit: " + str(len(self.imageCacheThreads))) + self.logMsg("Waiting for all threads to exit", 1) + while len(self.imageCacheThreads) > 0: + for thread in self.imageCacheThreads: + if thread.isFinished: + self.imageCacheThreads.remove(thread) + dialog.update(percentage, message="Waiting for all threads to exit: " + str(len(self.imageCacheThreads))) + self.logMsg("Waiting for all threads to exit: " + str(len(self.imageCacheThreads)), 1) + xbmc.sleep(500) + + dialog.close() + def addWorkerImageCacheThread(self, urlToAdd): + + while(True): + # removed finished + for thread in self.imageCacheThreads: + if thread.isFinished: + self.imageCacheThreads.remove(thread) + + # add a new thread or wait and retry if we hit our limit + if(len(self.imageCacheThreads) < self.imageCacheLimitThreads): + newThread = image_cache_thread.image_cache_thread() + newThread.setUrl(self.double_urlencode(urlToAdd)) + newThread.setHost(self.xbmc_host, self.xbmc_port) + newThread.setAuth(self.xbmc_username, self.xbmc_password) + newThread.start() + self.imageCacheThreads.append(newThread) + return + else: + self.logMsg("Waiting for empty queue spot: " + str(len(self.imageCacheThreads)), 2) + xbmc.sleep(50) + + def CacheTexture(self, url): # Cache a single image url to the texture cache if url and self.enableTextureCache: self.logMsg("Processing: %s" % url, 2) + + if(self.imageCacheLimitThreads == 0 or self.imageCacheLimitThreads == None): + #Add image to texture cache by simply calling it at the http endpoint + + url = self.double_urlencode(url) + try: # Extreme short timeouts so we will have a exception. + response = requests.head( + url=( + "http://%s:%s/image/image://%s" + % (self.xbmc_host, self.xbmc_port, url)), + auth=(self.xbmc_username, self.xbmc_password), + timeout=(0.01, 0.01)) + # We don't need the result + except: pass + + else: + self.addWorkerImageCacheThread(url) - # Add image to texture cache by simply calling it at the http endpoint - url = self.double_urlencode(url) - try: # Extreme short timeouts so we will have a exception. - response = requests.head( - url=( - "http://%s:%s/image/image://%s" - % (self.xbmc_host, self.xbmc_port, url)), - auth=(self.xbmc_username, self.xbmc_password), - timeout=(0.01, 0.01)) - # We don't need the result - except: pass - def addArtwork(self, artwork, kodiId, mediaType, cursor): # Kodi conversion table diff --git a/resources/lib/entrypoint.py b/resources/lib/entrypoint.py index 9afd5f41..2338a512 100644 --- a/resources/lib/entrypoint.py +++ b/resources/lib/entrypoint.py @@ -72,27 +72,15 @@ def doMainListing(): addDirectoryItem(label, path) # some extra entries for settings and stuff. TODO --> localize the labels - addDirectoryItem("Network credentials", "plugin://plugin.video.emby/?mode=passwords", False) - addDirectoryItem("Settings", "plugin://plugin.video.emby/?mode=settings", False) - addDirectoryItem("Add user to session", "plugin://plugin.video.emby/?mode=adduser", False) - #addDirectoryItem("Cache all images to Kodi texture cache (advanced)", "plugin://plugin.video.emby/?mode=texturecache") - addDirectoryItem( - label="Refresh Emby playlists", - path="plugin://plugin.video.emby/?mode=refreshplaylist", - folder=False) - addDirectoryItem("Perform manual sync", "plugin://plugin.video.emby/?mode=manualsync", False) - addDirectoryItem( - label="Repair local database (force update all content)", - path="plugin://plugin.video.emby/?mode=repair", - folder=False) - addDirectoryItem( - label="Perform local database reset (full resync)", - path="plugin://plugin.video.emby/?mode=reset", - folder=False) - addDirectoryItem( - label="Sync Emby Theme Media to Kodi", - path="plugin://plugin.video.emby/?mode=thememedia", - folder=False) + addDirectoryItem("Network credentials", "plugin://plugin.video.emby/?mode=passwords") + addDirectoryItem("Settings", "plugin://plugin.video.emby/?mode=settings") + addDirectoryItem("Add user to session", "plugin://plugin.video.emby/?mode=adduser") + addDirectoryItem("Refresh Emby playlists", "plugin://plugin.video.emby/?mode=refreshplaylist") + addDirectoryItem("Perform manual sync", "plugin://plugin.video.emby/?mode=manualsync") + addDirectoryItem("Repair local database (force update all content)", "plugin://plugin.video.emby/?mode=repair") + addDirectoryItem("Perform local database reset (full resync)", "plugin://plugin.video.emby/?mode=reset") + addDirectoryItem("Cache all images to Kodi texture cache", "plugin://plugin.video.emby/?mode=texturecache") + addDirectoryItem("Sync Emby Theme Media to Kodi", "plugin://plugin.video.emby/?mode=thememedia") xbmcplugin.endOfDirectory(int(sys.argv[1])) diff --git a/resources/lib/image_cache_thread.py b/resources/lib/image_cache_thread.py new file mode 100644 index 00000000..626be481 --- /dev/null +++ b/resources/lib/image_cache_thread.py @@ -0,0 +1,52 @@ +import threading +import utils +import xbmc +import requests + +class image_cache_thread(threading.Thread): + + urlToProcess = None + isFinished = False + + xbmc_host = "" + xbmc_port = "" + xbmc_username = "" + xbmc_password = "" + + def __init__(self): + self.monitor = xbmc.Monitor() + threading.Thread.__init__(self) + + def logMsg(self, msg, lvl=1): + className = self.__class__.__name__ + utils.logMsg("%s" % className, msg, lvl) + + def setUrl(self, url): + self.urlToProcess = url + + def setHost(self, host, port): + self.xbmc_host = host + self.xbmc_port = port + + def setAuth(self, user, pwd): + self.xbmc_username = user + self.xbmc_password = pwd + + def run(self): + + self.logMsg("Image Caching Thread Processing : " + self.urlToProcess, 2) + + try: + response = requests.head( + url=( + "http://%s:%s/image/image://%s" + % (self.xbmc_host, self.xbmc_port, self.urlToProcess)), + auth=(self.xbmc_username, self.xbmc_password), + timeout=(35.1, 35.1)) + # We don't need the result + except: pass + + self.logMsg("Image Caching Thread Exited", 2) + + self.isFinished = True + \ No newline at end of file diff --git a/resources/settings.xml b/resources/settings.xml index 5293dc61..e26d4564 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -31,6 +31,7 @@ +