add a thread pool option to the image cache

addon.xml

@@ -1,7 +1,7 @@
 <?xml version="1.0" encoding="UTF-8" standalone="yes"?>
 <addon  id="plugin.video.emby"
         name="Emby" 
-        version="1.1.72"
+        version="1.1.75"
         provider-name="Emby.media">
   <requires>
     <import addon="xbmc.python" version="2.1.0"/>

resources/lib/artwork.py

@@ -8,10 +8,12 @@ import os
 import urllib
 
 import xbmc
+import xbmcgui
 import xbmcvfs
 
 import utils
 import clientinfo
+import image_cache_thread
 
 #################################################################################################
 
@@ -23,13 +25,18 @@ class Artwork():
     xbmc_username = None
     xbmc_password = None
     
+    imageCacheThreads = []
+    imageCacheLimitThreads = 0
 
     def __init__(self):
-        
         self.clientinfo = clientinfo.ClientInfo()
         self.addonName = self.clientinfo.getAddonName()
 
         self.enableTextureCache = utils.settings('enableTextureCache') == "true"
+        self.imageCacheLimitThreads = int(utils.settings("imageCacheLimit"))
+        self.imageCacheLimitThreads = int(self.imageCacheLimitThreads * 5);
+        utils.logMsg("Using Image Cache Thread Count: " + str(self.imageCacheLimitThreads), 1)
+        
         if not self.xbmc_port and self.enableTextureCache:
             self.setKodiWebServerDetails()
 
@@ -37,7 +44,6 @@ class Artwork():
         self.server = utils.window('emby_server%s' % self.userId)
 
     def logMsg(self, msg, lvl=1):
-
         className = self.__class__.__name__
         utils.logMsg("%s %s" % (self.addonName, className), msg, lvl)
     
@@ -159,63 +165,130 @@ class Artwork():
     def FullTextureCacheSync(self):
         # This method will sync all Kodi artwork to textures13.db
         # and cache them locally. This takes diskspace!
-
-        # Remove all existing textures first
-        path = xbmc.translatePath("special://thumbnails/").decode('utf-8')
-        if xbmcvfs.exists(path):
-            allDirs, allFiles = xbmcvfs.listdir(path)
-            for dir in allDirs:
-                allDirs, allFiles = xbmcvfs.listdir(path+dir)
-                for file in allFiles:
-                    xbmcvfs.delete(os.path.join(path+dir,file))
         
-        textureconnection = utils.KodiSQL('texture')
+        if not xbmcgui.Dialog().yesno("Image Texture Cache", "Running the image cache process can take some time.", "Are you sure you want continue?"):
-        texturecursor = textureconnection.cursor()
+            return
-        texturecursor.execute('SELECT tbl_name FROM sqlite_master WHERE type="table"')
+            
-        rows = texturecursor.fetchall()
-        for row in rows:
-            tableName = row[0]
-            if(tableName != "version"):
-                texturecursor.execute("DELETE FROM " + tableName)
-        textureconnection.commit()
-        texturecursor.close()
 
+        self.logMsg("Doing Image Cache Sync", 1)
         
+        dialog = xbmcgui.DialogProgressBG()
+        dialog.create("Emby for Kodi", "Image Cache Sync")
+            
+        # ask to rest all existing or not
+        if xbmcgui.Dialog().yesno("Image Texture Cache", "Reset all existing cache data first?", ""):
+            self.logMsg("Resetting all cache data first", 1)
+            # Remove all existing textures first
+            path = xbmc.translatePath("special://thumbnails/").decode('utf-8')
+            if xbmcvfs.exists(path):
+                allDirs, allFiles = xbmcvfs.listdir(path)
+                for dir in allDirs:
+                    allDirs, allFiles = xbmcvfs.listdir(path+dir)
+                    for file in allFiles:
+                        xbmcvfs.delete(os.path.join(path+dir,file))
+            
+            # remove all existing data from texture DB
+            textureconnection = utils.kodiSQL('texture')
+            texturecursor = textureconnection.cursor()
+            texturecursor.execute('SELECT tbl_name FROM sqlite_master WHERE type="table"')
+            rows = texturecursor.fetchall()
+            for row in rows:
+                tableName = row[0]
+                if(tableName != "version"):
+                    texturecursor.execute("DELETE FROM " + tableName)
+            textureconnection.commit()
+            texturecursor.close()
+
         # Cache all entries in video DB
-        connection = utils.KodiSQL('video')
+        connection = utils.kodiSQL('video')
         cursor = connection.cursor()
-        cursor.execute("SELECT url FROM art")
+        cursor.execute("SELECT url FROM art WHERE media_type != 'actor'") # dont include actors
         result = cursor.fetchall()
+        total = len(result)
+        count = 1     
+        percentage = 0.0    
+        self.logMsg("Image cache sync about to process " + str(total) + " images", 1)
         for url in result:
+            percentage = int((float(count) / float(total))*100)
+            textMessage = str(count) + " of " + str(total) + " (" + str(len(self.imageCacheThreads)) + ")"
+            dialog.update(percentage, message="Updating Image Cache: " + textMessage)
             self.CacheTexture(url[0])
-        cursor.close()    
+            count += 1
+        cursor.close()
         
         # Cache all entries in music DB
-        connection = utils.KodiSQL('music')
+        connection = utils.kodiSQL('music')
         cursor = connection.cursor()
         cursor.execute("SELECT url FROM art")
         result = cursor.fetchall()
+        total = len(result)
+        count = 1     
+        percentage = 0.0
+        self.logMsg("Image cache sync about to process " + str(total) + " images", 1)
         for url in result:
+            percentage = int((float(count) / float(total))*100)
+            textMessage = str(count) + " of " + str(total)
+            dialog.update(percentage, message="Updating Image Cache: " + textMessage)
             self.CacheTexture(url[0])
+            count += 1
         cursor.close()
+        
+        dialog.update(percentage, message="Waiting for all threads to exit: " + str(len(self.imageCacheThreads)))
+        self.logMsg("Waiting for all threads to exit", 1)
+        while len(self.imageCacheThreads) > 0:
+            for thread in self.imageCacheThreads:
+                if thread.isFinished:
+                    self.imageCacheThreads.remove(thread)
+            dialog.update(percentage, message="Waiting for all threads to exit: " + str(len(self.imageCacheThreads)))
+            self.logMsg("Waiting for all threads to exit: " + str(len(self.imageCacheThreads)), 1)
+            xbmc.sleep(500)
+        
+        dialog.close()
 
+    def addWorkerImageCacheThread(self, urlToAdd):
+    
+        while(True):
+            # removed finished
+            for thread in self.imageCacheThreads:
+                if thread.isFinished:
+                    self.imageCacheThreads.remove(thread)    
+
+            # add a new thread or wait and retry if we hit our limit
+            if(len(self.imageCacheThreads) < self.imageCacheLimitThreads):
+                newThread = image_cache_thread.image_cache_thread()
+                newThread.setUrl(self.double_urlencode(urlToAdd))
+                newThread.setHost(self.xbmc_host, self.xbmc_port)
+                newThread.setAuth(self.xbmc_username, self.xbmc_password)
+                newThread.start()
+                self.imageCacheThreads.append(newThread)
+                return
+            else:
+                self.logMsg("Waiting for empty queue spot: " + str(len(self.imageCacheThreads)), 2)
+                xbmc.sleep(50)
+         
+        
     def CacheTexture(self, url):
         # Cache a single image url to the texture cache
         if url and self.enableTextureCache:
             self.logMsg("Processing: %s" % url, 2)
+            
+            if(self.imageCacheLimitThreads == 0 or self.imageCacheLimitThreads == None):
+                #Add image to texture cache by simply calling it at the http endpoint
+                
+                url = self.double_urlencode(url)
+                try: # Extreme short timeouts so we will have a exception.
+                    response = requests.head(
+                                        url=(
+                                            "http://%s:%s/image/image://%s"
+                                            % (self.xbmc_host, self.xbmc_port, url)),
+                                        auth=(self.xbmc_username, self.xbmc_password),
+                                        timeout=(0.01, 0.01))
+                # We don't need the result
+                except: pass
+                
+            else:
+                self.addWorkerImageCacheThread(url)
 
-            # Add image to texture cache by simply calling it at the http endpoint
-            url = self.double_urlencode(url)
-            try: # Extreme short timeouts so we will have a exception.
-                response = requests.head(
-                                    url=(
-                                        "http://%s:%s/image/image://%s"
-                                        % (self.xbmc_host, self.xbmc_port, url)),
-                                    auth=(self.xbmc_username, self.xbmc_password),
-                                    timeout=(0.01, 0.01))
-            # We don't need the result
-            except: pass
-    
     
     def addArtwork(self, artwork, kodiId, mediaType, cursor):
         # Kodi conversion table

resources/lib/entrypoint.py

@@ -72,27 +72,15 @@ def doMainListing():
                 addDirectoryItem(label, path)
     
     # some extra entries for settings and stuff. TODO --> localize the labels
-    addDirectoryItem("Network credentials", "plugin://plugin.video.emby/?mode=passwords", False)
+    addDirectoryItem("Network credentials", "plugin://plugin.video.emby/?mode=passwords")
-    addDirectoryItem("Settings", "plugin://plugin.video.emby/?mode=settings", False)
+    addDirectoryItem("Settings", "plugin://plugin.video.emby/?mode=settings")
-    addDirectoryItem("Add user to session", "plugin://plugin.video.emby/?mode=adduser", False)
+    addDirectoryItem("Add user to session", "plugin://plugin.video.emby/?mode=adduser")
-    #addDirectoryItem("Cache all images to Kodi texture cache (advanced)", "plugin://plugin.video.emby/?mode=texturecache")
+    addDirectoryItem("Refresh Emby playlists", "plugin://plugin.video.emby/?mode=refreshplaylist")
-    addDirectoryItem(
+    addDirectoryItem("Perform manual sync", "plugin://plugin.video.emby/?mode=manualsync")
-        label="Refresh Emby playlists",
+    addDirectoryItem("Repair local database (force update all content)", "plugin://plugin.video.emby/?mode=repair")
-        path="plugin://plugin.video.emby/?mode=refreshplaylist",
+    addDirectoryItem("Perform local database reset (full resync)", "plugin://plugin.video.emby/?mode=reset")
-        folder=False)
+    addDirectoryItem("Cache all images to Kodi texture cache", "plugin://plugin.video.emby/?mode=texturecache")
-    addDirectoryItem("Perform manual sync", "plugin://plugin.video.emby/?mode=manualsync", False)
+    addDirectoryItem("Sync Emby Theme Media to Kodi", "plugin://plugin.video.emby/?mode=thememedia")
-    addDirectoryItem(
-        label="Repair local database (force update all content)",
-        path="plugin://plugin.video.emby/?mode=repair",
-        folder=False)
-    addDirectoryItem(
-        label="Perform local database reset (full resync)",
-        path="plugin://plugin.video.emby/?mode=reset",
-        folder=False)
-    addDirectoryItem(
-        label="Sync Emby Theme Media to Kodi",
-        path="plugin://plugin.video.emby/?mode=thememedia",
-        folder=False)
     
     xbmcplugin.endOfDirectory(int(sys.argv[1]))
 

resources/lib/image_cache_thread.py

@@ -0,0 +1,52 @@
+import threading
+import utils
+import xbmc
+import requests
+
+class image_cache_thread(threading.Thread):
+
+    urlToProcess = None
+    isFinished = False
+    
+    xbmc_host = ""
+    xbmc_port = ""
+    xbmc_username = ""
+    xbmc_password = ""
+    
+    def __init__(self):
+        self.monitor = xbmc.Monitor()
+        threading.Thread.__init__(self)
+         
+    def logMsg(self, msg, lvl=1):
+        className = self.__class__.__name__
+        utils.logMsg("%s" % className, msg, lvl)
+        
+    def setUrl(self, url):
+        self.urlToProcess = url
+        
+    def setHost(self, host, port):
+        self.xbmc_host = host
+        self.xbmc_port = port
+        
+    def setAuth(self, user, pwd):
+        self.xbmc_username = user
+        self.xbmc_password = pwd
+         
+    def run(self):
+        
+        self.logMsg("Image Caching Thread Processing : " + self.urlToProcess, 2)
+        
+        try:
+            response = requests.head(
+                                url=(
+                                    "http://%s:%s/image/image://%s"
+                                    % (self.xbmc_host, self.xbmc_port, self.urlToProcess)),
+                                auth=(self.xbmc_username, self.xbmc_password),
+                                timeout=(35.1, 35.1))
+        # We don't need the result
+        except: pass
+        
+        self.logMsg("Image Caching Thread Exited", 2)
+        
+        self.isFinished = True
+        

resources/settings.xml

@@ -31,6 +31,7 @@
 		<setting id="streamMusic" type="bool" label="Direct stream music library" default="false" visible="eq(-1,true)" subsetting="true" />
 		<setting id="useDirectPaths" type="enum" label="Playback Mode" values="Addon(Default)|Native(Direct paths)" default="0" />
 		<setting id="enableTextureCache" label="Auto add images to the Kodi texture cache"  type="bool" default="true" />
+        <setting id="imageCacheLimit" type="enum" label="Limit image cache import threads" values="None|5|10|15|20|25" default="0" visible="eq(-1,true)"/>
 		<setting id="serverSync" type="bool" label="Enable fast starting sync" default="true" />
 		<setting id="limitindex" type="number" label="Maximum items to request at once from server" default="200" option="int" />
 	</category>