add a thread pool option to the image cache
This commit is contained in:
Shaun
parent
82f117222a
commit
9e2f789e53
5 changed files with 173 additions and 59 deletions
|
|
@ -1,7 +1,7 @@
|
|||
<?xml version="1.0" encoding="UTF-8" standalone="yes"?>
|
||||
<addon id="plugin.video.emby"
|
||||
name="Emby"
|
||||
version="1.1.72"
|
||||
version="1.1.75"
|
||||
provider-name="Emby.media">
|
||||
<requires>
|
||||
<import addon="xbmc.python" version="2.1.0"/>
|
||||
|
|
|
|||
|
|
@ -8,10 +8,12 @@ import os
|
|||
import urllib
|
||||
|
||||
import xbmc
|
||||
import xbmcgui
|
||||
import xbmcvfs
|
||||
|
||||
import utils
|
||||
import clientinfo
|
||||
import image_cache_thread
|
||||
|
||||
#################################################################################################
|
||||
|
||||
|
|
@ -23,13 +25,18 @@ class Artwork():
|
|||
xbmc_username = None
|
||||
xbmc_password = None
|
||||
|
||||
imageCacheThreads = []
|
||||
imageCacheLimitThreads = 0
|
||||
|
||||
def __init__(self):
|
||||
|
||||
self.clientinfo = clientinfo.ClientInfo()
|
||||
self.addonName = self.clientinfo.getAddonName()
|
||||
|
||||
self.enableTextureCache = utils.settings('enableTextureCache') == "true"
|
||||
self.imageCacheLimitThreads = int(utils.settings("imageCacheLimit"))
|
||||
self.imageCacheLimitThreads = int(self.imageCacheLimitThreads * 5);
|
||||
utils.logMsg("Using Image Cache Thread Count: " + str(self.imageCacheLimitThreads), 1)
|
||||
|
||||
if not self.xbmc_port and self.enableTextureCache:
|
||||
self.setKodiWebServerDetails()
|
||||
|
||||
|
|
@ -37,7 +44,6 @@ class Artwork():
|
|||
self.server = utils.window('emby_server%s' % self.userId)
|
||||
|
||||
def logMsg(self, msg, lvl=1):
|
||||
|
||||
className = self.__class__.__name__
|
||||
utils.logMsg("%s %s" % (self.addonName, className), msg, lvl)
|
||||
|
||||
|
|
@ -159,63 +165,130 @@ class Artwork():
|
|||
def FullTextureCacheSync(self):
|
||||
# This method will sync all Kodi artwork to textures13.db
|
||||
# and cache them locally. This takes diskspace!
|
||||
|
||||
# Remove all existing textures first
|
||||
path = xbmc.translatePath("special://thumbnails/").decode('utf-8')
|
||||
if xbmcvfs.exists(path):
|
||||
allDirs, allFiles = xbmcvfs.listdir(path)
|
||||
for dir in allDirs:
|
||||
allDirs, allFiles = xbmcvfs.listdir(path+dir)
|
||||
for file in allFiles:
|
||||
xbmcvfs.delete(os.path.join(path+dir,file))
|
||||
|
||||
textureconnection = utils.KodiSQL('texture')
|
||||
texturecursor = textureconnection.cursor()
|
||||
texturecursor.execute('SELECT tbl_name FROM sqlite_master WHERE type="table"')
|
||||
rows = texturecursor.fetchall()
|
||||
for row in rows:
|
||||
tableName = row[0]
|
||||
if(tableName != "version"):
|
||||
texturecursor.execute("DELETE FROM " + tableName)
|
||||
textureconnection.commit()
|
||||
texturecursor.close()
|
||||
if not xbmcgui.Dialog().yesno("Image Texture Cache", "Running the image cache process can take some time.", "Are you sure you want continue?"):
|
||||
return
|
||||
|
||||
|
||||
self.logMsg("Doing Image Cache Sync", 1)
|
||||
|
||||
dialog = xbmcgui.DialogProgressBG()
|
||||
dialog.create("Emby for Kodi", "Image Cache Sync")
|
||||
|
||||
# ask to rest all existing or not
|
||||
if xbmcgui.Dialog().yesno("Image Texture Cache", "Reset all existing cache data first?", ""):
|
||||
self.logMsg("Resetting all cache data first", 1)
|
||||
# Remove all existing textures first
|
||||
path = xbmc.translatePath("special://thumbnails/").decode('utf-8')
|
||||
if xbmcvfs.exists(path):
|
||||
allDirs, allFiles = xbmcvfs.listdir(path)
|
||||
for dir in allDirs:
|
||||
allDirs, allFiles = xbmcvfs.listdir(path+dir)
|
||||
for file in allFiles:
|
||||
xbmcvfs.delete(os.path.join(path+dir,file))
|
||||
|
||||
# remove all existing data from texture DB
|
||||
textureconnection = utils.kodiSQL('texture')
|
||||
texturecursor = textureconnection.cursor()
|
||||
texturecursor.execute('SELECT tbl_name FROM sqlite_master WHERE type="table"')
|
||||
rows = texturecursor.fetchall()
|
||||
for row in rows:
|
||||
tableName = row[0]
|
||||
if(tableName != "version"):
|
||||
texturecursor.execute("DELETE FROM " + tableName)
|
||||
textureconnection.commit()
|
||||
texturecursor.close()
|
||||
|
||||
# Cache all entries in video DB
|
||||
connection = utils.KodiSQL('video')
|
||||
connection = utils.kodiSQL('video')
|
||||
cursor = connection.cursor()
|
||||
cursor.execute("SELECT url FROM art")
|
||||
cursor.execute("SELECT url FROM art WHERE media_type != 'actor'") # dont include actors
|
||||
result = cursor.fetchall()
|
||||
total = len(result)
|
||||
count = 1
|
||||
percentage = 0.0
|
||||
self.logMsg("Image cache sync about to process " + str(total) + " images", 1)
|
||||
for url in result:
|
||||
percentage = int((float(count) / float(total))*100)
|
||||
textMessage = str(count) + " of " + str(total) + " (" + str(len(self.imageCacheThreads)) + ")"
|
||||
dialog.update(percentage, message="Updating Image Cache: " + textMessage)
|
||||
self.CacheTexture(url[0])
|
||||
cursor.close()
|
||||
count += 1
|
||||
cursor.close()
|
||||
|
||||
# Cache all entries in music DB
|
||||
connection = utils.KodiSQL('music')
|
||||
connection = utils.kodiSQL('music')
|
||||
cursor = connection.cursor()
|
||||
cursor.execute("SELECT url FROM art")
|
||||
result = cursor.fetchall()
|
||||
total = len(result)
|
||||
count = 1
|
||||
percentage = 0.0
|
||||
self.logMsg("Image cache sync about to process " + str(total) + " images", 1)
|
||||
for url in result:
|
||||
percentage = int((float(count) / float(total))*100)
|
||||
textMessage = str(count) + " of " + str(total)
|
||||
dialog.update(percentage, message="Updating Image Cache: " + textMessage)
|
||||
self.CacheTexture(url[0])
|
||||
count += 1
|
||||
cursor.close()
|
||||
|
||||
dialog.update(percentage, message="Waiting for all threads to exit: " + str(len(self.imageCacheThreads)))
|
||||
self.logMsg("Waiting for all threads to exit", 1)
|
||||
while len(self.imageCacheThreads) > 0:
|
||||
for thread in self.imageCacheThreads:
|
||||
if thread.isFinished:
|
||||
self.imageCacheThreads.remove(thread)
|
||||
dialog.update(percentage, message="Waiting for all threads to exit: " + str(len(self.imageCacheThreads)))
|
||||
self.logMsg("Waiting for all threads to exit: " + str(len(self.imageCacheThreads)), 1)
|
||||
xbmc.sleep(500)
|
||||
|
||||
dialog.close()
|
||||
|
||||
def addWorkerImageCacheThread(self, urlToAdd):
|
||||
|
||||
while(True):
|
||||
# removed finished
|
||||
for thread in self.imageCacheThreads:
|
||||
if thread.isFinished:
|
||||
self.imageCacheThreads.remove(thread)
|
||||
|
||||
# add a new thread or wait and retry if we hit our limit
|
||||
if(len(self.imageCacheThreads) < self.imageCacheLimitThreads):
|
||||
newThread = image_cache_thread.image_cache_thread()
|
||||
newThread.setUrl(self.double_urlencode(urlToAdd))
|
||||
newThread.setHost(self.xbmc_host, self.xbmc_port)
|
||||
newThread.setAuth(self.xbmc_username, self.xbmc_password)
|
||||
newThread.start()
|
||||
self.imageCacheThreads.append(newThread)
|
||||
return
|
||||
else:
|
||||
self.logMsg("Waiting for empty queue spot: " + str(len(self.imageCacheThreads)), 2)
|
||||
xbmc.sleep(50)
|
||||
|
||||
|
||||
def CacheTexture(self, url):
|
||||
# Cache a single image url to the texture cache
|
||||
if url and self.enableTextureCache:
|
||||
self.logMsg("Processing: %s" % url, 2)
|
||||
|
||||
if(self.imageCacheLimitThreads == 0 or self.imageCacheLimitThreads == None):
|
||||
#Add image to texture cache by simply calling it at the http endpoint
|
||||
|
||||
url = self.double_urlencode(url)
|
||||
try: # Extreme short timeouts so we will have a exception.
|
||||
response = requests.head(
|
||||
url=(
|
||||
"http://%s:%s/image/image://%s"
|
||||
% (self.xbmc_host, self.xbmc_port, url)),
|
||||
auth=(self.xbmc_username, self.xbmc_password),
|
||||
timeout=(0.01, 0.01))
|
||||
# We don't need the result
|
||||
except: pass
|
||||
|
||||
else:
|
||||
self.addWorkerImageCacheThread(url)
|
||||
|
||||
# Add image to texture cache by simply calling it at the http endpoint
|
||||
url = self.double_urlencode(url)
|
||||
try: # Extreme short timeouts so we will have a exception.
|
||||
response = requests.head(
|
||||
url=(
|
||||
"http://%s:%s/image/image://%s"
|
||||
% (self.xbmc_host, self.xbmc_port, url)),
|
||||
auth=(self.xbmc_username, self.xbmc_password),
|
||||
timeout=(0.01, 0.01))
|
||||
# We don't need the result
|
||||
except: pass
|
||||
|
||||
|
||||
def addArtwork(self, artwork, kodiId, mediaType, cursor):
|
||||
# Kodi conversion table
|
||||
|
|
|
|||
|
|
@ -72,27 +72,15 @@ def doMainListing():
|
|||
addDirectoryItem(label, path)
|
||||
|
||||
# some extra entries for settings and stuff. TODO --> localize the labels
|
||||
addDirectoryItem("Network credentials", "plugin://plugin.video.emby/?mode=passwords", False)
|
||||
addDirectoryItem("Settings", "plugin://plugin.video.emby/?mode=settings", False)
|
||||
addDirectoryItem("Add user to session", "plugin://plugin.video.emby/?mode=adduser", False)
|
||||
#addDirectoryItem("Cache all images to Kodi texture cache (advanced)", "plugin://plugin.video.emby/?mode=texturecache")
|
||||
addDirectoryItem(
|
||||
label="Refresh Emby playlists",
|
||||
path="plugin://plugin.video.emby/?mode=refreshplaylist",
|
||||
folder=False)
|
||||
addDirectoryItem("Perform manual sync", "plugin://plugin.video.emby/?mode=manualsync", False)
|
||||
addDirectoryItem(
|
||||
label="Repair local database (force update all content)",
|
||||
path="plugin://plugin.video.emby/?mode=repair",
|
||||
folder=False)
|
||||
addDirectoryItem(
|
||||
label="Perform local database reset (full resync)",
|
||||
path="plugin://plugin.video.emby/?mode=reset",
|
||||
folder=False)
|
||||
addDirectoryItem(
|
||||
label="Sync Emby Theme Media to Kodi",
|
||||
path="plugin://plugin.video.emby/?mode=thememedia",
|
||||
folder=False)
|
||||
addDirectoryItem("Network credentials", "plugin://plugin.video.emby/?mode=passwords")
|
||||
addDirectoryItem("Settings", "plugin://plugin.video.emby/?mode=settings")
|
||||
addDirectoryItem("Add user to session", "plugin://plugin.video.emby/?mode=adduser")
|
||||
addDirectoryItem("Refresh Emby playlists", "plugin://plugin.video.emby/?mode=refreshplaylist")
|
||||
addDirectoryItem("Perform manual sync", "plugin://plugin.video.emby/?mode=manualsync")
|
||||
addDirectoryItem("Repair local database (force update all content)", "plugin://plugin.video.emby/?mode=repair")
|
||||
addDirectoryItem("Perform local database reset (full resync)", "plugin://plugin.video.emby/?mode=reset")
|
||||
addDirectoryItem("Cache all images to Kodi texture cache", "plugin://plugin.video.emby/?mode=texturecache")
|
||||
addDirectoryItem("Sync Emby Theme Media to Kodi", "plugin://plugin.video.emby/?mode=thememedia")
|
||||
|
||||
xbmcplugin.endOfDirectory(int(sys.argv[1]))
|
||||
|
||||
|
|
|
|||
52
resources/lib/image_cache_thread.py
Normal file
52
resources/lib/image_cache_thread.py
Normal file
|
|
@ -0,0 +1,52 @@
|
|||
import threading
|
||||
import utils
|
||||
import xbmc
|
||||
import requests
|
||||
|
||||
class image_cache_thread(threading.Thread):
|
||||
|
||||
urlToProcess = None
|
||||
isFinished = False
|
||||
|
||||
xbmc_host = ""
|
||||
xbmc_port = ""
|
||||
xbmc_username = ""
|
||||
xbmc_password = ""
|
||||
|
||||
def __init__(self):
|
||||
self.monitor = xbmc.Monitor()
|
||||
threading.Thread.__init__(self)
|
||||
|
||||
def logMsg(self, msg, lvl=1):
|
||||
className = self.__class__.__name__
|
||||
utils.logMsg("%s" % className, msg, lvl)
|
||||
|
||||
def setUrl(self, url):
|
||||
self.urlToProcess = url
|
||||
|
||||
def setHost(self, host, port):
|
||||
self.xbmc_host = host
|
||||
self.xbmc_port = port
|
||||
|
||||
def setAuth(self, user, pwd):
|
||||
self.xbmc_username = user
|
||||
self.xbmc_password = pwd
|
||||
|
||||
def run(self):
|
||||
|
||||
self.logMsg("Image Caching Thread Processing : " + self.urlToProcess, 2)
|
||||
|
||||
try:
|
||||
response = requests.head(
|
||||
url=(
|
||||
"http://%s:%s/image/image://%s"
|
||||
% (self.xbmc_host, self.xbmc_port, self.urlToProcess)),
|
||||
auth=(self.xbmc_username, self.xbmc_password),
|
||||
timeout=(35.1, 35.1))
|
||||
# We don't need the result
|
||||
except: pass
|
||||
|
||||
self.logMsg("Image Caching Thread Exited", 2)
|
||||
|
||||
self.isFinished = True
|
||||
|
||||
|
|
@ -31,6 +31,7 @@
|
|||
<setting id="streamMusic" type="bool" label="Direct stream music library" default="false" visible="eq(-1,true)" subsetting="true" />
|
||||
<setting id="useDirectPaths" type="enum" label="Playback Mode" values="Addon(Default)|Native(Direct paths)" default="0" />
|
||||
<setting id="enableTextureCache" label="Auto add images to the Kodi texture cache" type="bool" default="true" />
|
||||
<setting id="imageCacheLimit" type="enum" label="Limit image cache import threads" values="None|5|10|15|20|25" default="0" visible="eq(-1,true)"/>
|
||||
<setting id="serverSync" type="bool" label="Enable fast starting sync" default="true" />
|
||||
<setting id="limitindex" type="number" label="Maximum items to request at once from server" default="200" option="int" />
|
||||
</category>
|
||||
|
|
|
|||
Loading…
Reference in a new issue