From 497d71caccb09350546861a054781e501aea5eae Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 10 Sep 2016 11:27:27 +0200 Subject: [PATCH 01/86] Optimize sleep for librarysync threads --- resources/lib/librarysync.py | 12 ++++++------ 1 file changed, 6 insertions(+), 6 deletions(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 77fd88d9..1cae5d4d 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -62,7 +62,7 @@ class ThreadedGetMetadata(Thread): try: self.queue.get(block=False) except Queue.Empty: - xbmc.sleep(50) + xbmc.sleep(10) continue else: self.queue.task_done() @@ -73,7 +73,7 @@ class ThreadedGetMetadata(Thread): try: self.out_queue.get(block=False) except Queue.Empty: - xbmc.sleep(50) + xbmc.sleep(10) continue else: self.out_queue.task_done() @@ -93,7 +93,7 @@ class ThreadedGetMetadata(Thread): updateItem = queue.get(block=False) # Empty queue except Queue.Empty: - xbmc.sleep(100) + xbmc.sleep(10) continue # Download Metadata plexXML = PF.GetPlexMetadata(updateItem['itemId']) @@ -155,7 +155,7 @@ class ThreadedProcessMetadata(Thread): try: self.queue.get(block=False) except Queue.Empty: - xbmc.sleep(100) + xbmc.sleep(10) continue else: self.queue.task_done() @@ -175,7 +175,7 @@ class ThreadedProcessMetadata(Thread): try: updateItem = queue.get(block=False) except Queue.Empty: - xbmc.sleep(50) + xbmc.sleep(10) continue # Do the work plexitem = updateItem['XML'] @@ -250,7 +250,7 @@ class ThreadedShowSyncInfo(Thread): processMetadataProgress, viewName)) # Sleep for x milliseconds - xbmc.sleep(500) + xbmc.sleep(200) dialog.close() log.debug('Dialog Infobox thread terminated') From 8070e921ed9715d35afc31a8802172067bfbc29d Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 10 Sep 2016 11:48:29 +0200 Subject: [PATCH 02/86] Optimize import --- resources/lib/itemtypes.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/lib/itemtypes.py b/resources/lib/itemtypes.py index fd0ee550..c662722a 100644 --- a/resources/lib/itemtypes.py +++ b/resources/lib/itemtypes.py @@ -3,7 +3,7 @@ ############################################################################### import logging -import urllib +from urllib import urlencode from ntpath import dirname from datetime import datetime @@ -394,7 +394,7 @@ class Movies(Items): 'dbid': movieid, 'mode': "play" } - filename = "%s?%s" % (path, urllib.urlencode(params)) + filename = "%s?%s" % (path, urlencode(params)) playurl = filename # movie table: @@ -906,7 +906,7 @@ class TVShows(Items): 'dbid': episodeid, 'mode': "play" } - filename = "%s?%s" % (path, tryDecode(urllib.urlencode(params))) + filename = "%s?%s" % (path, tryDecode(urlencode(params))) playurl = filename parentPathId = self.kodi_db.addPath( 'plugin://plugin.video.plexkodiconnect.tvshows/') From d310b6c7e821368e46ac879aee7016adf2f83483 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 10 Sep 2016 11:49:14 +0200 Subject: [PATCH 03/86] Optimize import --- resources/lib/itemtypes.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/lib/itemtypes.py b/resources/lib/itemtypes.py index c662722a..0d6b8467 100644 --- a/resources/lib/itemtypes.py +++ b/resources/lib/itemtypes.py @@ -11,8 +11,8 @@ import xbmc import xbmcgui import artwork -from utils import settings, window, kodiSQL, CatchExceptions -from utils import tryEncode, tryDecode +from utils import tryEncode, tryDecode, settings, window, kodiSQL, \ + CatchExceptions import embydb_functions as embydb import kodidb_functions as kodidb From 24d6514b7f3b0c61bcf6c1c9199d53dc042c2484 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 10 Sep 2016 11:59:42 +0200 Subject: [PATCH 04/86] Code optimization --- resources/lib/itemtypes.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) diff --git a/resources/lib/itemtypes.py b/resources/lib/itemtypes.py index 0d6b8467..b4d0ccbe 100644 --- a/resources/lib/itemtypes.py +++ b/resources/lib/itemtypes.py @@ -469,16 +469,13 @@ class Movies(Items): # Process countries self.kodi_db.addCountries(movieid, countries, "movie") # Process cast - people = API.getPeopleList() - self.kodi_db.addPeople(movieid, people, "movie") + self.kodi_db.addPeople(movieid, API.getPeopleList(), "movie") # Process genres self.kodi_db.addGenres(movieid, genres, "movie") # Process artwork - allartworks = API.getAllArtwork() - artwork.addArtwork(allartworks, movieid, "movie", kodicursor) + artwork.addArtwork(API.getAllArtwork(), movieid, "movie", kodicursor) # Process stream details - streams = API.getMediaStreams() - self.kodi_db.addStreams(fileid, streams, runtime) + self.kodi_db.addStreams(fileid, API.getMediaStreams(), runtime) # Process studios self.kodi_db.addStudios(movieid, studios, "movie") # Process tags: view, Plex collection tags From 2fbdd5432441454fb91fe4e39b3d215227b7e99d Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 10 Sep 2016 19:49:03 +0200 Subject: [PATCH 05/86] Speed up sync - download art in the background --- resources/lib/PlexAPI.py | 36 +++++--- resources/lib/embydb_functions.py | 29 +++++- resources/lib/itemtypes.py | 24 ++++- resources/lib/kodidb_functions.py | 50 +++++++++-- resources/lib/librarysync.py | 142 ++++++++++++++++++++++++++++++ 5 files changed, 259 insertions(+), 22 deletions(-) diff --git a/resources/lib/PlexAPI.py b/resources/lib/PlexAPI.py index 962dd6d1..1a1e2190 100644 --- a/resources/lib/PlexAPI.py +++ b/resources/lib/PlexAPI.py @@ -1841,16 +1841,14 @@ class API(): 'Backdrop' : LIST with the first entry xml key "art" } """ - item = self.item.attrib - allartworks = { - 'Primary': "", + 'Primary': "", # corresponds to Plex poster ('thumb') 'Art': "", - 'Banner': "", + 'Banner': "", # corresponds to Plex banner ('banner') for series 'Logo': "", - 'Thumb': "", + 'Thumb': "", # corresponds to Plex (grand)parent posters (thumb) 'Disc': "", - 'Backdrop': [] + 'Backdrop': [] # Corresponds to Plex fanart ('art') } # Process backdrops # Get background artwork URL @@ -1870,14 +1868,26 @@ class API(): self.__getOneArtwork('parentArt')) if not allartworks['Primary']: allartworks['Primary'] = self.__getOneArtwork('parentThumb') + return allartworks - # Plex does not get much artwork - go ahead and get the rest from - # fanart tv only for movie or tv show - if settings('FanartTV') == 'true': - if item.get('type') in ('movie', 'show'): - externalId = self.getExternalItemId() - if externalId is not None: - allartworks = self.getFanartTVArt(externalId, allartworks) + def getFanartArtwork(self, allartworks, parentInfo=False): + """ + Downloads additional fanart from third party sources (well, link to + fanart only). + + allartworks = { + 'Primary': "", + 'Art': "", + 'Banner': "", + 'Logo': "", + 'Thumb': "", + 'Disc': "", + 'Backdrop': [] + } + """ + externalId = self.getExternalItemId() + if externalId is not None: + allartworks = self.getFanartTVArt(externalId, allartworks) return allartworks def getExternalItemId(self, collection=False): diff --git a/resources/lib/embydb_functions.py b/resources/lib/embydb_functions.py index 05a9f4d5..3f173e70 100644 --- a/resources/lib/embydb_functions.py +++ b/resources/lib/embydb_functions.py @@ -372,4 +372,31 @@ class Embydb_Functions(): query = "DELETE FROM emby WHERE emby_id LIKE ?" self.embycursor.execute(query, (plexid+"%",)) - \ No newline at end of file + + def itemsByType(self, plextype): + """ + Returns a list of dictionaries for all Kodi DB items present for + plextype. One dict is of the type + + { + 'plexId': the Plex id + 'kodiId': the Kodi id + 'kodi_type': e.g. 'movie', 'tvshow' + 'plex_type': e.g. 'Movie', 'Series', the input plextype + } + """ + query = ' '.join(( + "SELECT emby_id, kodi_id, media_type", + "FROM emby", + "WHERE emby_type = ?", + )) + self.embycursor.execute(query, (plextype, )) + result = [] + for row in self.embycursor.fetchall(): + result.append({ + 'plexId': row[0], + 'kodiId': row[1], + 'kodi_type': row[2], + 'plex_type': plextype + }) + return result diff --git a/resources/lib/itemtypes.py b/resources/lib/itemtypes.py index b4d0ccbe..170ddf5f 100644 --- a/resources/lib/itemtypes.py +++ b/resources/lib/itemtypes.py @@ -65,6 +65,28 @@ class Items(object): self.kodiconn.close() return self + @CatchExceptions(warnuser=True) + def getfanart(self, item, kodiId, mediaType, allartworks=None): + """ + """ + API = PlexAPI.API(item) + if allartworks is None: + allartworks = API.getAllArtwork() + self.artwork.addArtwork(API.getFanartArtwork(allartworks), + kodiId, + mediaType, + self.kodicursor) + # Also get artwork for collections/movie sets + if mediaType == 'movie': + for setname in API.getCollections(): + log.debug('Getting artwork for movie set %s' % setname) + setid = self.kodi_db.createBoxset(setname) + self.artwork.addArtwork(API.getSetArtwork(), + setid, + "set", + self.kodicursor) + self.kodi_db.assignBoxset(setid, kodiId) + def itemsbyId(self, items, process, pdialog=None): # Process items by itemid. Process can be added, update, userdata, remove embycursor = self.embycursor @@ -485,7 +507,7 @@ class Movies(Items): tags.append("Favorite movies") self.kodi_db.addTags(movieid, tags, "movie") # Add any sets from Plex collection tags - self.kodi_db.addSets(movieid, collections, kodicursor, API) + self.kodi_db.addSets(movieid, collections, kodicursor) # Process playstates self.kodi_db.addPlaystate(fileid, resume, runtime, playcount, dateplayed) diff --git a/resources/lib/kodidb_functions.py b/resources/lib/kodidb_functions.py index 0a5a565a..d2950017 100644 --- a/resources/lib/kodidb_functions.py +++ b/resources/lib/kodidb_functions.py @@ -508,6 +508,48 @@ class Kodidb_Functions(): self.artwork.addOrUpdateArt(thumb, actorid, arttype, "thumb", self.cursor) + def existingArt(self, kodiId, mediaType, refresh=False): + """ + For kodiId, returns an artwork dict with already existing art from + the Kodi db + """ + # Only get EITHER poster OR thumb (should have same URL) + kodiToPKC = { + 'banner': 'Banner', + 'clearart': 'Art', + 'clearlogo': 'Logo', + 'discart': 'Disc', + 'landscape': 'Thumb', + 'thumb': 'Primary' + } + # BoxRear yet unused + result = {'BoxRear': ''} + for art in kodiToPKC: + query = ' '.join(( + "SELECT url", + "FROM art", + "WHERE media_id = ?", + "AND media_type = ?", + "AND type = ?" + )) + self.cursor.execute(query, (kodiId, mediaType, art,)) + try: + url = self.cursor.fetchone()[0] + except TypeError: + url = "" + result[kodiToPKC[art]] = url + # There may be several fanart URLs saved + query = ' '.join(( + "SELECT url", + "FROM art", + "WHERE media_id = ?", + "AND media_type = ?", + "AND type LIKE ?" + )) + data = self.cursor.execute(query, (kodiId, mediaType, "fanart%",)) + result['Backdrop'] = [d[0] for d in data] + return result + def addGenres(self, kodiid, genres, mediatype): @@ -1180,15 +1222,9 @@ class Kodidb_Functions(): )) self.cursor.execute(query, (kodiid, mediatype, tag_id,)) - def addSets(self, movieid, collections, kodicursor, API): + def addSets(self, movieid, collections, kodicursor): for setname in collections: setid = self.createBoxset(setname) - # Process artwork - if settings('setFanartTV') == 'true': - self.artwork.addArtwork(API.getSetArtwork(), - setid, - "set", - kodicursor) self.assignBoxset(setid, movieid) def createBoxset(self, boxsetname): diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 1cae5d4d..1a729be5 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -5,6 +5,7 @@ import logging from threading import Thread, Lock import Queue +from random import shuffle import xbmc import xbmcgui @@ -255,6 +256,100 @@ class ThreadedShowSyncInfo(Thread): log.debug('Dialog Infobox thread terminated') +@ThreadMethodsAdditionalSuspend('suspend_LibraryThread') +@ThreadMethodsAdditionalStop('plex_shouldStop') +@ThreadMethods +class ProcessFanartThread(Thread): + """ + Threaded download of additional fanart in the background + + Input: + queue Queue.Queue() object that you will need to fill with + dicts of the following form: + { + 'itemId': the Plex id as a string + 'class': the itemtypes class, e.g. 'Movies' + 'mediaType': the kodi media type, e.g. 'movie' + 'refresh': True/False if true, will overwrite any 3rd party + fanart. If False, will only get missing + } + """ + def __init__(self, queue): + self.queue = queue + Thread.__init__(self) + + def run(self): + threadStopped = self.threadStopped + threadSuspended = self.threadSuspended + queue = self.queue + log.debug('Started Fanart thread') + while not threadStopped(): + # In the event the server goes offline + while threadSuspended(): + # Set in service.py + if threadStopped(): + # Abort was requested while waiting. We should exit + log.debug('Fanart thread terminated while suspended') + break + xbmc.sleep(1000) + # grabs Plex item from queue + try: + item = queue.get(block=False) + except Queue.Empty: + xbmc.sleep(50) + continue + if item['refresh'] is True: + # Leave the Plex art untouched + allartworks = None + else: + with embydb.GetEmbyDB() as emby_db: + try: + kodiId = emby_db.getItem_byId(item['itemId'])[0] + except TypeError: + log.error('Could not get Kodi id for plex id %s' + % item['itemId']) + queue.task_done() + continue + with kodidb.GetKodiDB('video') as kodi_db: + allartworks = kodi_db.existingArt(kodiId, + item['mediaType']) + # Check if we even need to get additional art + needsupdate = False + for key, value in allartworks.iteritems(): + if not value and not key == 'BoxRear': + needsupdate = True + if needsupdate is False: + log.debug('Already got all art for Plex id %s' + % item['itemId']) + queue.task_done() + continue + + log.debug('Getting additional fanart for Plex id %s' + % item['itemId']) + # Download Metadata + xml = PF.GetPlexMetadata(item['itemId']) + if xml is None: + # Did not receive a valid XML - skip that item for now + log.warn("Could not get metadata for %s. Skipping that item " + "for now" % item['itemId']) + queue.task_done() + continue + elif xml == 401: + log.warn('HTTP 401 returned by PMS. Too much strain? ' + 'Cancelling sync for now') + # Kill remaining items in queue (for main thread to cont.) + queue.task_done() + continue + + # Do the work + with getattr(itemtypes, item['class'])() as cls: + cls.getfanart(xml[0], kodiId, item['mediaType'], allartworks) + # signals to queue job is done + log.debug('Done getting fanart for Plex id %s' % item['itemId']) + queue.task_done() + log.debug('Fanart thread terminated') + + @ThreadMethodsAdditionalSuspend('suspend_LibraryThread') @ThreadMethodsAdditionalStop('plex_shouldStop') @ThreadMethods @@ -275,6 +370,9 @@ class LibrarySync(Thread): self.queue = queue self.itemsToProcess = [] self.sessionKeys = [] + if settings('FanartTV') == 'true': + self.fanartqueue = Queue.Queue() + self.fanartthread = ProcessFanartThread(self.fanartqueue) # How long should we wait at least to process new/changed PMS items? self.saftyMargin = int(settings('saftyMargin')) @@ -887,6 +985,16 @@ class LibrarySync(Thread): except: pass log.info("Sync threads finished") + if settings('FanartTV') == 'true': + # Save to queue for later processing + typus = {'Movies': 'movie', 'TVShows': 'tvshow'}[itemType] + for item in self.updatelist: + self.fanartqueue.put({ + 'itemId': item['itemId'], + 'class': itemType, + 'mediaType': typus, + 'refresh': False + }) self.updatelist = [] @LogTime @@ -1484,6 +1592,30 @@ class LibrarySync(Thread): with itemFkt() as Fkt: Fkt.updatePlaystate(item) + def fanartSync(self, refresh=False): + """ + Checks all Plex movies and TV shows whether they still need fanart + + refresh=True Force refresh all external fanart + """ + items = [] + typus = { + 'Movie': 'Movies', + 'Series': 'TVShows' + } + with embydb.GetEmbyDB() as emby_db: + for plextype in typus: + items.extend(emby_db.itemsByType(plextype)) + # Shuffle the list to not always start out identically + items = shuffle(items) + for item in items: + self.fanartqueue.put({ + 'itemId': item['plexId'], + 'mediaType': item['kodi_type'], + 'class': typus[item['plex_type']], + 'refresh': refresh + }) + def run(self): try: self.run_internal() @@ -1508,6 +1640,7 @@ class LibrarySync(Thread): fullSyncInterval = self.fullSyncInterval lastSync = 0 lastTimeSync = 0 + lastFanartSync = 0 lastProcessing = 0 oneDay = 60*60*24 @@ -1527,6 +1660,9 @@ class LibrarySync(Thread): if self.enableMusic: advancedSettingsXML() + if settings('FanartTV') == 'true': + self.fanartthread.start() + while not threadStopped(): # In the event the server goes offline @@ -1661,6 +1797,12 @@ class LibrarySync(Thread): window('plex_dbScan', value="true") self.syncPMStime() window('plex_dbScan', clear=True) + elif (now - lastFanartSync > oneDay and + settings('FanartTV') == 'true'): + lastFanartSync = now + log.info('Starting daily fanart sync') + self.fanartSync() + log.info('Finished init of daily fanart sync') elif enableBackgroundSync: # Check back whether we should process something # Only do this once every 10 seconds From bcb3b4950772e59a4ee655f2fba16cf8d3afda44 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 10:14:29 +0200 Subject: [PATCH 06/86] Quit ProcessFanartThread quicker --- resources/lib/librarysync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 1a729be5..e40ef334 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -290,7 +290,7 @@ class ProcessFanartThread(Thread): if threadStopped(): # Abort was requested while waiting. We should exit log.debug('Fanart thread terminated while suspended') - break + return xbmc.sleep(1000) # grabs Plex item from queue try: From 3e261a75e2174fae0e9e170648024c99d1245641 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 10:20:29 +0200 Subject: [PATCH 07/86] Prevent errors if changing setting --- resources/lib/librarysync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index e40ef334..f842e85e 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -370,8 +370,8 @@ class LibrarySync(Thread): self.queue = queue self.itemsToProcess = [] self.sessionKeys = [] + self.fanartqueue = Queue.Queue() if settings('FanartTV') == 'true': - self.fanartqueue = Queue.Queue() self.fanartthread = ProcessFanartThread(self.fanartqueue) # How long should we wait at least to process new/changed PMS items? self.saftyMargin = int(settings('saftyMargin')) From 1370bda4323752abefb5958926663dd04a08717e Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 10:36:28 +0200 Subject: [PATCH 08/86] Improve logging --- resources/lib/librarysync.py | 7 +++---- 1 file changed, 3 insertions(+), 4 deletions(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index f842e85e..bbc22953 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -281,15 +281,14 @@ class ProcessFanartThread(Thread): def run(self): threadStopped = self.threadStopped threadSuspended = self.threadSuspended - queue = self.queue - log.debug('Started Fanart thread') + log.info("---===### Starting FanartSync ###===---") while not threadStopped(): # In the event the server goes offline while threadSuspended(): # Set in service.py if threadStopped(): # Abort was requested while waiting. We should exit - log.debug('Fanart thread terminated while suspended') + log.info("---===### Stopped FanartSync ###===---") return xbmc.sleep(1000) # grabs Plex item from queue @@ -347,7 +346,7 @@ class ProcessFanartThread(Thread): # signals to queue job is done log.debug('Done getting fanart for Plex id %s' % item['itemId']) queue.task_done() - log.debug('Fanart thread terminated') + log.info("---===### Stopped FanartSync ###===---") @ThreadMethodsAdditionalSuspend('suspend_LibraryThread') From ddf4e89165ee386cb7b9875541d8a9c4756a8106 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 10:39:40 +0200 Subject: [PATCH 09/86] Prevent KeyError --- resources/lib/librarysync.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index bbc22953..31be22c8 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -984,7 +984,8 @@ class LibrarySync(Thread): except: pass log.info("Sync threads finished") - if settings('FanartTV') == 'true': + if (settings('FanartTV') == 'true' and + itemType in ('Movies', 'TVShows')): # Save to queue for later processing typus = {'Movies': 'movie', 'TVShows': 'tvshow'}[itemType] for item in self.updatelist: From fb7905e0b62d0741a9691c9ae67905d933647d4a Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 10:42:40 +0200 Subject: [PATCH 10/86] Fix processing of complicated PMS messages --- resources/lib/librarysync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 31be22c8..a1545a7d 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -1477,7 +1477,7 @@ class LibrarySync(Thread): # Have we already added this element? for existingItem in self.itemsToProcess: if existingItem['ratingKey'] == itemId: - break + continue else: # Haven't added this element to the queue yet self.itemsToProcess.append({ From 5346426bd77540128968999720e9cadf6017bbea Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 10:43:16 +0200 Subject: [PATCH 11/86] Revert "Fix processing of complicated PMS messages" This reverts commit fb7905e0b62d0741a9691c9ae67905d933647d4a. --- resources/lib/librarysync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index a1545a7d..31be22c8 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -1477,7 +1477,7 @@ class LibrarySync(Thread): # Have we already added this element? for existingItem in self.itemsToProcess: if existingItem['ratingKey'] == itemId: - continue + break else: # Haven't added this element to the queue yet self.itemsToProcess.append({ From 751e55b9d5dd74e1cc06ab650168fc58ea5108e7 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 11:12:25 +0200 Subject: [PATCH 12/86] Get fanart for background sync items --- resources/lib/librarysync.py | 13 +++++++++++++ 1 file changed, 13 insertions(+) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 31be22c8..5db3f028 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -281,6 +281,7 @@ class ProcessFanartThread(Thread): def run(self): threadStopped = self.threadStopped threadSuspended = self.threadSuspended + queue = self.queue log.info("---===### Starting FanartSync ###===---") while not threadStopped(): # In the event the server goes offline @@ -1395,6 +1396,16 @@ class LibrarySync(Thread): successful = self.process_newitems(item) if successful is True: deleteListe.append(i) + if (settings('FanartTV') == 'true' and + item['mediatype'] in ('movie')): + mediaType = {'movie': 'Movie'}[item['mediatype']] + cls = {'movie': 'Movies'}[item['mediatype']] + self.fanartqueue.put({ + 'itemId': item['ratingKey'], + 'class': cls, + 'mediaType': mediaType, + 'refresh': False + }) else: # Safety net if we can't process an item item['attempt'] += 1 @@ -1425,6 +1436,8 @@ class LibrarySync(Thread): viewtag = xml.attrib.get('librarySectionTitle') viewid = xml.attrib.get('librarySectionID') mediatype = xml[0].attrib.get('type') + # Attach mediatype for later + item['mediatype'] = mediatype if mediatype == 'movie': self.videoLibUpdate = True with itemtypes.Movies() as movie: From 7b578603ca52be3d4005f7315868052b8ad14669 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 11:29:51 +0200 Subject: [PATCH 13/86] Fix TypeError --- resources/lib/librarysync.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 5db3f028..f277162a 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -1620,7 +1620,7 @@ class LibrarySync(Thread): for plextype in typus: items.extend(emby_db.itemsByType(plextype)) # Shuffle the list to not always start out identically - items = shuffle(items) + shuffle(items) for item in items: self.fanartqueue.put({ 'itemId': item['plexId'], From cb4e084305660f914bb1fd9a781b0810b4da8f04 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 11:32:18 +0200 Subject: [PATCH 14/86] Don't look for fanart twice on fresh system --- resources/lib/librarysync.py | 11 ----------- 1 file changed, 11 deletions(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index f277162a..abec145c 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -985,17 +985,6 @@ class LibrarySync(Thread): except: pass log.info("Sync threads finished") - if (settings('FanartTV') == 'true' and - itemType in ('Movies', 'TVShows')): - # Save to queue for later processing - typus = {'Movies': 'movie', 'TVShows': 'tvshow'}[itemType] - for item in self.updatelist: - self.fanartqueue.put({ - 'itemId': item['itemId'], - 'class': itemType, - 'mediaType': typus, - 'refresh': False - }) self.updatelist = [] @LogTime From 5cf77d3292fcefc3debac7f7ce1809882cd93721 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 11:46:42 +0200 Subject: [PATCH 15/86] Revert "Don't look for fanart twice on fresh system" This reverts commit cb4e084305660f914bb1fd9a781b0810b4da8f04. --- resources/lib/librarysync.py | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index abec145c..f277162a 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -985,6 +985,17 @@ class LibrarySync(Thread): except: pass log.info("Sync threads finished") + if (settings('FanartTV') == 'true' and + itemType in ('Movies', 'TVShows')): + # Save to queue for later processing + typus = {'Movies': 'movie', 'TVShows': 'tvshow'}[itemType] + for item in self.updatelist: + self.fanartqueue.put({ + 'itemId': item['itemId'], + 'class': itemType, + 'mediaType': typus, + 'refresh': False + }) self.updatelist = [] @LogTime From 7c86270ece80fc8d1b92e8d0a50b5c1e76b45dc6 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 11:47:55 +0200 Subject: [PATCH 16/86] Don't do daily lookup of fanart --- resources/lib/librarysync.py | 7 ------- 1 file changed, 7 deletions(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index f277162a..4b7c937e 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -1653,7 +1653,6 @@ class LibrarySync(Thread): fullSyncInterval = self.fullSyncInterval lastSync = 0 lastTimeSync = 0 - lastFanartSync = 0 lastProcessing = 0 oneDay = 60*60*24 @@ -1810,12 +1809,6 @@ class LibrarySync(Thread): window('plex_dbScan', value="true") self.syncPMStime() window('plex_dbScan', clear=True) - elif (now - lastFanartSync > oneDay and - settings('FanartTV') == 'true'): - lastFanartSync = now - log.info('Starting daily fanart sync') - self.fanartSync() - log.info('Finished init of daily fanart sync') elif enableBackgroundSync: # Check back whether we should process something # Only do this once every 10 seconds From c3010645bc2a603272ef08022c91914fe25cdbf6 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 12:15:26 +0200 Subject: [PATCH 17/86] New PKC setting to look for missing fanart --- default.py | 5 +++++ resources/language/English/strings.xml | 4 ++++ resources/language/German/strings.xml | 5 +++++ resources/lib/entrypoint.py | 3 ++- resources/lib/librarysync.py | 9 +++++++++ 5 files changed, 25 insertions(+), 1 deletion(-) diff --git a/default.py b/default.py index 5c661387..8cfb9ae9 100644 --- a/default.py +++ b/default.py @@ -99,6 +99,11 @@ class Main(): entrypoint.getVideoFiles(plexid, plexpath) return + if mode == 'fanart': + log.info('User requested fanarttv refresh') + utils.window('plex_runLibScan', value='fanart') + return + # Called by e.g. 3rd party plugin video extras if ("/Extras" in sys.argv[0] or "/VideoFiles" in sys.argv[0] or "/Extras" in sys.argv[2]): diff --git a/resources/language/English/strings.xml b/resources/language/English/strings.xml index 5850605a..db6fb9dc 100644 --- a/resources/language/English/strings.xml +++ b/resources/language/English/strings.xml @@ -446,6 +446,10 @@ Abort (Yes) or save address anyway (No)? connected plex.tv toggle successful + Look for missing fanart on FanartTV + Only look for missing fanart or refresh all fanart? The scan will take quite a while and happen in the background. + Refresh all + Missing only diff --git a/resources/language/German/strings.xml b/resources/language/German/strings.xml index d35b2963..16176b81 100644 --- a/resources/language/German/strings.xml +++ b/resources/language/German/strings.xml @@ -384,6 +384,11 @@ Abbrechen (Ja) oder PMS Adresse trotzdem speichern (Nein)? verbunden plex.tv wechsel OK + Nach zusätzlicher Fanart auf FanartTV suchen + Nur nach fehlender Fanart suchen oder alle Fanart neu herunterladen? Die Suche wird lange dauern und komplett im Hintergrund stattfinden! + Alle + Fehlend + Alle Plex Bilder in Kodi zwischenzuspeichern kann sehr lange dauern. Möchten Sie wirklich fortfahren? diff --git a/resources/lib/entrypoint.py b/resources/lib/entrypoint.py index e1397ce7..d2a5ebd7 100644 --- a/resources/lib/entrypoint.py +++ b/resources/lib/entrypoint.py @@ -275,7 +275,8 @@ def doMainListing(): # addDirectoryItem("Add user to session", "plugin://plugin.video.plexkodiconnect/?mode=adduser") addDirectoryItem(lang(39203), "plugin://plugin.video.plexkodiconnect/?mode=refreshplaylist") addDirectoryItem(lang(39204), "plugin://plugin.video.plexkodiconnect/?mode=manualsync") - + if settings('FanartTV') == 'true': + addDirectoryItem(lang(39222), "plugin://plugin.video.plexkodiconnect/?mode=fanart") xbmcplugin.endOfDirectory(int(sys.argv[1])) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 4b7c937e..0bc25f31 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -1788,6 +1788,15 @@ class LibrarySync(Thread): forced=True, icon="error") window('plex_dbScan', clear=True) + elif window('plex_runLibScan') == 'fanart': + window('plex_runLibScan', clear=True) + # Only look for missing fanart (No) + # or refresh all fanart (Yes) + self.fanartSync(refresh=self.dialog.yesno( + heading=addonName, + line1=lang(39223), + nolabel=lang(39224), + yeslabel=lang(39225))) else: now = getUnixTimestamp() if (now - lastSync > fullSyncInterval and From bfc024972079a2a93c278d508b5b112e75b758bf Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 12:21:13 +0200 Subject: [PATCH 18/86] Optimize import --- resources/lib/artwork.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index 7f4f93aa..dd1f330d 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -14,8 +14,8 @@ import xbmcgui import xbmcvfs import image_cache_thread -from utils import window, settings, language as lang, kodiSQL -from utils import tryEncode, tryDecode, IfExists +from utils import window, settings, language as lang, kodiSQL, tryEncode, \ + tryDecode, IfExists # Disable annoying requests warnings import requests.packages.urllib3 From 59882a7be81fbb03ce8ca62fab90ea83f384dcca Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 13:51:53 +0200 Subject: [PATCH 19/86] Improve + make artwork thread-save --- resources/lib/artwork.py | 574 ++++++++++++++------------------------- 1 file changed, 211 insertions(+), 363 deletions(-) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index dd1f330d..50add7b9 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -8,6 +8,7 @@ import requests import os import urllib from sqlite3 import OperationalError +from threading import Lock import xbmc import xbmcgui @@ -27,152 +28,133 @@ log = logging.getLogger("PLEX."+__name__) ############################################################################### +def setKodiWebServerDetails(): + """ + Get the Kodi webserver details - used to set the texture cache + """ + xbmc_port = None + xbmc_username = None + xbmc_password = None + web_query = { + "jsonrpc": "2.0", + "id": 1, + "method": "Settings.GetSettingValue", + "params": { + "setting": "services.webserver" + } + } + result = xbmc.executeJSONRPC(json.dumps(web_query)) + result = json.loads(result) + try: + xbmc_webserver_enabled = result['result']['value'] + except (KeyError, TypeError): + xbmc_webserver_enabled = False + if not xbmc_webserver_enabled: + # Enable the webserver, it is disabled + web_port = { + "jsonrpc": "2.0", + "id": 1, + "method": "Settings.SetSettingValue", + "params": { + "setting": "services.webserverport", + "value": 8080 + } + } + result = xbmc.executeJSONRPC(json.dumps(web_port)) + xbmc_port = 8080 + web_user = { + "jsonrpc": "2.0", + "id": 1, + "method": "Settings.SetSettingValue", + "params": { + "setting": "services.webserver", + "value": True + } + } + result = xbmc.executeJSONRPC(json.dumps(web_user)) + xbmc_username = "kodi" + # Webserver already enabled + web_port = { + "jsonrpc": "2.0", + "id": 1, + "method": "Settings.GetSettingValue", + "params": { + "setting": "services.webserverport" + } + } + result = xbmc.executeJSONRPC(json.dumps(web_port)) + result = json.loads(result) + try: + xbmc_port = result['result']['value'] + except (TypeError, KeyError): + pass + web_user = { + "jsonrpc": "2.0", + "id": 1, + "method": "Settings.GetSettingValue", + "params": { + "setting": "services.webserverusername" + } + } + result = xbmc.executeJSONRPC(json.dumps(web_user)) + result = json.loads(result) + try: + xbmc_username = result['result']['value'] + except TypeError: + pass + web_pass = { + "jsonrpc": "2.0", + "id": 1, + "method": "Settings.GetSettingValue", + "params": { + "setting": "services.webserverpassword" + } + } + result = xbmc.executeJSONRPC(json.dumps(web_pass)) + result = json.loads(result) + try: + xbmc_password = result['result']['value'] + except TypeError: + pass + return (xbmc_port, xbmc_username, xbmc_password) + + class Artwork(): + lock = Lock() + + enableTextureCache = settings('enableTextureCache') == "true" + imageCacheLimitThreads = int(settings('imageCacheLimit')) + imageCacheLimitThreads = imageCacheLimitThreads * 5 + log.info("Using Image Cache Thread Count: %s" % imageCacheLimitThreads) + xbmc_host = 'localhost' xbmc_port = None xbmc_username = None xbmc_password = None + if enableTextureCache: + xbmc_port, xbmc_username, xbmc_password = setKodiWebServerDetails() imageCacheThreads = [] - imageCacheLimitThreads = 0 - - - def __init__(self): - - self.enableTextureCache = settings('enableTextureCache') == "true" - self.imageCacheLimitThreads = int(settings('imageCacheLimit')) - self.imageCacheLimitThreads = int(self.imageCacheLimitThreads * 5) - log.info("Using Image Cache Thread Count: %s" % self.imageCacheLimitThreads) - - if not self.xbmc_port and self.enableTextureCache: - self.setKodiWebServerDetails() - - self.userId = window('currUserId') - self.server = window('pms_server') def double_urlencode(self, text): text = self.single_urlencode(text) text = self.single_urlencode(text) - return text def single_urlencode(self, text): - - text = urllib.urlencode({'blahblahblah': tryEncode(text)}) #urlencode needs a utf- string + # urlencode needs a utf- string + text = urllib.urlencode({'blahblahblah': tryEncode(text)}) text = text[13:] - - return tryDecode(text) #return the result again as unicode - - def setKodiWebServerDetails(self): - # Get the Kodi webserver details - used to set the texture cache - web_query = { - - "jsonrpc": "2.0", - "id": 1, - "method": "Settings.GetSettingValue", - "params": { - - "setting": "services.webserver" - } - } - result = xbmc.executeJSONRPC(json.dumps(web_query)) - result = json.loads(result) - try: - xbmc_webserver_enabled = result['result']['value'] - except (KeyError, TypeError): - xbmc_webserver_enabled = False - - if not xbmc_webserver_enabled: - # Enable the webserver, it is disabled - web_port = { - - "jsonrpc": "2.0", - "id": 1, - "method": "Settings.SetSettingValue", - "params": { - - "setting": "services.webserverport", - "value": 8080 - } - } - result = xbmc.executeJSONRPC(json.dumps(web_port)) - self.xbmc_port = 8080 - - web_user = { - - "jsonrpc": "2.0", - "id": 1, - "method": "Settings.SetSettingValue", - "params": { - - "setting": "services.webserver", - "value": True - } - } - result = xbmc.executeJSONRPC(json.dumps(web_user)) - self.xbmc_username = "kodi" - - - # Webserver already enabled - web_port = { - - "jsonrpc": "2.0", - "id": 1, - "method": "Settings.GetSettingValue", - "params": { - - "setting": "services.webserverport" - } - } - result = xbmc.executeJSONRPC(json.dumps(web_port)) - result = json.loads(result) - try: - self.xbmc_port = result['result']['value'] - except (TypeError, KeyError): - pass - - web_user = { - - "jsonrpc": "2.0", - "id": 1, - "method": "Settings.GetSettingValue", - "params": { - - "setting": "services.webserverusername" - } - } - result = xbmc.executeJSONRPC(json.dumps(web_user)) - result = json.loads(result) - try: - self.xbmc_username = result['result']['value'] - except TypeError: - pass - - web_pass = { - - "jsonrpc": "2.0", - "id": 1, - "method": "Settings.GetSettingValue", - "params": { - - "setting": "services.webserverpassword" - } - } - result = xbmc.executeJSONRPC(json.dumps(web_pass)) - result = json.loads(result) - try: - self.xbmc_password = result['result']['value'] - except TypeError: - pass + # return the result again as unicode + return tryDecode(text) def fullTextureCacheSync(self): - # This method will sync all Kodi artwork to textures13.db - # and cache them locally. This takes diskspace! - import xbmcaddon - string = xbmcaddon.Addon().getLocalizedString - + """ + This method will sync all Kodi artwork to textures13.db + and cache them locally. This takes diskspace! + """ if not xbmcgui.Dialog().yesno( - "Image Texture Cache", string(39250)): + "Image Texture Cache", lang(39250)): return log.info("Doing Image Cache Sync") @@ -182,7 +164,7 @@ class Artwork(): # ask to rest all existing or not if xbmcgui.Dialog().yesno( - "Image Texture Cache", string(39251), ""): + "Image Texture Cache", lang(39251), ""): log.info("Resetting all cache data first") # Remove all existing textures first path = tryDecode(xbmc.translatePath("special://thumbnails/")) @@ -210,20 +192,20 @@ class Artwork(): if tableName != "version": cursor.execute("DELETE FROM " + tableName) connection.commit() - cursor.close() + connection.close() # Cache all entries in video DB connection = kodiSQL('video') cursor = connection.cursor() - cursor.execute("SELECT url FROM art WHERE media_type != 'actor'") # dont include actors + # dont include actors + cursor.execute("SELECT url FROM art WHERE media_type != 'actor'") result = cursor.fetchall() total = len(result) log.info("Image cache sync about to process %s images" % total) - cursor.close() + connection.close() count = 0 for url in result: - if pdialog.iscanceled(): break @@ -232,7 +214,6 @@ class Artwork(): pdialog.update(percentage, "%s %s" % (lang(33045), message)) self.cacheTexture(url[0]) count += 1 - # Cache all entries in music DB connection = kodiSQL('music') cursor = connection.cursor() @@ -240,11 +221,10 @@ class Artwork(): result = cursor.fetchall() total = len(result) log.info("Image cache sync about to process %s images" % total) - cursor.close() + connection.close() count = 0 for url in result: - if pdialog.iscanceled(): break @@ -253,63 +233,64 @@ class Artwork(): pdialog.update(percentage, "%s %s" % (lang(33045), message)) self.cacheTexture(url[0]) count += 1 - - pdialog.update(100, "%s %s" % (lang(33046), len(self.imageCacheThreads))) + pdialog.update(100, "%s %s" + % (lang(33046), len(self.imageCacheThreads))) log.info("Waiting for all threads to exit") - while len(self.imageCacheThreads): - for thread in self.imageCacheThreads: - if thread.is_finished: - self.imageCacheThreads.remove(thread) - pdialog.update(100, "%s %s" % (lang(33046), len(self.imageCacheThreads))) - log.info("Waiting for all threads to exit: %s" % len(self.imageCacheThreads)) + with self.lock: + for thread in self.imageCacheThreads: + if thread.is_finished: + self.imageCacheThreads.remove(thread) + pdialog.update(100, "%s %s" + % (lang(33046), len(self.imageCacheThreads))) + log.info("Waiting for all threads to exit: %s" + % len(self.imageCacheThreads)) xbmc.sleep(500) pdialog.close() def addWorkerImageCacheThread(self, url): - while True: # removed finished - for thread in self.imageCacheThreads: - if thread.is_finished: - self.imageCacheThreads.remove(thread) - + with self.lock: + for thread in self.imageCacheThreads: + if thread.is_finished: + self.imageCacheThreads.remove(thread) # add a new thread or wait and retry if we hit our limit - if len(self.imageCacheThreads) < self.imageCacheLimitThreads: - newThread = image_cache_thread.ImageCacheThread() - newThread.set_url(self.double_urlencode(url)) - newThread.set_host(self.xbmc_host, self.xbmc_port) - newThread.set_auth(self.xbmc_username, self.xbmc_password) - newThread.start() - self.imageCacheThreads.append(newThread) - return - else: - log.info("Waiting for empty queue spot: %s" % len(self.imageCacheThreads)) - xbmc.sleep(50) + with self.lock: + if len(self.imageCacheThreads) < self.imageCacheLimitThreads: + newThread = image_cache_thread.ImageCacheThread() + newThread.set_url(self.double_urlencode(url)) + newThread.set_host(self.xbmc_host, self.xbmc_port) + newThread.set_auth(self.xbmc_username, self.xbmc_password) + newThread.start() + self.imageCacheThreads.append(newThread) + return + log.debug("Waiting for empty queue spot: %s" + % len(self.imageCacheThreads)) + xbmc.sleep(50) def cacheTexture(self, url): # Cache a single image url to the texture cache if url and self.enableTextureCache: log.debug("Processing: %s" % url) - if not self.imageCacheLimitThreads: - # Add image to texture cache by simply calling it at the http endpoint - + # Add image to texture cache by simply calling it at the http + # endpoint url = self.double_urlencode(url) - try: # Extreme short timeouts so we will have a exception. - response = requests.head( - url=("http://%s:%s/image/image://%s" - % (self.xbmc_host, self.xbmc_port, url)), - auth=(self.xbmc_username, self.xbmc_password), - timeout=(0.01, 0.01)) + try: + # Extreme short timeouts so we will have a exception. + requests.head( + url=("http://%s:%s/image/image://%s" + % (self.xbmc_host, self.xbmc_port, url)), + auth=(self.xbmc_username, self.xbmc_password), + timeout=(0.01, 0.01)) # We don't need the result - except: pass - + except: + pass else: self.addWorkerImageCacheThread(url) - def addArtwork(self, artwork, kodiId, mediaType, cursor): # Kodi conversion table kodiart = { @@ -328,7 +309,8 @@ class Artwork(): for art in artwork: if art == "Backdrop": # Backdrop entry is a list - # Process extra fanart for artwork downloader (fanart, fanart1, fanart2...) + # Process extra fanart for artwork downloader (fanart, fanart1, + # fanart2...) backdrops = artwork[art] backdropsNumber = len(backdrops) @@ -390,67 +372,58 @@ class Artwork(): cursor=cursor) def addOrUpdateArt(self, imageUrl, kodiId, mediaType, imageType, cursor): - # Possible that the imageurl is an empty string - if imageUrl: - cacheimage = False + if not imageUrl: + # Possible that the imageurl is an empty string + return + query = ' '.join(( + "SELECT url", + "FROM art", + "WHERE media_id = ?", + "AND media_type = ?", + "AND type = ?" + )) + cursor.execute(query, (kodiId, mediaType, imageType,)) + try: + # Update the artwork + url = cursor.fetchone()[0] + except TypeError: + # Add the artwork + log.debug("Adding Art Link for kodiId: %s (%s)" + % (kodiId, imageUrl)) + query = ( + ''' + INSERT INTO art(media_id, media_type, type, url) + VALUES (?, ?, ?, ?) + ''' + ) + cursor.execute(query, (kodiId, mediaType, imageType, imageUrl)) + else: + if url == imageUrl: + # Only cache artwork if it changed + return + # Only for the main backdrop, poster + if (window('plex_initialScan') != "true" and + imageType in ("fanart", "poster")): + # Delete current entry before updating with the new one + self.deleteCachedArtwork(url) + log.debug("Updating Art url for %s kodiId %s %s -> (%s)" + % (imageType, kodiId, url, imageUrl)) query = ' '.join(( - - "SELECT url", - "FROM art", + "UPDATE art", + "SET url = ?", "WHERE media_id = ?", "AND media_type = ?", "AND type = ?" )) - cursor.execute(query, (kodiId, mediaType, imageType,)) - try: # Update the artwork - url = cursor.fetchone()[0] + cursor.execute(query, (imageUrl, kodiId, mediaType, imageType)) - except TypeError: # Add the artwork - cacheimage = True - log.debug("Adding Art Link for kodiId: %s (%s)" % (kodiId, imageUrl)) - - query = ( - ''' - INSERT INTO art(media_id, media_type, type, url) - - VALUES (?, ?, ?, ?) - ''' - ) - cursor.execute(query, (kodiId, mediaType, imageType, imageUrl)) - - else: # Only cache artwork if it changed - if url != imageUrl: - cacheimage = True - - # Only for the main backdrop, poster - if (window('plex_initialScan') != "true" and - imageType in ("fanart", "poster")): - # Delete current entry before updating with the new one - self.deleteCachedArtwork(url) - - log.info("Updating Art url for %s kodiId: %s (%s) -> (%s)" - % (imageType, kodiId, url, imageUrl)) - - query = ' '.join(( - - "UPDATE art", - "SET url = ?", - "WHERE media_id = ?", - "AND media_type = ?", - "AND type = ?" - )) - cursor.execute(query, (imageUrl, kodiId, mediaType, imageType)) - - # Cache fanart and poster in Kodi texture cache - if cacheimage: - self.cacheTexture(imageUrl) + # Cache fanart and poster in Kodi texture cache + self.cacheTexture(imageUrl) def deleteArtwork(self, kodiId, mediaType, cursor): - query = ' '.join(( - - "SELECT url, type", + "SELECT url", "FROM art", "WHERE media_id = ?", "AND media_type = ?" @@ -458,159 +431,34 @@ class Artwork(): cursor.execute(query, (kodiId, mediaType,)) rows = cursor.fetchall() for row in rows: - - url = row[0] - imageType = row[1] - if imageType in ("poster", "fanart"): - self.deleteCachedArtwork(url) + self.deleteCachedArtwork(row[0]) def deleteCachedArtwork(self, url): # Only necessary to remove and apply a new backdrop or poster connection = kodiSQL('texture') cursor = connection.cursor() - try: - cursor.execute("SELECT cachedurl FROM texture WHERE url = ?", (url,)) + cursor.execute("SELECT cachedurl FROM texture WHERE url = ?", + (url,)) cachedurl = cursor.fetchone()[0] - except TypeError: log.info("Could not find cached url.") - except OperationalError: - log.info("Database is locked. Skip deletion process.") - - else: # Delete thumbnail as well as the entry + log.warn("Database is locked. Skip deletion process.") + else: + # Delete thumbnail as well as the entry thumbnails = tryDecode( xbmc.translatePath("special://thumbnails/%s" % cachedurl)) - log.info("Deleting cached thumbnail: %s" % thumbnails) - xbmcvfs.delete(thumbnails) - + log.debug("Deleting cached thumbnail: %s" % thumbnails) + try: + xbmcvfs.delete(thumbnails) + except Exception as e: + log.error('Could not delete cached artwork %s. Error: %s' + % (thumbnails, e)) try: cursor.execute("DELETE FROM texture WHERE url = ?", (url,)) connection.commit() except OperationalError: - log.debug("Issue deleting url from cache. Skipping.") - + log.error("OperationalError deleting url from cache.") finally: - cursor.close() - - def getPeopleArtwork(self, people): - # append imageurl if existing - for person in people: - - personId = person['Id'] - tag = person.get('PrimaryImageTag') - - image = "" - if tag: - image = ( - "%s/emby/Items/%s/Images/Primary?" - "MaxWidth=400&MaxHeight=400&Index=0&Tag=%s" - % (self.server, personId, tag)) - - person['imageurl'] = image - - return people - - def getUserArtwork(self, itemId, itemType): - # Load user information set by UserClient - image = ("%s/emby/Users/%s/Images/%s?Format=original" - % (self.server, itemId, itemType)) - return image - - -def getAllArtwork(self, item, parentInfo=False): - - itemid = item['Id'] - artworks = item['ImageTags'] - backdrops = item.get('BackdropImageTags', []) - - maxHeight = 10000 - maxWidth = 10000 - customquery = "" - - # if utils.settings('compressArt') == "true": - # customquery = "&Quality=90" - - # if utils.settings('enableCoverArt') == "false": - # customquery += "&EnableImageEnhancers=false" - - allartworks = { - - 'Primary': "", - 'Art': "", - 'Banner': "", - 'Logo': "", - 'Thumb': "", - 'Disc': "", - 'Backdrop': [] - } - - # Process backdrops - for index, tag in enumerate(backdrops): - artwork = ( - "%s/emby/Items/%s/Images/Backdrop/%s?" - "MaxWidth=%s&MaxHeight=%s&Format=original&Tag=%s%s" - % (self.server, itemid, index, maxWidth, maxHeight, tag, customquery)) - allartworks['Backdrop'].append(artwork) - - # Process the rest of the artwork - for art in artworks: - # Filter backcover - if art != "BoxRear": - tag = artworks[art] - artwork = ( - "%s/emby/Items/%s/Images/%s/0?" - "MaxWidth=%s&MaxHeight=%s&Format=original&Tag=%s%s" - % (self.server, itemid, art, maxWidth, maxHeight, tag, customquery)) - allartworks[art] = artwork - - # Process parent items if the main item is missing artwork - if parentInfo: - - # Process parent backdrops - if not allartworks['Backdrop']: - - parentId = item.get('ParentBackdropItemId') - if parentId: - # If there is a parentId, go through the parent backdrop list - parentbackdrops = item['ParentBackdropImageTags'] - - for index, tag in enumerate(parentbackdrops): - artwork = ( - "%s/emby/Items/%s/Images/Backdrop/%s?" - "MaxWidth=%s&MaxHeight=%s&Format=original&Tag=%s%s" - % (self.server, parentId, index, maxWidth, maxHeight, tag, customquery)) - allartworks['Backdrop'].append(artwork) - - # Process the rest of the artwork - parentartwork = ['Logo', 'Art', 'Thumb'] - for parentart in parentartwork: - - if not allartworks[parentart]: - - parentId = item.get('Parent%sItemId' % parentart) - if parentId: - - parentTag = item['Parent%sImageTag' % parentart] - artwork = ( - "%s/emby/Items/%s/Images/%s/0?" - "MaxWidth=%s&MaxHeight=%s&Format=original&Tag=%s%s" - % (self.server, parentId, parentart, - maxWidth, maxHeight, parentTag, customquery)) - allartworks[parentart] = artwork - - # Parent album works a bit differently - if not allartworks['Primary']: - - parentId = item.get('AlbumId') - if parentId and item.get('AlbumPrimaryImageTag'): - - parentTag = item['AlbumPrimaryImageTag'] - artwork = ( - "%s/emby/Items/%s/Images/Primary/0?" - "MaxWidth=%s&MaxHeight=%s&Format=original&Tag=%s%s" - % (self.server, parentId, maxWidth, maxHeight, parentTag, customquery)) - allartworks['Primary'] = artwork - - return allartworks \ No newline at end of file + connection.close() From 940e41921f1fa7d5b6619fb7f39aeab798896507 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 13:52:14 +0200 Subject: [PATCH 20/86] Don't warn user of OperationalErrors --- resources/lib/itemtypes.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/resources/lib/itemtypes.py b/resources/lib/itemtypes.py index 170ddf5f..fbfd4297 100644 --- a/resources/lib/itemtypes.py +++ b/resources/lib/itemtypes.py @@ -65,7 +65,8 @@ class Items(object): self.kodiconn.close() return self - @CatchExceptions(warnuser=True) + # Don't warn the user - ain't that bad + @CatchExceptions() def getfanart(self, item, kodiId, mediaType, allartworks=None): """ """ From 168014c33ba02dd6c0bd2498d108d34bd0cea446 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 15:49:37 +0200 Subject: [PATCH 21/86] constructor (with Artwork() as art) - Way too slow --- default.py | 4 +- resources/lib/artwork.py | 65 ++++++++++++++++++--------- resources/lib/itemtypes.py | 74 ++++++++++++++++--------------- resources/lib/kodidb_functions.py | 4 +- resources/lib/playbackutils.py | 2 - 5 files changed, 87 insertions(+), 62 deletions(-) diff --git a/default.py b/default.py index 8cfb9ae9..b0ed6ce3 100644 --- a/default.py +++ b/default.py @@ -173,8 +173,8 @@ class Main(): elif mode == "texturecache": import artwork - artwork.Artwork().fullTextureCacheSync() - + with artwork.Artwork('music') as art: + art.fullTextureCacheSync() else: entrypoint.doMainListing() diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index 50add7b9..f0c17e2d 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -120,6 +120,13 @@ def setKodiWebServerDetails(): class Artwork(): + """ + Use like this: + with Artwork(db_type) as art: + art.method() + + db_type: the Kodi db to open: 'video' or 'music' + """ lock = Lock() enableTextureCache = settings('enableTextureCache') == "true" @@ -136,6 +143,25 @@ class Artwork(): imageCacheThreads = [] + def __init__(self, db_type): + self.db_type = db_type + + def __enter__(self): + """ + Open DB connections and cursors + """ + self.connection = kodiSQL(self.db_type) + self.cursor = self.connection.cursor() + return self + + def __exit__(self, exc_type, exc_val, exc_tb): + """ + Make sure DB changes are committed and connection to DB is closed. + """ + self.connection.commit() + self.connection.close() + return self + def double_urlencode(self, text): text = self.single_urlencode(text) text = self.single_urlencode(text) @@ -291,7 +317,7 @@ class Artwork(): else: self.addWorkerImageCacheThread(url) - def addArtwork(self, artwork, kodiId, mediaType, cursor): + def addArtwork(self, artwork, kodiId, mediaType): # Kodi conversion table kodiart = { @@ -322,8 +348,8 @@ class Artwork(): "AND media_type = ?", "AND type LIKE ?" )) - cursor.execute(query, (kodiId, mediaType, "fanart%",)) - rows = cursor.fetchall() + self.cursor.execute(query, (kodiId, mediaType, "fanart%",)) + rows = self.cursor.fetchall() if len(rows) > backdropsNumber: # More backdrops in database. Delete extra fanart. @@ -334,7 +360,7 @@ class Artwork(): "AND media_type = ?", "AND type LIKE ?" )) - cursor.execute(query, (kodiId, mediaType, "fanart_",)) + self.cursor.execute(query, (kodiId, mediaType, "fanart_",)) # Process backdrops and extra fanart index = "" @@ -343,11 +369,11 @@ class Artwork(): imageUrl=backdrop, kodiId=kodiId, mediaType=mediaType, - imageType="%s%s" % ("fanart", index), - cursor=cursor) + imageType="%s%s" % ("fanart", index)) if backdropsNumber > 1: - try: # Will only fail on the first try, str to int. + try: + # Will only fail on the first try, str to int. index += 1 except TypeError: index = 1 @@ -359,19 +385,16 @@ class Artwork(): imageUrl=artwork[art], kodiId=kodiId, mediaType=mediaType, - imageType=artType, - cursor=cursor) - + imageType=artType) elif kodiart.get(art): # Process the rest artwork type that Kodi can use self.addOrUpdateArt( imageUrl=artwork[art], kodiId=kodiId, mediaType=mediaType, - imageType=kodiart[art], - cursor=cursor) + imageType=kodiart[art]) - def addOrUpdateArt(self, imageUrl, kodiId, mediaType, imageType, cursor): + def addOrUpdateArt(self, imageUrl, kodiId, mediaType, imageType): if not imageUrl: # Possible that the imageurl is an empty string return @@ -383,10 +406,10 @@ class Artwork(): "AND media_type = ?", "AND type = ?" )) - cursor.execute(query, (kodiId, mediaType, imageType,)) + self.cursor.execute(query, (kodiId, mediaType, imageType,)) try: # Update the artwork - url = cursor.fetchone()[0] + url = self.cursor.fetchone()[0] except TypeError: # Add the artwork log.debug("Adding Art Link for kodiId: %s (%s)" @@ -397,7 +420,8 @@ class Artwork(): VALUES (?, ?, ?, ?) ''' ) - cursor.execute(query, (kodiId, mediaType, imageType, imageUrl)) + self.cursor.execute(query, + (kodiId, mediaType, imageType, imageUrl)) else: if url == imageUrl: # Only cache artwork if it changed @@ -416,20 +440,21 @@ class Artwork(): "AND media_type = ?", "AND type = ?" )) - cursor.execute(query, (imageUrl, kodiId, mediaType, imageType)) + self.cursor.execute(query, + (imageUrl, kodiId, mediaType, imageType)) # Cache fanart and poster in Kodi texture cache self.cacheTexture(imageUrl) - def deleteArtwork(self, kodiId, mediaType, cursor): + def deleteArtwork(self, kodiId, mediaType): query = ' '.join(( "SELECT url", "FROM art", "WHERE media_id = ?", "AND media_type = ?" )) - cursor.execute(query, (kodiId, mediaType,)) - rows = cursor.fetchall() + self.cursor.execute(query, (kodiId, mediaType,)) + rows = self.cursor.fetchall() for row in rows: self.deleteCachedArtwork(row[0]) diff --git a/resources/lib/itemtypes.py b/resources/lib/itemtypes.py index fbfd4297..a53c5193 100644 --- a/resources/lib/itemtypes.py +++ b/resources/lib/itemtypes.py @@ -39,7 +39,6 @@ class Items(object): self.kodiversion = int(xbmc.getInfoLabel("System.BuildVersion")[:2]) self.directpath = window('useDirectPaths') == 'true' - self.artwork = artwork.Artwork() self.userid = window('currUserId') self.server = window('pms_server') @@ -73,19 +72,19 @@ class Items(object): API = PlexAPI.API(item) if allartworks is None: allartworks = API.getAllArtwork() - self.artwork.addArtwork(API.getFanartArtwork(allartworks), - kodiId, - mediaType, - self.kodicursor) + with artwork.Artwork('video') as art: + art.addArtwork(API.getFanartArtwork(allartworks), + kodiId, + mediaType) # Also get artwork for collections/movie sets if mediaType == 'movie': for setname in API.getCollections(): log.debug('Getting artwork for movie set %s' % setname) setid = self.kodi_db.createBoxset(setname) - self.artwork.addArtwork(API.getSetArtwork(), - setid, - "set", - self.kodicursor) + with artwork.Artwork('video') as art: + art.addArtwork(API.getSetArtwork(), + setid, + "set") self.kodi_db.assignBoxset(setid, kodiId) def itemsbyId(self, items, process, pdialog=None): @@ -309,7 +308,6 @@ class Movies(Items): # Process single movie kodicursor = self.kodicursor emby_db = self.emby_db - artwork = self.artwork API = PlexAPI.API(item) # If the item already exist in the local Kodi DB we'll perform a full @@ -496,7 +494,8 @@ class Movies(Items): # Process genres self.kodi_db.addGenres(movieid, genres, "movie") # Process artwork - artwork.addArtwork(API.getAllArtwork(), movieid, "movie", kodicursor) + with artwork.Artwork('video') as art: + art.addArtwork(API.getAllArtwork(), movieid, "movie") # Process stream details self.kodi_db.addStreams(fileid, API.getMediaStreams(), runtime) # Process studios @@ -516,8 +515,6 @@ class Movies(Items): # Remove movieid, fileid, emby reference emby_db = self.emby_db kodicursor = self.kodicursor - artwork = self.artwork - emby_dbitem = emby_db.getItem_byId(itemid) try: kodiid = emby_dbitem[0] @@ -531,7 +528,8 @@ class Movies(Items): # Remove the emby reference emby_db.removeItem(itemid) # Remove artwork - artwork.deleteArtwork(kodiid, mediatype, kodicursor) + with artwork.Artwork('video') as art: + art.deleteArtwork(kodiid, mediatype) if mediatype == "movie": # Delete kodi movie and file @@ -561,7 +559,6 @@ class TVShows(Items): # Process single tvshow kodicursor = self.kodicursor emby_db = self.emby_db - artwork = self.artwork API = PlexAPI.API(item) update_item = True @@ -719,8 +716,8 @@ class TVShows(Items): # Process genres self.kodi_db.addGenres(showid, genres, "tvshow") # Process artwork - allartworks = API.getAllArtwork() - artwork.addArtwork(allartworks, showid, "tvshow", kodicursor) + with artwork.Artwork('video') as art: + art.addArtwork(API.getAllArtwork(), showid, "tvshow") # Process studios self.kodi_db.addStudios(showid, studios, "tvshow") # Process tags: view, PMS collection tags @@ -743,7 +740,6 @@ class TVShows(Items): return kodicursor = self.kodicursor emby_db = self.emby_db - artwork = self.artwork seasonnum = API.getIndex() # Get parent tv show Plex id plexshowid = item.attrib.get('parentRatingKey') @@ -767,8 +763,8 @@ class TVShows(Items): update_item = False # Process artwork - allartworks = API.getAllArtwork() - artwork.addArtwork(allartworks, seasonid, "season", kodicursor) + with artwork.Artwork('video') as art: + art.addArtwork(API.getAllArtwork(), seasonid, "season") if update_item: # Update a reference: checksum in emby table @@ -790,7 +786,6 @@ class TVShows(Items): # Process single episode kodicursor = self.kodicursor emby_db = self.emby_db - artwork = self.artwork API = PlexAPI.API(item) # If the item already exist in the local Kodi DB we'll perform a full @@ -1037,8 +1032,8 @@ class TVShows(Items): if poster: poster = API.addPlexCredentialsToUrl( "%s%s" % (self.server, poster)) - artwork.addOrUpdateArt( - poster, episodeid, "episode", "thumb", kodicursor) + with artwork.Artwork('video') as art: + art.addOrUpdateArt(poster, episodeid, "episode", "thumb") # poster of TV show itself # poster = item.attrib.get('grandparentThumb') # if poster: @@ -1177,19 +1172,22 @@ class TVShows(Items): def removeShow(self, kodiid): kodicursor = self.kodicursor - self.artwork.deleteArtwork(kodiid, "tvshow", kodicursor) + with artwork.Artwork('video') as art: + art.deleteArtwork(kodiid, "tvshow") kodicursor.execute("DELETE FROM tvshow WHERE idShow = ?", (kodiid,)) log.info("Removed tvshow: %s." % kodiid) def removeSeason(self, kodiid): kodicursor = self.kodicursor - self.artwork.deleteArtwork(kodiid, "season", kodicursor) + with artwork.Artwork('video') as art: + art.deleteArtwork(kodiid, "season") kodicursor.execute("DELETE FROM seasons WHERE idSeason = ?", (kodiid,)) log.info("Removed season: %s." % kodiid) def removeEpisode(self, kodiid, fileid): kodicursor = self.kodicursor - self.artwork.deleteArtwork(kodiid, "episode", kodicursor) + with artwork.Artwork('video') as art: + art.deleteArtwork(kodiid, "episode") kodicursor.execute("DELETE FROM episode WHERE idEpisode = ?", (kodiid,)) kodicursor.execute("DELETE FROM files WHERE idFile = ?", (fileid,)) log.info("Removed episode: %s." % kodiid) @@ -1224,7 +1222,6 @@ class Music(Items): artisttype="MusicArtist"): kodicursor = self.kodicursor emby_db = self.emby_db - artwork = self.artwork API = PlexAPI.API(item) update_item = True @@ -1299,13 +1296,13 @@ class Music(Items): dateadded, artistid)) # Update artwork - artwork.addArtwork(artworks, artistid, "artist", kodicursor) + with artwork.Artwork('music') as art: + art.addArtwork(artworks, artistid, "artist") @CatchExceptions(warnuser=True) def add_updateAlbum(self, item, viewtag=None, viewid=None): kodicursor = self.kodicursor emby_db = self.emby_db - artwork = self.artwork API = PlexAPI.API(item) update_item = True @@ -1488,14 +1485,14 @@ class Music(Items): # Add genres self.kodi_db.addMusicGenres(albumid, genres, "album") # Update artwork - artwork.addArtwork(artworks, albumid, "album", kodicursor) + with artwork.Artwork('music') as art: + art.addArtwork(artworks, albumid, "album") @CatchExceptions(warnuser=True) def add_updateSong(self, item, viewtag=None, viewid=None): # Process single song kodicursor = self.kodicursor emby_db = self.emby_db - artwork = self.artwork API = PlexAPI.API(item) update_item = True @@ -1821,12 +1818,14 @@ class Music(Items): allart = API.getAllArtwork(parentInfo=True) if hasEmbeddedCover: allart["Primary"] = "image://music@" + artwork.single_urlencode( playurl ) - artwork.addArtwork(allart, songid, "song", kodicursor) + with artwork.Artwork('music') as art: + art.addArtwork(allart, songid, "song") # if item.get('AlbumId') is None: if item.get('parentKey') is None: # Update album artwork - artwork.addArtwork(allart, albumid, "album", kodicursor) + with artwork.Artwork('music') as art: + art.addArtwork(allart, albumid, "album") def remove(self, itemid): # Remove kodiid, fileid, pathid, emby reference @@ -1906,16 +1905,19 @@ class Music(Items): log.info("Deleted %s: %s from kodi database" % (mediatype, itemid)) def removeSong(self, kodiid): - self.artwork.deleteArtwork(kodiid, "song", self.kodicursor) + with artwork.Artwork('music') as art: + art.deleteArtwork(kodiid, "song") self.kodicursor.execute("DELETE FROM song WHERE idSong = ?", (kodiid,)) def removeAlbum(self, kodiid): - self.artwork.deleteArtwork(kodiid, "album", self.kodicursor) + with artwork.Artwork('music') as art: + art.deleteArtwork(kodiid, "album") self.kodicursor.execute("DELETE FROM album WHERE idAlbum = ?", (kodiid,)) def removeArtist(self, kodiid): - self.artwork.deleteArtwork(kodiid, "artist", self.kodicursor) + with artwork.Artwork('music') as art: + art.deleteArtwork(kodiid, "artist") self.kodicursor.execute("DELETE FROM artist WHERE idArtist = ?", (kodiid,)) diff --git a/resources/lib/kodidb_functions.py b/resources/lib/kodidb_functions.py index d2950017..6427af4a 100644 --- a/resources/lib/kodidb_functions.py +++ b/resources/lib/kodidb_functions.py @@ -50,7 +50,6 @@ class Kodidb_Functions(): self.cursor = cursor self.clientInfo = clientinfo.ClientInfo() - self.artwork = artwork.Artwork() def pathHack(self): """ @@ -506,7 +505,8 @@ class Kodidb_Functions(): if "writing" in arttype: arttype = "writer" - self.artwork.addOrUpdateArt(thumb, actorid, arttype, "thumb", self.cursor) + with artwork.Artwork('video') as art: + art.addOrUpdateArt(thumb, actorid, arttype, "thumb") def existingArt(self, kodiId, mediaType, refresh=False): """ diff --git a/resources/lib/playbackutils.py b/resources/lib/playbackutils.py index 65a8f5d1..b5f82ee2 100644 --- a/resources/lib/playbackutils.py +++ b/resources/lib/playbackutils.py @@ -11,7 +11,6 @@ import xbmc import xbmcgui import xbmcplugin -import artwork import playutils as putils import playlist from utils import window, settings, tryEncode, tryDecode @@ -39,7 +38,6 @@ class PlaybackUtils(): self.userid = window('currUserId') self.server = window('pms_server') - self.artwork = artwork.Artwork() if self.API.getType() == 'track': self.pl = playlist.Playlist(typus='music') else: From c4de7587bfdd22e88b626747e3827786c27460e2 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 15:49:50 +0200 Subject: [PATCH 22/86] Revert "constructor (with Artwork() as art)" This reverts commit 168014c33ba02dd6c0bd2498d108d34bd0cea446. --- default.py | 4 +- resources/lib/artwork.py | 65 +++++++++------------------ resources/lib/itemtypes.py | 74 +++++++++++++++---------------- resources/lib/kodidb_functions.py | 4 +- resources/lib/playbackutils.py | 2 + 5 files changed, 62 insertions(+), 87 deletions(-) diff --git a/default.py b/default.py index b0ed6ce3..8cfb9ae9 100644 --- a/default.py +++ b/default.py @@ -173,8 +173,8 @@ class Main(): elif mode == "texturecache": import artwork - with artwork.Artwork('music') as art: - art.fullTextureCacheSync() + artwork.Artwork().fullTextureCacheSync() + else: entrypoint.doMainListing() diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index f0c17e2d..50add7b9 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -120,13 +120,6 @@ def setKodiWebServerDetails(): class Artwork(): - """ - Use like this: - with Artwork(db_type) as art: - art.method() - - db_type: the Kodi db to open: 'video' or 'music' - """ lock = Lock() enableTextureCache = settings('enableTextureCache') == "true" @@ -143,25 +136,6 @@ class Artwork(): imageCacheThreads = [] - def __init__(self, db_type): - self.db_type = db_type - - def __enter__(self): - """ - Open DB connections and cursors - """ - self.connection = kodiSQL(self.db_type) - self.cursor = self.connection.cursor() - return self - - def __exit__(self, exc_type, exc_val, exc_tb): - """ - Make sure DB changes are committed and connection to DB is closed. - """ - self.connection.commit() - self.connection.close() - return self - def double_urlencode(self, text): text = self.single_urlencode(text) text = self.single_urlencode(text) @@ -317,7 +291,7 @@ class Artwork(): else: self.addWorkerImageCacheThread(url) - def addArtwork(self, artwork, kodiId, mediaType): + def addArtwork(self, artwork, kodiId, mediaType, cursor): # Kodi conversion table kodiart = { @@ -348,8 +322,8 @@ class Artwork(): "AND media_type = ?", "AND type LIKE ?" )) - self.cursor.execute(query, (kodiId, mediaType, "fanart%",)) - rows = self.cursor.fetchall() + cursor.execute(query, (kodiId, mediaType, "fanart%",)) + rows = cursor.fetchall() if len(rows) > backdropsNumber: # More backdrops in database. Delete extra fanart. @@ -360,7 +334,7 @@ class Artwork(): "AND media_type = ?", "AND type LIKE ?" )) - self.cursor.execute(query, (kodiId, mediaType, "fanart_",)) + cursor.execute(query, (kodiId, mediaType, "fanart_",)) # Process backdrops and extra fanart index = "" @@ -369,11 +343,11 @@ class Artwork(): imageUrl=backdrop, kodiId=kodiId, mediaType=mediaType, - imageType="%s%s" % ("fanart", index)) + imageType="%s%s" % ("fanart", index), + cursor=cursor) if backdropsNumber > 1: - try: - # Will only fail on the first try, str to int. + try: # Will only fail on the first try, str to int. index += 1 except TypeError: index = 1 @@ -385,16 +359,19 @@ class Artwork(): imageUrl=artwork[art], kodiId=kodiId, mediaType=mediaType, - imageType=artType) + imageType=artType, + cursor=cursor) + elif kodiart.get(art): # Process the rest artwork type that Kodi can use self.addOrUpdateArt( imageUrl=artwork[art], kodiId=kodiId, mediaType=mediaType, - imageType=kodiart[art]) + imageType=kodiart[art], + cursor=cursor) - def addOrUpdateArt(self, imageUrl, kodiId, mediaType, imageType): + def addOrUpdateArt(self, imageUrl, kodiId, mediaType, imageType, cursor): if not imageUrl: # Possible that the imageurl is an empty string return @@ -406,10 +383,10 @@ class Artwork(): "AND media_type = ?", "AND type = ?" )) - self.cursor.execute(query, (kodiId, mediaType, imageType,)) + cursor.execute(query, (kodiId, mediaType, imageType,)) try: # Update the artwork - url = self.cursor.fetchone()[0] + url = cursor.fetchone()[0] except TypeError: # Add the artwork log.debug("Adding Art Link for kodiId: %s (%s)" @@ -420,8 +397,7 @@ class Artwork(): VALUES (?, ?, ?, ?) ''' ) - self.cursor.execute(query, - (kodiId, mediaType, imageType, imageUrl)) + cursor.execute(query, (kodiId, mediaType, imageType, imageUrl)) else: if url == imageUrl: # Only cache artwork if it changed @@ -440,21 +416,20 @@ class Artwork(): "AND media_type = ?", "AND type = ?" )) - self.cursor.execute(query, - (imageUrl, kodiId, mediaType, imageType)) + cursor.execute(query, (imageUrl, kodiId, mediaType, imageType)) # Cache fanart and poster in Kodi texture cache self.cacheTexture(imageUrl) - def deleteArtwork(self, kodiId, mediaType): + def deleteArtwork(self, kodiId, mediaType, cursor): query = ' '.join(( "SELECT url", "FROM art", "WHERE media_id = ?", "AND media_type = ?" )) - self.cursor.execute(query, (kodiId, mediaType,)) - rows = self.cursor.fetchall() + cursor.execute(query, (kodiId, mediaType,)) + rows = cursor.fetchall() for row in rows: self.deleteCachedArtwork(row[0]) diff --git a/resources/lib/itemtypes.py b/resources/lib/itemtypes.py index a53c5193..fbfd4297 100644 --- a/resources/lib/itemtypes.py +++ b/resources/lib/itemtypes.py @@ -39,6 +39,7 @@ class Items(object): self.kodiversion = int(xbmc.getInfoLabel("System.BuildVersion")[:2]) self.directpath = window('useDirectPaths') == 'true' + self.artwork = artwork.Artwork() self.userid = window('currUserId') self.server = window('pms_server') @@ -72,19 +73,19 @@ class Items(object): API = PlexAPI.API(item) if allartworks is None: allartworks = API.getAllArtwork() - with artwork.Artwork('video') as art: - art.addArtwork(API.getFanartArtwork(allartworks), - kodiId, - mediaType) + self.artwork.addArtwork(API.getFanartArtwork(allartworks), + kodiId, + mediaType, + self.kodicursor) # Also get artwork for collections/movie sets if mediaType == 'movie': for setname in API.getCollections(): log.debug('Getting artwork for movie set %s' % setname) setid = self.kodi_db.createBoxset(setname) - with artwork.Artwork('video') as art: - art.addArtwork(API.getSetArtwork(), - setid, - "set") + self.artwork.addArtwork(API.getSetArtwork(), + setid, + "set", + self.kodicursor) self.kodi_db.assignBoxset(setid, kodiId) def itemsbyId(self, items, process, pdialog=None): @@ -308,6 +309,7 @@ class Movies(Items): # Process single movie kodicursor = self.kodicursor emby_db = self.emby_db + artwork = self.artwork API = PlexAPI.API(item) # If the item already exist in the local Kodi DB we'll perform a full @@ -494,8 +496,7 @@ class Movies(Items): # Process genres self.kodi_db.addGenres(movieid, genres, "movie") # Process artwork - with artwork.Artwork('video') as art: - art.addArtwork(API.getAllArtwork(), movieid, "movie") + artwork.addArtwork(API.getAllArtwork(), movieid, "movie", kodicursor) # Process stream details self.kodi_db.addStreams(fileid, API.getMediaStreams(), runtime) # Process studios @@ -515,6 +516,8 @@ class Movies(Items): # Remove movieid, fileid, emby reference emby_db = self.emby_db kodicursor = self.kodicursor + artwork = self.artwork + emby_dbitem = emby_db.getItem_byId(itemid) try: kodiid = emby_dbitem[0] @@ -528,8 +531,7 @@ class Movies(Items): # Remove the emby reference emby_db.removeItem(itemid) # Remove artwork - with artwork.Artwork('video') as art: - art.deleteArtwork(kodiid, mediatype) + artwork.deleteArtwork(kodiid, mediatype, kodicursor) if mediatype == "movie": # Delete kodi movie and file @@ -559,6 +561,7 @@ class TVShows(Items): # Process single tvshow kodicursor = self.kodicursor emby_db = self.emby_db + artwork = self.artwork API = PlexAPI.API(item) update_item = True @@ -716,8 +719,8 @@ class TVShows(Items): # Process genres self.kodi_db.addGenres(showid, genres, "tvshow") # Process artwork - with artwork.Artwork('video') as art: - art.addArtwork(API.getAllArtwork(), showid, "tvshow") + allartworks = API.getAllArtwork() + artwork.addArtwork(allartworks, showid, "tvshow", kodicursor) # Process studios self.kodi_db.addStudios(showid, studios, "tvshow") # Process tags: view, PMS collection tags @@ -740,6 +743,7 @@ class TVShows(Items): return kodicursor = self.kodicursor emby_db = self.emby_db + artwork = self.artwork seasonnum = API.getIndex() # Get parent tv show Plex id plexshowid = item.attrib.get('parentRatingKey') @@ -763,8 +767,8 @@ class TVShows(Items): update_item = False # Process artwork - with artwork.Artwork('video') as art: - art.addArtwork(API.getAllArtwork(), seasonid, "season") + allartworks = API.getAllArtwork() + artwork.addArtwork(allartworks, seasonid, "season", kodicursor) if update_item: # Update a reference: checksum in emby table @@ -786,6 +790,7 @@ class TVShows(Items): # Process single episode kodicursor = self.kodicursor emby_db = self.emby_db + artwork = self.artwork API = PlexAPI.API(item) # If the item already exist in the local Kodi DB we'll perform a full @@ -1032,8 +1037,8 @@ class TVShows(Items): if poster: poster = API.addPlexCredentialsToUrl( "%s%s" % (self.server, poster)) - with artwork.Artwork('video') as art: - art.addOrUpdateArt(poster, episodeid, "episode", "thumb") + artwork.addOrUpdateArt( + poster, episodeid, "episode", "thumb", kodicursor) # poster of TV show itself # poster = item.attrib.get('grandparentThumb') # if poster: @@ -1172,22 +1177,19 @@ class TVShows(Items): def removeShow(self, kodiid): kodicursor = self.kodicursor - with artwork.Artwork('video') as art: - art.deleteArtwork(kodiid, "tvshow") + self.artwork.deleteArtwork(kodiid, "tvshow", kodicursor) kodicursor.execute("DELETE FROM tvshow WHERE idShow = ?", (kodiid,)) log.info("Removed tvshow: %s." % kodiid) def removeSeason(self, kodiid): kodicursor = self.kodicursor - with artwork.Artwork('video') as art: - art.deleteArtwork(kodiid, "season") + self.artwork.deleteArtwork(kodiid, "season", kodicursor) kodicursor.execute("DELETE FROM seasons WHERE idSeason = ?", (kodiid,)) log.info("Removed season: %s." % kodiid) def removeEpisode(self, kodiid, fileid): kodicursor = self.kodicursor - with artwork.Artwork('video') as art: - art.deleteArtwork(kodiid, "episode") + self.artwork.deleteArtwork(kodiid, "episode", kodicursor) kodicursor.execute("DELETE FROM episode WHERE idEpisode = ?", (kodiid,)) kodicursor.execute("DELETE FROM files WHERE idFile = ?", (fileid,)) log.info("Removed episode: %s." % kodiid) @@ -1222,6 +1224,7 @@ class Music(Items): artisttype="MusicArtist"): kodicursor = self.kodicursor emby_db = self.emby_db + artwork = self.artwork API = PlexAPI.API(item) update_item = True @@ -1296,13 +1299,13 @@ class Music(Items): dateadded, artistid)) # Update artwork - with artwork.Artwork('music') as art: - art.addArtwork(artworks, artistid, "artist") + artwork.addArtwork(artworks, artistid, "artist", kodicursor) @CatchExceptions(warnuser=True) def add_updateAlbum(self, item, viewtag=None, viewid=None): kodicursor = self.kodicursor emby_db = self.emby_db + artwork = self.artwork API = PlexAPI.API(item) update_item = True @@ -1485,14 +1488,14 @@ class Music(Items): # Add genres self.kodi_db.addMusicGenres(albumid, genres, "album") # Update artwork - with artwork.Artwork('music') as art: - art.addArtwork(artworks, albumid, "album") + artwork.addArtwork(artworks, albumid, "album", kodicursor) @CatchExceptions(warnuser=True) def add_updateSong(self, item, viewtag=None, viewid=None): # Process single song kodicursor = self.kodicursor emby_db = self.emby_db + artwork = self.artwork API = PlexAPI.API(item) update_item = True @@ -1818,14 +1821,12 @@ class Music(Items): allart = API.getAllArtwork(parentInfo=True) if hasEmbeddedCover: allart["Primary"] = "image://music@" + artwork.single_urlencode( playurl ) - with artwork.Artwork('music') as art: - art.addArtwork(allart, songid, "song") + artwork.addArtwork(allart, songid, "song", kodicursor) # if item.get('AlbumId') is None: if item.get('parentKey') is None: # Update album artwork - with artwork.Artwork('music') as art: - art.addArtwork(allart, albumid, "album") + artwork.addArtwork(allart, albumid, "album", kodicursor) def remove(self, itemid): # Remove kodiid, fileid, pathid, emby reference @@ -1905,19 +1906,16 @@ class Music(Items): log.info("Deleted %s: %s from kodi database" % (mediatype, itemid)) def removeSong(self, kodiid): - with artwork.Artwork('music') as art: - art.deleteArtwork(kodiid, "song") + self.artwork.deleteArtwork(kodiid, "song", self.kodicursor) self.kodicursor.execute("DELETE FROM song WHERE idSong = ?", (kodiid,)) def removeAlbum(self, kodiid): - with artwork.Artwork('music') as art: - art.deleteArtwork(kodiid, "album") + self.artwork.deleteArtwork(kodiid, "album", self.kodicursor) self.kodicursor.execute("DELETE FROM album WHERE idAlbum = ?", (kodiid,)) def removeArtist(self, kodiid): - with artwork.Artwork('music') as art: - art.deleteArtwork(kodiid, "artist") + self.artwork.deleteArtwork(kodiid, "artist", self.kodicursor) self.kodicursor.execute("DELETE FROM artist WHERE idArtist = ?", (kodiid,)) diff --git a/resources/lib/kodidb_functions.py b/resources/lib/kodidb_functions.py index 6427af4a..d2950017 100644 --- a/resources/lib/kodidb_functions.py +++ b/resources/lib/kodidb_functions.py @@ -50,6 +50,7 @@ class Kodidb_Functions(): self.cursor = cursor self.clientInfo = clientinfo.ClientInfo() + self.artwork = artwork.Artwork() def pathHack(self): """ @@ -505,8 +506,7 @@ class Kodidb_Functions(): if "writing" in arttype: arttype = "writer" - with artwork.Artwork('video') as art: - art.addOrUpdateArt(thumb, actorid, arttype, "thumb") + self.artwork.addOrUpdateArt(thumb, actorid, arttype, "thumb", self.cursor) def existingArt(self, kodiId, mediaType, refresh=False): """ diff --git a/resources/lib/playbackutils.py b/resources/lib/playbackutils.py index b5f82ee2..65a8f5d1 100644 --- a/resources/lib/playbackutils.py +++ b/resources/lib/playbackutils.py @@ -11,6 +11,7 @@ import xbmc import xbmcgui import xbmcplugin +import artwork import playutils as putils import playlist from utils import window, settings, tryEncode, tryDecode @@ -38,6 +39,7 @@ class PlaybackUtils(): self.userid = window('currUserId') self.server = window('pms_server') + self.artwork = artwork.Artwork() if self.API.getType() == 'track': self.pl = playlist.Playlist(typus='music') else: From 61a2457a1dfe50051d1389a5ec06d9208dec97cf Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 16:05:05 +0200 Subject: [PATCH 23/86] Less logging --- resources/lib/image_cache_thread.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/resources/lib/image_cache_thread.py b/resources/lib/image_cache_thread.py index e0714e93..594b6f52 100644 --- a/resources/lib/image_cache_thread.py +++ b/resources/lib/image_cache_thread.py @@ -47,9 +47,6 @@ class ImageCacheThread(threading.Thread): self.xbmc_password = password def run(self): - - log.debug("Image Caching Thread Processing: %s", self.url_to_process) - try: response = requests.head( url=( @@ -60,6 +57,4 @@ class ImageCacheThread(threading.Thread): # We don't need the result except Exception: pass - - log.debug("Image Caching Thread Exited") self.is_finished = True From 9c46757b424c3717551a861575ddd367692de849 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 17:31:38 +0200 Subject: [PATCH 24/86] Do warn user --- resources/lib/itemtypes.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/resources/lib/itemtypes.py b/resources/lib/itemtypes.py index fbfd4297..170ddf5f 100644 --- a/resources/lib/itemtypes.py +++ b/resources/lib/itemtypes.py @@ -65,8 +65,7 @@ class Items(object): self.kodiconn.close() return self - # Don't warn the user - ain't that bad - @CatchExceptions() + @CatchExceptions(warnuser=True) def getfanart(self, item, kodiId, mediaType, allartworks=None): """ """ From 4cfb8c4610bbb05aa1e2d96b69e3e904da14a22a Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 17:51:00 +0200 Subject: [PATCH 25/86] Fix OperationalError for Kodi DB - Fanart sync is too taxing, needs to be done after regular syncs --- resources/lib/librarysync.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 0bc25f31..3401ec85 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -292,11 +292,20 @@ class ProcessFanartThread(Thread): log.info("---===### Stopped FanartSync ###===---") return xbmc.sleep(1000) + while window('plex_dbScan'): + # Don't do background sync if there is another sync + # going - otherwise we will have OperationalError for + # Kodi DB changes! + if threadStopped(): + # Abort was requested while waiting. We should exit + log.info("---===### Stopped FanartSync ###===---") + return + xbmc.sleep(1000) # grabs Plex item from queue try: item = queue.get(block=False) except Queue.Empty: - xbmc.sleep(50) + xbmc.sleep(200) continue if item['refresh'] is True: # Leave the Plex art untouched From c9dd44f49870afb5596db6073dce2c773bdaff9f Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 17:59:00 +0200 Subject: [PATCH 26/86] Don't try to get fanart for episodes, seasons --- resources/lib/librarysync.py | 44 +++++++++++++++++++++--------------- 1 file changed, 26 insertions(+), 18 deletions(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 3401ec85..29597d9d 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -867,6 +867,7 @@ class LibrarySync(Thread): 'viewName': xxx, 'viewId': xxx, 'title': xxx + 'mediaType': xxx, e.g. 'movie', 'episode' self.allPlexElementsId APPENDED(!!) dict = {itemid: checksum} @@ -886,12 +887,15 @@ class LibrarySync(Thread): # Only update if movie is not in Kodi or checksum is # different if kodi_checksum != plex_checksum: - self.updatelist.append({'itemId': itemId, - 'itemType': itemType, - 'method': method, - 'viewName': viewName, - 'viewId': viewId, - 'title': title}) + self.updatelist.append({ + 'itemId': itemId, + 'itemType': itemType, + 'method': method, + 'viewName': viewName, + 'viewId': viewId, + 'title': title, + 'mediaType': item.attrib.get('type') + }) else: # Initial or repair sync: get all Plex movies for item in xml: @@ -903,12 +907,15 @@ class LibrarySync(Thread): plex_checksum = ("K%s%s" % (itemId, item.attrib.get('updatedAt', ''))) self.allPlexElementsId[itemId] = plex_checksum - self.updatelist.append({'itemId': itemId, - 'itemType': itemType, - 'method': method, - 'viewName': viewName, - 'viewId': viewId, - 'title': title}) + self.updatelist.append({ + 'itemId': itemId, + 'itemType': itemType, + 'method': method, + 'viewName': viewName, + 'viewId': viewId, + 'title': title, + 'mediaType': item.attrib.get('type') + }) def GetAndProcessXMLs(self, itemType, showProgress=True): """ @@ -999,12 +1006,13 @@ class LibrarySync(Thread): # Save to queue for later processing typus = {'Movies': 'movie', 'TVShows': 'tvshow'}[itemType] for item in self.updatelist: - self.fanartqueue.put({ - 'itemId': item['itemId'], - 'class': itemType, - 'mediaType': typus, - 'refresh': False - }) + if item['mediaType'] in ('movie', 'tvshow'): + self.fanartqueue.put({ + 'itemId': item['itemId'], + 'class': itemType, + 'mediaType': typus, + 'refresh': False + }) self.updatelist = [] @LogTime From a2a53cddf6aa936766e2ed4228aad82081298cb0 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 11 Sep 2016 18:31:25 +0200 Subject: [PATCH 27/86] Fix AttributeError --- resources/lib/image_cache_thread.py | 9 ++++----- 1 file changed, 4 insertions(+), 5 deletions(-) diff --git a/resources/lib/image_cache_thread.py b/resources/lib/image_cache_thread.py index 594b6f52..b01377a4 100644 --- a/resources/lib/image_cache_thread.py +++ b/resources/lib/image_cache_thread.py @@ -49,11 +49,10 @@ class ImageCacheThread(threading.Thread): def run(self): try: response = requests.head( - url=( - "http://%s:%s/image/image://%s" - % (self.xbmc_host, self.xbmc_port, self.urlToProcess)), - auth=(self.xbmc_username, self.xbmc_password), - timeout=(5, 5)) + url=("http://%s:%s/image/image://%s" + % (self.xbmc_host, self.xbmc_port, self.url_to_process)), + auth=(self.xbmc_username, self.xbmc_password), + timeout=(5, 5)) # We don't need the result except Exception: pass From a3d643c6435f92a1a5f77769a43a1808c9599eab Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Fri, 16 Sep 2016 17:42:39 +0200 Subject: [PATCH 28/86] Use some file settings instead of window settings - In case the user changes something while PKC is running --- resources/lib/PlexAPI.py | 15 +++++++-------- resources/lib/librarysync.py | 3 --- resources/lib/userclient.py | 16 ---------------- service.py | 3 --- 4 files changed, 7 insertions(+), 30 deletions(-) diff --git a/resources/lib/PlexAPI.py b/resources/lib/PlexAPI.py index 1a1e2190..54017ea6 100644 --- a/resources/lib/PlexAPI.py +++ b/resources/lib/PlexAPI.py @@ -2515,16 +2515,16 @@ class API(): 'photo': 'photo' } typus = types[typus] - if window('remapSMB') == 'true': - path = path.replace(window('remapSMB%sOrg' % typus), - window('remapSMB%sNew' % typus), + if settings('remapSMB') == 'true': + path = path.replace(settings('remapSMB%sOrg' % typus), + settings('remapSMB%sNew' % typus), 1) # There might be backslashes left over: path = path.replace('\\', '/') - elif window('replaceSMB') == 'true': + elif settings('replaceSMB') == 'true': if path.startswith('\\\\'): path = 'smb:' + path.replace('\\', '/') - if window('plex_pathverified') == 'true' and forceCheck is False: + if settings('plex_pathverified') == 'true' and forceCheck is False: return path # exist() needs a / or \ at the end to work for directories @@ -2545,13 +2545,12 @@ class API(): if self.askToValidate(path): window('plex_shouldStop', value="true") path = None - window('plex_pathverified', value='true') + settings('plex_pathverified', value='true') settings('plex_pathverified', value='true') else: path = None elif forceCheck is False: - if window('plex_pathverified') != 'true': - window('plex_pathverified', value='true') + if settings('plex_pathverified') != 'true': settings('plex_pathverified', value='true') return path diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 29597d9d..3be231bd 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -399,9 +399,6 @@ class LibrarySync(Thread): 'enableBackgroundSync') == "true" self.limitindex = int(settings('limitindex')) - if settings('plex_pathverified') == 'true': - window('plex_pathverified', value='true') - # Just in case a time sync goes wrong self.timeoffset = int(settings('kodiplextimeoffset')) window('kodiplextimeoffset', value=str(self.timeoffset)) diff --git a/resources/lib/userclient.py b/resources/lib/userclient.py index ed9be930..308dc06e 100644 --- a/resources/lib/userclient.py +++ b/resources/lib/userclient.py @@ -178,22 +178,6 @@ class UserClient(threading.Thread): window('useDirectPaths', value='true' if settings('useDirectPaths') == "1" else 'false') - window('replaceSMB', value='true' - if settings('replaceSMB') == "true" else 'false') - window('remapSMB', value='true' - if settings('remapSMB') == "true" else 'false') - if window('remapSMB') == 'true': - items = ('movie', 'tv', 'music') - for item in items: - # Normalize! Get rid of potential (back)slashes at the end - org = settings('remapSMB%sOrg' % item) - new = settings('remapSMB%sNew' % item) - if org.endswith('\\') or org.endswith('/'): - org = org[:-1] - if new.endswith('\\') or new.endswith('/'): - new = new[:-1] - window('remapSMB%sOrg' % item, value=org) - window('remapSMB%sNew' % item, value=new) # Start DownloadUtils session doUtils.startSession(reset=True) diff --git a/service.py b/service.py index 180dac3c..8183043f 100644 --- a/service.py +++ b/service.py @@ -97,9 +97,6 @@ class Service(): "plex_runLibScan", "plex_username", "pms_token", "plex_token", "pms_server", "plex_machineIdentifier", "plex_servername", "plex_authenticated", "PlexUserImage", "useDirectPaths", - "replaceSMB", "remapSMB", "remapSMBmovieOrg", "remapSMBtvOrg", - "remapSMBmusicOrg", "remapSMBmovieNew", "remapSMBtvNew", - "remapSMBmusicNew", "remapSMBphotoOrg", "remapSMBphotoNew", "suspend_LibraryThread", "plex_terminateNow", "kodiplextimeoffset", "countError", "countUnauthorized" ] From 4d7af7912d3c36cb031e44debdda379c47ddb668 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Fri, 16 Sep 2016 17:47:09 +0200 Subject: [PATCH 29/86] Use file settings instead of window settings --- resources/lib/librarysync.py | 4 ++-- resources/lib/userclient.py | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 3be231bd..d5ae345b 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -1535,7 +1535,7 @@ class LibrarySync(Thread): sessionKey = item.get('sessionKey') # Do we already have a sessionKey stored? if sessionKey not in self.sessionKeys: - if window('plex_serverowned') == 'false': + if settings('plex_serverowned') == 'false': # Not our PMS, we are not authorized to get the # sessions # On the bright side, it must be us playing :-) @@ -1554,7 +1554,7 @@ class LibrarySync(Thread): continue currSess = self.sessionKeys[sessionKey] - if window('plex_serverowned') != 'false': + if settings('plex_serverowned') != 'false': # Identify the user - same one as signed on with PKC? Skip # update if neither session's username nor userid match # (Owner sometime's returns id '1', not always) diff --git a/resources/lib/userclient.py b/resources/lib/userclient.py index 308dc06e..78aff86b 100644 --- a/resources/lib/userclient.py +++ b/resources/lib/userclient.py @@ -174,7 +174,6 @@ class UserClient(threading.Thread): window('plex_machineIdentifier', value=self.machineIdentifier) window('plex_servername', value=self.servername) window('plex_authenticated', value='true') - window('plex_serverowned', value=settings('plex_serverowned')) window('useDirectPaths', value='true' if settings('useDirectPaths') == "1" else 'false') From 8743ce13af21702700da70674679c17fc50a0f7e Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Fri, 16 Sep 2016 17:51:30 +0200 Subject: [PATCH 30/86] Remove obsolete code --- resources/lib/PlexAPI.py | 1 - 1 file changed, 1 deletion(-) diff --git a/resources/lib/PlexAPI.py b/resources/lib/PlexAPI.py index 54017ea6..146c787d 100644 --- a/resources/lib/PlexAPI.py +++ b/resources/lib/PlexAPI.py @@ -2546,7 +2546,6 @@ class API(): window('plex_shouldStop', value="true") path = None settings('plex_pathverified', value='true') - settings('plex_pathverified', value='true') else: path = None elif forceCheck is False: From daf23fcc55d41ae12984c8d24f5b5e298985769c Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 09:48:38 +0200 Subject: [PATCH 31/86] Merge while loops --- resources/lib/librarysync.py | 11 +---------- 1 file changed, 1 insertion(+), 10 deletions(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index d5ae345b..d342582f 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -285,22 +285,13 @@ class ProcessFanartThread(Thread): log.info("---===### Starting FanartSync ###===---") while not threadStopped(): # In the event the server goes offline - while threadSuspended(): + while threadSuspended() or window('plex_dbScan'): # Set in service.py if threadStopped(): # Abort was requested while waiting. We should exit log.info("---===### Stopped FanartSync ###===---") return xbmc.sleep(1000) - while window('plex_dbScan'): - # Don't do background sync if there is another sync - # going - otherwise we will have OperationalError for - # Kodi DB changes! - if threadStopped(): - # Abort was requested while waiting. We should exit - log.info("---===### Stopped FanartSync ###===---") - return - xbmc.sleep(1000) # grabs Plex item from queue try: item = queue.get(block=False) From 83029b75603c6d788aaef6eebbca9fa17b410c68 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 09:55:06 +0200 Subject: [PATCH 32/86] Shorten for loop --- resources/lib/librarysync.py | 1 + 1 file changed, 1 insertion(+) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index d342582f..868fcd4a 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -318,6 +318,7 @@ class ProcessFanartThread(Thread): for key, value in allartworks.iteritems(): if not value and not key == 'BoxRear': needsupdate = True + break if needsupdate is False: log.debug('Already got all art for Plex id %s' % item['itemId']) From dc94cc6bfe8a328d98f35577d641c8c7879fda67 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 10:58:03 +0200 Subject: [PATCH 33/86] Dedicated art downloading queue --- resources/lib/artwork.py | 107 ++++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 35 deletions(-) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index 50add7b9..ee1f6132 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -8,7 +8,8 @@ import requests import os import urllib from sqlite3 import OperationalError -from threading import Lock +from threading import Lock, Thread +import Queue import xbmc import xbmcgui @@ -16,7 +17,8 @@ import xbmcvfs import image_cache_thread from utils import window, settings, language as lang, kodiSQL, tryEncode, \ - tryDecode, IfExists + tryDecode, IfExists, ThreadMethods, ThreadMethodsAdditionalSuspend, \ + ThreadMethodsAdditionalStop # Disable annoying requests warnings import requests.packages.urllib3 @@ -119,22 +121,78 @@ def setKodiWebServerDetails(): return (xbmc_port, xbmc_username, xbmc_password) -class Artwork(): - lock = Lock() - - enableTextureCache = settings('enableTextureCache') == "true" +@ThreadMethodsAdditionalSuspend('suspend_LibraryThread') +@ThreadMethodsAdditionalStop('plex_shouldStop') +@ThreadMethods +class Image_Cache_Thread(Thread): imageCacheLimitThreads = int(settings('imageCacheLimit')) imageCacheLimitThreads = imageCacheLimitThreads * 5 log.info("Using Image Cache Thread Count: %s" % imageCacheLimitThreads) + xbmc_port, xbmc_username, xbmc_password = setKodiWebServerDetails() + threads = [] - xbmc_host = 'localhost' - xbmc_port = None - xbmc_username = None - xbmc_password = None + def __init__(self, queue): + self.queue = queue + Thread.__init__(self) + + def run(self): + threadStopped = self.threadStopped + threadSuspended = self.threadSuspended + queue = self.queue + threads = self.threads + imageCacheLimitThreads = self.imageCacheLimitThreads + log.info("---===### Starting Image_Cache_Thread ###===---") + while not threadStopped(): + # In the event the server goes offline + while threadSuspended(): + # Set in service.py + if threadStopped(): + # Abort was requested while waiting. We should exit + log.info("---===### Stopped Image_Cache_Thread ###===---") + return + xbmc.sleep(1000) + try: + url = queue.get(block=False) + except Queue.Empty: + xbmc.sleep(200) + continue + while True: + for thread in threads: + if thread.isAlive() is False: + threads.remove(thread) + if len(threads) < imageCacheLimitThreads: + log.debug('Downloading %s' % url) + thread = Thread(target=self.download, + args=(url,)) + thread.setDaemon(True) + thread.start() + threads.append(thread) + break + log.info("---===### Stopped Image_Cache_Thread ###===---") + + def download(self): + try: + requests.head( + url=("http://%s:%s/image/image://%s" + % (self.xbmc_host, self.xbmc_port, self.url)), + auth=(self.xbmc_username, self.xbmc_password), + timeout=(5, 5)) + # We don't need the result + except Exception as e: + log.error('Image_Cache_Thread exception: %s' % e) + import traceback + log.error("Traceback:\n%s" % traceback.format_exc()) + + +class Artwork(): + lock = Lock() + imageCacheLimitThreads = int(settings('imageCacheLimit')) + imageCacheLimitThreads = imageCacheLimitThreads * 5 + enableTextureCache = settings('enableTextureCache') == "true" if enableTextureCache: - xbmc_port, xbmc_username, xbmc_password = setKodiWebServerDetails() - - imageCacheThreads = [] + queue = Queue.Queue() + download_thread = Image_Cache_Thread(queue) + log.info('Artwork initiated with caching textures = True') def double_urlencode(self, text): text = self.single_urlencode(text) @@ -249,27 +307,6 @@ class Artwork(): pdialog.close() - def addWorkerImageCacheThread(self, url): - while True: - # removed finished - with self.lock: - for thread in self.imageCacheThreads: - if thread.is_finished: - self.imageCacheThreads.remove(thread) - # add a new thread or wait and retry if we hit our limit - with self.lock: - if len(self.imageCacheThreads) < self.imageCacheLimitThreads: - newThread = image_cache_thread.ImageCacheThread() - newThread.set_url(self.double_urlencode(url)) - newThread.set_host(self.xbmc_host, self.xbmc_port) - newThread.set_auth(self.xbmc_username, self.xbmc_password) - newThread.start() - self.imageCacheThreads.append(newThread) - return - log.debug("Waiting for empty queue spot: %s" - % len(self.imageCacheThreads)) - xbmc.sleep(50) - def cacheTexture(self, url): # Cache a single image url to the texture cache if url and self.enableTextureCache: @@ -289,7 +326,7 @@ class Artwork(): except: pass else: - self.addWorkerImageCacheThread(url) + self.queue.put(url) def addArtwork(self, artwork, kodiId, mediaType, cursor): # Kodi conversion table From 6621cc5a8d37f98570f7dcf13523eb56497003bd Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 10:58:08 +0200 Subject: [PATCH 34/86] Revert "Dedicated art downloading queue" This reverts commit dc94cc6bfe8a328d98f35577d641c8c7879fda67. --- resources/lib/artwork.py | 107 +++++++++++++-------------------------- 1 file changed, 35 insertions(+), 72 deletions(-) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index ee1f6132..50add7b9 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -8,8 +8,7 @@ import requests import os import urllib from sqlite3 import OperationalError -from threading import Lock, Thread -import Queue +from threading import Lock import xbmc import xbmcgui @@ -17,8 +16,7 @@ import xbmcvfs import image_cache_thread from utils import window, settings, language as lang, kodiSQL, tryEncode, \ - tryDecode, IfExists, ThreadMethods, ThreadMethodsAdditionalSuspend, \ - ThreadMethodsAdditionalStop + tryDecode, IfExists # Disable annoying requests warnings import requests.packages.urllib3 @@ -121,78 +119,22 @@ def setKodiWebServerDetails(): return (xbmc_port, xbmc_username, xbmc_password) -@ThreadMethodsAdditionalSuspend('suspend_LibraryThread') -@ThreadMethodsAdditionalStop('plex_shouldStop') -@ThreadMethods -class Image_Cache_Thread(Thread): +class Artwork(): + lock = Lock() + + enableTextureCache = settings('enableTextureCache') == "true" imageCacheLimitThreads = int(settings('imageCacheLimit')) imageCacheLimitThreads = imageCacheLimitThreads * 5 log.info("Using Image Cache Thread Count: %s" % imageCacheLimitThreads) - xbmc_port, xbmc_username, xbmc_password = setKodiWebServerDetails() - threads = [] - def __init__(self, queue): - self.queue = queue - Thread.__init__(self) - - def run(self): - threadStopped = self.threadStopped - threadSuspended = self.threadSuspended - queue = self.queue - threads = self.threads - imageCacheLimitThreads = self.imageCacheLimitThreads - log.info("---===### Starting Image_Cache_Thread ###===---") - while not threadStopped(): - # In the event the server goes offline - while threadSuspended(): - # Set in service.py - if threadStopped(): - # Abort was requested while waiting. We should exit - log.info("---===### Stopped Image_Cache_Thread ###===---") - return - xbmc.sleep(1000) - try: - url = queue.get(block=False) - except Queue.Empty: - xbmc.sleep(200) - continue - while True: - for thread in threads: - if thread.isAlive() is False: - threads.remove(thread) - if len(threads) < imageCacheLimitThreads: - log.debug('Downloading %s' % url) - thread = Thread(target=self.download, - args=(url,)) - thread.setDaemon(True) - thread.start() - threads.append(thread) - break - log.info("---===### Stopped Image_Cache_Thread ###===---") - - def download(self): - try: - requests.head( - url=("http://%s:%s/image/image://%s" - % (self.xbmc_host, self.xbmc_port, self.url)), - auth=(self.xbmc_username, self.xbmc_password), - timeout=(5, 5)) - # We don't need the result - except Exception as e: - log.error('Image_Cache_Thread exception: %s' % e) - import traceback - log.error("Traceback:\n%s" % traceback.format_exc()) - - -class Artwork(): - lock = Lock() - imageCacheLimitThreads = int(settings('imageCacheLimit')) - imageCacheLimitThreads = imageCacheLimitThreads * 5 - enableTextureCache = settings('enableTextureCache') == "true" + xbmc_host = 'localhost' + xbmc_port = None + xbmc_username = None + xbmc_password = None if enableTextureCache: - queue = Queue.Queue() - download_thread = Image_Cache_Thread(queue) - log.info('Artwork initiated with caching textures = True') + xbmc_port, xbmc_username, xbmc_password = setKodiWebServerDetails() + + imageCacheThreads = [] def double_urlencode(self, text): text = self.single_urlencode(text) @@ -307,6 +249,27 @@ class Artwork(): pdialog.close() + def addWorkerImageCacheThread(self, url): + while True: + # removed finished + with self.lock: + for thread in self.imageCacheThreads: + if thread.is_finished: + self.imageCacheThreads.remove(thread) + # add a new thread or wait and retry if we hit our limit + with self.lock: + if len(self.imageCacheThreads) < self.imageCacheLimitThreads: + newThread = image_cache_thread.ImageCacheThread() + newThread.set_url(self.double_urlencode(url)) + newThread.set_host(self.xbmc_host, self.xbmc_port) + newThread.set_auth(self.xbmc_username, self.xbmc_password) + newThread.start() + self.imageCacheThreads.append(newThread) + return + log.debug("Waiting for empty queue spot: %s" + % len(self.imageCacheThreads)) + xbmc.sleep(50) + def cacheTexture(self, url): # Cache a single image url to the texture cache if url and self.enableTextureCache: @@ -326,7 +289,7 @@ class Artwork(): except: pass else: - self.queue.put(url) + self.addWorkerImageCacheThread(url) def addArtwork(self, artwork, kodiId, mediaType, cursor): # Kodi conversion table From afb0960260951b3f1a03a81b3d8e0f5b92f3eeeb Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 11:23:31 +0200 Subject: [PATCH 35/86] Optimize image_cache_thread - BUT: lead to DOS-seeming behavior and ConnectionError: ('Connection aborted.', error(10053)) from http://image.tmdb.org/ --- resources/lib/artwork.py | 22 ++++++------ resources/lib/image_cache_thread.py | 54 +++++++++-------------------- 2 files changed, 28 insertions(+), 48 deletions(-) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index 50add7b9..ed31c7d6 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -14,7 +14,7 @@ import xbmc import xbmcgui import xbmcvfs -import image_cache_thread +from image_cache_thread import ImageCacheThread from utils import window, settings, language as lang, kodiSQL, tryEncode, \ tryDecode, IfExists @@ -254,20 +254,22 @@ class Artwork(): # removed finished with self.lock: for thread in self.imageCacheThreads: - if thread.is_finished: + if thread.isAlive() is False: self.imageCacheThreads.remove(thread) # add a new thread or wait and retry if we hit our limit with self.lock: if len(self.imageCacheThreads) < self.imageCacheLimitThreads: - newThread = image_cache_thread.ImageCacheThread() - newThread.set_url(self.double_urlencode(url)) - newThread.set_host(self.xbmc_host, self.xbmc_port) - newThread.set_auth(self.xbmc_username, self.xbmc_password) - newThread.start() - self.imageCacheThreads.append(newThread) + thread = ImageCacheThread( + self.xbmc_username, + self.xbmc_password, + "http://%s:%s/image/image://%s" + % (self.xbmc_host, + self.xbmc_port, + self.double_urlencode(url))) + thread.start() + self.imageCacheThreads.append(thread) return - log.debug("Waiting for empty queue spot: %s" - % len(self.imageCacheThreads)) + log.error('Waiting for queue spot here') xbmc.sleep(50) def cacheTexture(self, url): diff --git a/resources/lib/image_cache_thread.py b/resources/lib/image_cache_thread.py index b01377a4..b259c864 100644 --- a/resources/lib/image_cache_thread.py +++ b/resources/lib/image_cache_thread.py @@ -1,7 +1,6 @@ # -*- coding: utf-8 -*- -################################################################################################# - +############################################################################### import logging import threading import requests @@ -9,51 +8,30 @@ import requests # Disable annoying requests warnings import requests.packages.urllib3 requests.packages.urllib3.disable_warnings() -################################################################################################# +############################################################################### log = logging.getLogger("PLEX."+__name__) -################################################################################################# +############################################################################### class ImageCacheThread(threading.Thread): - - url_to_process = None - is_finished = False - - xbmc_host = "" - xbmc_port = "" - xbmc_username = "" - xbmc_password = "" - - - def __init__(self): - + def __init__(self, xbmc_username, xbmc_password, url): + self.xbmc_username = xbmc_username + self.xbmc_password = xbmc_password + self.url = url threading.Thread.__init__(self) - - def set_url(self, url): - - self.url_to_process = url - - def set_host(self, host, port): - - self.xbmc_host = host - self.xbmc_port = port - - def set_auth(self, username, password): - - self.xbmc_username = username - self.xbmc_password = password - def run(self): try: - response = requests.head( - url=("http://%s:%s/image/image://%s" - % (self.xbmc_host, self.xbmc_port, self.url_to_process)), + requests.head( + url=self.url, auth=(self.xbmc_username, self.xbmc_password), - timeout=(5, 5)) - # We don't need the result - except Exception: + timeout=(0.01, 0.01)) + except requests.Timeout: + # We don't need the result, only trigger Kodi to start download pass - self.is_finished = True + except Exception as e: + log.error('Encountered exception: %s' % e) + import traceback + log.error("Traceback:\n%s" % traceback.format_exc()) From c0d91507c0000c358a683299663a00645d5afeda Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 11:24:06 +0200 Subject: [PATCH 36/86] Revert "Optimize image_cache_thread" This reverts commit afb0960260951b3f1a03a81b3d8e0f5b92f3eeeb. --- resources/lib/artwork.py | 22 ++++++------ resources/lib/image_cache_thread.py | 54 ++++++++++++++++++++--------- 2 files changed, 48 insertions(+), 28 deletions(-) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index ed31c7d6..50add7b9 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -14,7 +14,7 @@ import xbmc import xbmcgui import xbmcvfs -from image_cache_thread import ImageCacheThread +import image_cache_thread from utils import window, settings, language as lang, kodiSQL, tryEncode, \ tryDecode, IfExists @@ -254,22 +254,20 @@ class Artwork(): # removed finished with self.lock: for thread in self.imageCacheThreads: - if thread.isAlive() is False: + if thread.is_finished: self.imageCacheThreads.remove(thread) # add a new thread or wait and retry if we hit our limit with self.lock: if len(self.imageCacheThreads) < self.imageCacheLimitThreads: - thread = ImageCacheThread( - self.xbmc_username, - self.xbmc_password, - "http://%s:%s/image/image://%s" - % (self.xbmc_host, - self.xbmc_port, - self.double_urlencode(url))) - thread.start() - self.imageCacheThreads.append(thread) + newThread = image_cache_thread.ImageCacheThread() + newThread.set_url(self.double_urlencode(url)) + newThread.set_host(self.xbmc_host, self.xbmc_port) + newThread.set_auth(self.xbmc_username, self.xbmc_password) + newThread.start() + self.imageCacheThreads.append(newThread) return - log.error('Waiting for queue spot here') + log.debug("Waiting for empty queue spot: %s" + % len(self.imageCacheThreads)) xbmc.sleep(50) def cacheTexture(self, url): diff --git a/resources/lib/image_cache_thread.py b/resources/lib/image_cache_thread.py index b259c864..b01377a4 100644 --- a/resources/lib/image_cache_thread.py +++ b/resources/lib/image_cache_thread.py @@ -1,6 +1,7 @@ # -*- coding: utf-8 -*- -############################################################################### +################################################################################################# + import logging import threading import requests @@ -8,30 +9,51 @@ import requests # Disable annoying requests warnings import requests.packages.urllib3 requests.packages.urllib3.disable_warnings() -############################################################################### +################################################################################################# log = logging.getLogger("PLEX."+__name__) -############################################################################### +################################################################################################# class ImageCacheThread(threading.Thread): - def __init__(self, xbmc_username, xbmc_password, url): - self.xbmc_username = xbmc_username - self.xbmc_password = xbmc_password - self.url = url + + url_to_process = None + is_finished = False + + xbmc_host = "" + xbmc_port = "" + xbmc_username = "" + xbmc_password = "" + + + def __init__(self): + threading.Thread.__init__(self) + + def set_url(self, url): + + self.url_to_process = url + + def set_host(self, host, port): + + self.xbmc_host = host + self.xbmc_port = port + + def set_auth(self, username, password): + + self.xbmc_username = username + self.xbmc_password = password + def run(self): try: - requests.head( - url=self.url, + response = requests.head( + url=("http://%s:%s/image/image://%s" + % (self.xbmc_host, self.xbmc_port, self.url_to_process)), auth=(self.xbmc_username, self.xbmc_password), - timeout=(0.01, 0.01)) - except requests.Timeout: - # We don't need the result, only trigger Kodi to start download + timeout=(5, 5)) + # We don't need the result + except Exception: pass - except Exception as e: - log.error('Encountered exception: %s' % e) - import traceback - log.error("Traceback:\n%s" % traceback.format_exc()) + self.is_finished = True From cd141272331665637b17e04d6583f74f73e40a22 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 11:24:12 +0200 Subject: [PATCH 37/86] Revert "Revert "Dedicated art downloading queue"" This reverts commit 6621cc5a8d37f98570f7dcf13523eb56497003bd. --- resources/lib/artwork.py | 107 ++++++++++++++++++++++++++------------- 1 file changed, 72 insertions(+), 35 deletions(-) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index 50add7b9..ee1f6132 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -8,7 +8,8 @@ import requests import os import urllib from sqlite3 import OperationalError -from threading import Lock +from threading import Lock, Thread +import Queue import xbmc import xbmcgui @@ -16,7 +17,8 @@ import xbmcvfs import image_cache_thread from utils import window, settings, language as lang, kodiSQL, tryEncode, \ - tryDecode, IfExists + tryDecode, IfExists, ThreadMethods, ThreadMethodsAdditionalSuspend, \ + ThreadMethodsAdditionalStop # Disable annoying requests warnings import requests.packages.urllib3 @@ -119,22 +121,78 @@ def setKodiWebServerDetails(): return (xbmc_port, xbmc_username, xbmc_password) -class Artwork(): - lock = Lock() - - enableTextureCache = settings('enableTextureCache') == "true" +@ThreadMethodsAdditionalSuspend('suspend_LibraryThread') +@ThreadMethodsAdditionalStop('plex_shouldStop') +@ThreadMethods +class Image_Cache_Thread(Thread): imageCacheLimitThreads = int(settings('imageCacheLimit')) imageCacheLimitThreads = imageCacheLimitThreads * 5 log.info("Using Image Cache Thread Count: %s" % imageCacheLimitThreads) + xbmc_port, xbmc_username, xbmc_password = setKodiWebServerDetails() + threads = [] - xbmc_host = 'localhost' - xbmc_port = None - xbmc_username = None - xbmc_password = None + def __init__(self, queue): + self.queue = queue + Thread.__init__(self) + + def run(self): + threadStopped = self.threadStopped + threadSuspended = self.threadSuspended + queue = self.queue + threads = self.threads + imageCacheLimitThreads = self.imageCacheLimitThreads + log.info("---===### Starting Image_Cache_Thread ###===---") + while not threadStopped(): + # In the event the server goes offline + while threadSuspended(): + # Set in service.py + if threadStopped(): + # Abort was requested while waiting. We should exit + log.info("---===### Stopped Image_Cache_Thread ###===---") + return + xbmc.sleep(1000) + try: + url = queue.get(block=False) + except Queue.Empty: + xbmc.sleep(200) + continue + while True: + for thread in threads: + if thread.isAlive() is False: + threads.remove(thread) + if len(threads) < imageCacheLimitThreads: + log.debug('Downloading %s' % url) + thread = Thread(target=self.download, + args=(url,)) + thread.setDaemon(True) + thread.start() + threads.append(thread) + break + log.info("---===### Stopped Image_Cache_Thread ###===---") + + def download(self): + try: + requests.head( + url=("http://%s:%s/image/image://%s" + % (self.xbmc_host, self.xbmc_port, self.url)), + auth=(self.xbmc_username, self.xbmc_password), + timeout=(5, 5)) + # We don't need the result + except Exception as e: + log.error('Image_Cache_Thread exception: %s' % e) + import traceback + log.error("Traceback:\n%s" % traceback.format_exc()) + + +class Artwork(): + lock = Lock() + imageCacheLimitThreads = int(settings('imageCacheLimit')) + imageCacheLimitThreads = imageCacheLimitThreads * 5 + enableTextureCache = settings('enableTextureCache') == "true" if enableTextureCache: - xbmc_port, xbmc_username, xbmc_password = setKodiWebServerDetails() - - imageCacheThreads = [] + queue = Queue.Queue() + download_thread = Image_Cache_Thread(queue) + log.info('Artwork initiated with caching textures = True') def double_urlencode(self, text): text = self.single_urlencode(text) @@ -249,27 +307,6 @@ class Artwork(): pdialog.close() - def addWorkerImageCacheThread(self, url): - while True: - # removed finished - with self.lock: - for thread in self.imageCacheThreads: - if thread.is_finished: - self.imageCacheThreads.remove(thread) - # add a new thread or wait and retry if we hit our limit - with self.lock: - if len(self.imageCacheThreads) < self.imageCacheLimitThreads: - newThread = image_cache_thread.ImageCacheThread() - newThread.set_url(self.double_urlencode(url)) - newThread.set_host(self.xbmc_host, self.xbmc_port) - newThread.set_auth(self.xbmc_username, self.xbmc_password) - newThread.start() - self.imageCacheThreads.append(newThread) - return - log.debug("Waiting for empty queue spot: %s" - % len(self.imageCacheThreads)) - xbmc.sleep(50) - def cacheTexture(self, url): # Cache a single image url to the texture cache if url and self.enableTextureCache: @@ -289,7 +326,7 @@ class Artwork(): except: pass else: - self.addWorkerImageCacheThread(url) + self.queue.put(url) def addArtwork(self, artwork, kodiId, mediaType, cursor): # Kodi conversion table From ca6bb4e8ca4ad499cd52543d74ae5b21383eba9f Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 14:02:50 +0200 Subject: [PATCH 38/86] Rewiring of caching - daemon instead of threads --- resources/lib/artwork.py | 124 ++++++++++++++++----------------------- 1 file changed, 50 insertions(+), 74 deletions(-) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index ee1f6132..cbdc88c2 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -1,13 +1,11 @@ # -*- coding: utf-8 -*- ############################################################################### - import json import logging import requests import os -import urllib -from sqlite3 import OperationalError +from urllib import quote_plus, unquote from threading import Lock, Thread import Queue @@ -15,7 +13,6 @@ import xbmc import xbmcgui import xbmcvfs -import image_cache_thread from utils import window, settings, language as lang, kodiSQL, tryEncode, \ tryDecode, IfExists, ThreadMethods, ThreadMethodsAdditionalSuspend, \ ThreadMethodsAdditionalStop @@ -121,15 +118,20 @@ def setKodiWebServerDetails(): return (xbmc_port, xbmc_username, xbmc_password) +def double_urlencode(text): + return quote_plus(quote_plus(text)) + + +def double_urldecode(text): + return unquote(unquote(text)) + + @ThreadMethodsAdditionalSuspend('suspend_LibraryThread') @ThreadMethodsAdditionalStop('plex_shouldStop') @ThreadMethods class Image_Cache_Thread(Thread): - imageCacheLimitThreads = int(settings('imageCacheLimit')) - imageCacheLimitThreads = imageCacheLimitThreads * 5 - log.info("Using Image Cache Thread Count: %s" % imageCacheLimitThreads) + xbmc_host = 'localhost' xbmc_port, xbmc_username, xbmc_password = setKodiWebServerDetails() - threads = [] def __init__(self, queue): self.queue = queue @@ -139,9 +141,6 @@ class Image_Cache_Thread(Thread): threadStopped = self.threadStopped threadSuspended = self.threadSuspended queue = self.queue - threads = self.threads - imageCacheLimitThreads = self.imageCacheLimitThreads - log.info("---===### Starting Image_Cache_Thread ###===---") while not threadStopped(): # In the event the server goes offline while threadSuspended(): @@ -154,57 +153,55 @@ class Image_Cache_Thread(Thread): try: url = queue.get(block=False) except Queue.Empty: - xbmc.sleep(200) + xbmc.sleep(1000) continue + sleep = 0 while True: - for thread in threads: - if thread.isAlive() is False: - threads.remove(thread) - if len(threads) < imageCacheLimitThreads: - log.debug('Downloading %s' % url) - thread = Thread(target=self.download, - args=(url,)) - thread.setDaemon(True) - thread.start() - threads.append(thread) + try: + requests.head( + url="http://%s:%s/image/image://%s" + % (self.xbmc_host, self.xbmc_port, url), + auth=(self.xbmc_username, self.xbmc_password), + timeout=(0.01, 0.01)) + except requests.Timeout: + # We don't need the result, only trigger Kodi to start the + # download. All is well break + except requests.ConnectionError: + # Server thinks its a DOS attack, ('error 10053') + # Wait before trying again + if sleep > 5: + log.error('Repeatedly got ConnectionError for url %s' + % double_urldecode(url)) + break + log.debug('Were trying too hard to download art, server ' + 'over-loaded. Sleep %s seconds before trying ' + 'again to download %s' + % (2**sleep, double_urldecode(url))) + xbmc.sleep((2**sleep)*1000) + sleep += 1 + continue + except Exception as e: + log.error('Unknown exception for url %s: %s' + % (double_urldecode(url), e)) + import traceback + log.error("Traceback:\n%s" % traceback.format_exc()) + break + # We did not even get a timeout + break + queue.task_done() + # Sleep for a bit to reduce CPU strain + xbmc.sleep(20) log.info("---===### Stopped Image_Cache_Thread ###===---") - def download(self): - try: - requests.head( - url=("http://%s:%s/image/image://%s" - % (self.xbmc_host, self.xbmc_port, self.url)), - auth=(self.xbmc_username, self.xbmc_password), - timeout=(5, 5)) - # We don't need the result - except Exception as e: - log.error('Image_Cache_Thread exception: %s' % e) - import traceback - log.error("Traceback:\n%s" % traceback.format_exc()) - class Artwork(): lock = Lock() - imageCacheLimitThreads = int(settings('imageCacheLimit')) - imageCacheLimitThreads = imageCacheLimitThreads * 5 enableTextureCache = settings('enableTextureCache') == "true" if enableTextureCache: queue = Queue.Queue() download_thread = Image_Cache_Thread(queue) - log.info('Artwork initiated with caching textures = True') - - def double_urlencode(self, text): - text = self.single_urlencode(text) - text = self.single_urlencode(text) - return text - - def single_urlencode(self, text): - # urlencode needs a utf- string - text = urllib.urlencode({'blahblahblah': tryEncode(text)}) - text = text[13:] - # return the result again as unicode - return tryDecode(text) + download_thread.start() def fullTextureCacheSync(self): """ @@ -310,23 +307,7 @@ class Artwork(): def cacheTexture(self, url): # Cache a single image url to the texture cache if url and self.enableTextureCache: - log.debug("Processing: %s" % url) - if not self.imageCacheLimitThreads: - # Add image to texture cache by simply calling it at the http - # endpoint - url = self.double_urlencode(url) - try: - # Extreme short timeouts so we will have a exception. - requests.head( - url=("http://%s:%s/image/image://%s" - % (self.xbmc_host, self.xbmc_port, url)), - auth=(self.xbmc_username, self.xbmc_password), - timeout=(0.01, 0.01)) - # We don't need the result - except: - pass - else: - self.queue.put(url) + self.queue.put(double_urlencode(url)) def addArtwork(self, artwork, kodiId, mediaType, cursor): # Kodi conversion table @@ -480,8 +461,6 @@ class Artwork(): cachedurl = cursor.fetchone()[0] except TypeError: log.info("Could not find cached url.") - except OperationalError: - log.warn("Database is locked. Skip deletion process.") else: # Delete thumbnail as well as the entry thumbnails = tryDecode( @@ -492,10 +471,7 @@ class Artwork(): except Exception as e: log.error('Could not delete cached artwork %s. Error: %s' % (thumbnails, e)) - try: - cursor.execute("DELETE FROM texture WHERE url = ?", (url,)) - connection.commit() - except OperationalError: - log.error("OperationalError deleting url from cache.") + cursor.execute("DELETE FROM texture WHERE url = ?", (url,)) + connection.commit() finally: connection.close() From 703f2fb37d7eeb0a3e8afabded5dd6c587938ff8 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 14:48:40 +0200 Subject: [PATCH 39/86] Fix setting to cache all textures to Kodi --- default.py | 3 +- resources/language/English/strings.xml | 2 +- resources/language/German/strings.xml | 2 +- resources/lib/artwork.py | 42 +++----------------------- resources/lib/librarysync.py | 6 ++++ 5 files changed, 13 insertions(+), 42 deletions(-) diff --git a/default.py b/default.py index 8cfb9ae9..f2edfc06 100644 --- a/default.py +++ b/default.py @@ -172,8 +172,7 @@ class Main(): utils.window('plex_runLibScan', value="full") elif mode == "texturecache": - import artwork - artwork.Artwork().fullTextureCacheSync() + utils.window('plex_runLibScan', value='del_textures') else: entrypoint.doMainListing() diff --git a/resources/language/English/strings.xml b/resources/language/English/strings.xml index db6fb9dc..9db2ab39 100644 --- a/resources/language/English/strings.xml +++ b/resources/language/English/strings.xml @@ -453,7 +453,7 @@ - Running the image cache process can take some time. Are you sure you want continue? + Running the image cache process can take some time. It will happen in the background. Are you sure you want continue? Reset all existing cache data first? diff --git a/resources/language/German/strings.xml b/resources/language/German/strings.xml index 16176b81..16337583 100644 --- a/resources/language/German/strings.xml +++ b/resources/language/German/strings.xml @@ -391,7 +391,7 @@ - Alle Plex Bilder in Kodi zwischenzuspeichern kann sehr lange dauern. Möchten Sie wirklich fortfahren? + Alle Plex Bilder in Kodi zwischenzuspeichern kann lange dauern. Es wird im Hintergrund stattfinden. Möchten Sie wirklich fortfahren? Sollen erst alle bestehenden Bilder im Cache gelöscht werden? diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index cbdc88c2..9021c58e 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -191,7 +191,7 @@ class Image_Cache_Thread(Thread): break queue.task_done() # Sleep for a bit to reduce CPU strain - xbmc.sleep(20) + xbmc.sleep(50) log.info("---===### Stopped Image_Cache_Thread ###===---") @@ -214,9 +214,6 @@ class Artwork(): log.info("Doing Image Cache Sync") - pdialog = xbmcgui.DialogProgress() - pdialog.create("PlexKodiConnect", "Image Cache Sync") - # ask to rest all existing or not if xbmcgui.Dialog().yesno( "Image Texture Cache", lang(39251), ""): @@ -256,53 +253,22 @@ class Artwork(): cursor.execute("SELECT url FROM art WHERE media_type != 'actor'") result = cursor.fetchall() total = len(result) - log.info("Image cache sync about to process %s images" % total) + log.info("Image cache sync about to process %s video images" % total) connection.close() - count = 0 for url in result: - if pdialog.iscanceled(): - break - - percentage = int((float(count) / float(total))*100) - message = "%s of %s (%s)" % (count, total, self.imageCacheThreads) - pdialog.update(percentage, "%s %s" % (lang(33045), message)) self.cacheTexture(url[0]) - count += 1 # Cache all entries in music DB connection = kodiSQL('music') cursor = connection.cursor() cursor.execute("SELECT url FROM art") result = cursor.fetchall() total = len(result) - log.info("Image cache sync about to process %s images" % total) + log.info("Image cache sync about to process %s music images" % total) connection.close() - - count = 0 for url in result: - if pdialog.iscanceled(): - break - - percentage = int((float(count) / float(total))*100) - message = "%s of %s" % (count, total) - pdialog.update(percentage, "%s %s" % (lang(33045), message)) self.cacheTexture(url[0]) - count += 1 - pdialog.update(100, "%s %s" - % (lang(33046), len(self.imageCacheThreads))) - log.info("Waiting for all threads to exit") - while len(self.imageCacheThreads): - with self.lock: - for thread in self.imageCacheThreads: - if thread.is_finished: - self.imageCacheThreads.remove(thread) - pdialog.update(100, "%s %s" - % (lang(33046), len(self.imageCacheThreads))) - log.info("Waiting for all threads to exit: %s" - % len(self.imageCacheThreads)) - xbmc.sleep(500) - - pdialog.close() + log.info('Done throwing URLs to art download daemon') def cacheTexture(self, url): # Cache a single image url to the texture cache diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 868fcd4a..a3f25bdc 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -1803,6 +1803,12 @@ class LibrarySync(Thread): line1=lang(39223), nolabel=lang(39224), yeslabel=lang(39225))) + elif window('plex_runLibScan') == 'del_textures': + window('plex_runLibScan', clear=True) + window('plex_dbScan', value="true") + import artwork + artwork.Artwork().fullTextureCacheSync() + window('plex_dbScan', clear=True) else: now = getUnixTimestamp() if (now - lastSync > fullSyncInterval and From 77d1df553cf7aa9dc15e1e87e113675101e8027d Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 14:49:57 +0200 Subject: [PATCH 40/86] More logging --- resources/lib/artwork.py | 1 + 1 file changed, 1 insertion(+) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index 9021c58e..a0652f9e 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -190,6 +190,7 @@ class Image_Cache_Thread(Thread): # We did not even get a timeout break queue.task_done() + log.debug('Downloaded art: %s' % double_urldecode(url)) # Sleep for a bit to reduce CPU strain xbmc.sleep(50) log.info("---===### Stopped Image_Cache_Thread ###===---") From 25a6d820223ec5a15ccf37f3c894da42c16ce120 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 15:17:01 +0200 Subject: [PATCH 41/86] Prevent infinite loops after requesting something --- default.py | 11 +++++------ resources/lib/artwork.py | 1 - 2 files changed, 5 insertions(+), 7 deletions(-) diff --git a/default.py b/default.py index f2edfc06..ff769506 100644 --- a/default.py +++ b/default.py @@ -10,6 +10,7 @@ import urlparse import xbmc import xbmcaddon import xbmcgui +import xbmcplugin _addon = xbmcaddon.Addon(id='plugin.video.plexkodiconnect') @@ -102,7 +103,6 @@ class Main(): if mode == 'fanart': log.info('User requested fanarttv refresh') utils.window('plex_runLibScan', value='fanart') - return # Called by e.g. 3rd party plugin video extras if ("/Extras" in sys.argv[0] or "/VideoFiles" in sys.argv[0] or @@ -161,21 +161,20 @@ class Main(): "Unable to run the sync, the add-on is not connected " "to a Plex server.") log.error("Not connected to a PMS.") - return - else: if mode == 'repair': utils.window('plex_runLibScan', value="repair") - log.warn("Requesting repair lib sync") + log.info("Requesting repair lib sync") elif mode == 'manualsync': - log.warn("Requesting full library scan") + log.info("Requesting full library scan") utils.window('plex_runLibScan', value="full") elif mode == "texturecache": utils.window('plex_runLibScan', value='del_textures') - else: entrypoint.doMainListing() + # Prevent Kodi from hanging in an infinite loop waiting for more + xbmcplugin.endOfDirectory(int(sys.argv[1])) if __name__ == "__main__": log.info('plugin.video.plexkodiconnect started') diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index a0652f9e..37cba78d 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -269,7 +269,6 @@ class Artwork(): connection.close() for url in result: self.cacheTexture(url[0]) - log.info('Done throwing URLs to art download daemon') def cacheTexture(self, url): # Cache a single image url to the texture cache From 1ff7970e8894028f96b110a1ca64e6a01dff9ba9 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 15:20:43 +0200 Subject: [PATCH 42/86] Remove imageCacheLimit and cap on thread-number --- resources/lib/PlexAPI.py | 4 ++-- resources/lib/initialsetup.py | 3 --- resources/settings.xml | 1 - 3 files changed, 2 insertions(+), 6 deletions(-) diff --git a/resources/lib/PlexAPI.py b/resources/lib/PlexAPI.py index 146c787d..904066e3 100644 --- a/resources/lib/PlexAPI.py +++ b/resources/lib/PlexAPI.py @@ -620,10 +620,10 @@ class PlexAPI(): args=(PMS, queue)) threadQueue.append(t) - maxThreads = int(settings('imageCacheLimit')) + maxThreads = 5 threads = [] # poke PMS, own thread for each PMS - while(True): + while True: # Remove finished threads for t in threads: if not t.isAlive(): diff --git a/resources/lib/initialsetup.py b/resources/lib/initialsetup.py index 3e5a6ff3..0ceab110 100644 --- a/resources/lib/initialsetup.py +++ b/resources/lib/initialsetup.py @@ -466,9 +466,6 @@ class InitialSetup(): # Is your Kodi installed on a low-powered device like a Raspberry Pi? # If yes, then we will reduce the strain on Kodi to prevent it from # crashing. - if dialog.yesno(heading=addonName, line1=lang(39072)): - log.debug('User thinks that PKC runs on a raspi or similar') - settings('imageCacheLimit', value='1') # Make sure that we only ask these questions upon first installation settings('InstallQuestionsAnswered', value='true') diff --git a/resources/settings.xml b/resources/settings.xml index 6e5624be..ab0990fb 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -39,7 +39,6 @@ - From ca3a06affc07066b6be5f07a77385f42c2c408d7 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 15:39:20 +0200 Subject: [PATCH 43/86] New setting: sleep between art downloads --- resources/language/English/strings.xml | 1 + resources/language/German/strings.xml | 1 + resources/lib/artwork.py | 4 +++- resources/settings.xml | 1 + 4 files changed, 6 insertions(+), 1 deletion(-) diff --git a/resources/language/English/strings.xml b/resources/language/English/strings.xml index 9db2ab39..7d8a774a 100644 --- a/resources/language/English/strings.xml +++ b/resources/language/English/strings.xml @@ -306,6 +306,7 @@ Download movie set/collection art from FanArtTV Don't ask to pick a certain stream/quality Always pick best quality for trailers + Sleep between art downloads [ms] diff --git a/resources/language/German/strings.xml b/resources/language/German/strings.xml index 16337583..3d89f34d 100644 --- a/resources/language/German/strings.xml +++ b/resources/language/German/strings.xml @@ -32,6 +32,7 @@ FanArtTV Film-Sets/Collections Bilder herunterladen Nicht fragen, welcher Stream/Qualität gespielt wird Trailer immer in der besten Qualität abspielen + Wartezeit zwischen Bilder downloads [ms] Verbindung Netzwerk diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index 37cba78d..b226e82c 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -132,6 +132,7 @@ def double_urldecode(text): class Image_Cache_Thread(Thread): xbmc_host = 'localhost' xbmc_port, xbmc_username, xbmc_password = setKodiWebServerDetails() + sleep_between = int(settings('sleep_between_art_downloads')) def __init__(self, queue): self.queue = queue @@ -141,6 +142,7 @@ class Image_Cache_Thread(Thread): threadStopped = self.threadStopped threadSuspended = self.threadSuspended queue = self.queue + sleep_between = self.sleep_between while not threadStopped(): # In the event the server goes offline while threadSuspended(): @@ -192,7 +194,7 @@ class Image_Cache_Thread(Thread): queue.task_done() log.debug('Downloaded art: %s' % double_urldecode(url)) # Sleep for a bit to reduce CPU strain - xbmc.sleep(50) + xbmc.sleep(sleep_between) log.info("---===### Stopped Image_Cache_Thread ###===---") diff --git a/resources/settings.xml b/resources/settings.xml index ab0990fb..101ec9c4 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -39,6 +39,7 @@ + From a4ed5c47abd875484ee8fa2223c7091a3d115d1a Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 16:11:00 +0200 Subject: [PATCH 44/86] Reorder settings --- resources/language/English/strings.xml | 12 ++++---- resources/language/German/strings.xml | 11 ++++---- resources/lib/entrypoint.py | 2 -- resources/settings.xml | 38 ++++++++++++-------------- 4 files changed, 30 insertions(+), 33 deletions(-) diff --git a/resources/language/English/strings.xml b/resources/language/English/strings.xml index 7d8a774a..a55487aa 100644 --- a/resources/language/English/strings.xml +++ b/resources/language/English/strings.xml @@ -302,12 +302,12 @@ Users must log in every time Kodi restarts RESTART KODI IF YOU MAKE ANY CHANGES Complete Re-Sync necessary - Download additional art from FanArtTV (slower!) + Download additional art from FanArtTV Download movie set/collection art from FanArtTV Don't ask to pick a certain stream/quality Always pick best quality for trailers Sleep between art downloads [ms] - + Artwork Welcome @@ -354,7 +354,7 @@ Plex Companion Port (change only if needed) Activate Plex Companion debug log Activate Plex Companion GDM debug log - Allows flinging media to Kodi through Plex + Plex Companion: Allows flinging media to Kodi through Plex Could not login to plex.tv. Please try signing in again. Problems connecting to plex.tv. Network or internet issue? Could not find any Plex server in the network. Aborting... @@ -367,7 +367,7 @@ [COLOR yellow]Repair local database (force update all content)[/COLOR] [COLOR red]Partial or full reset of Database and PKC[/COLOR] - [COLOR yellow]Cache all images to Kodi texture cache[/COLOR] + [COLOR yellow]Cache all images to Kodi texture cache now[/COLOR] [COLOR yellow]Sync Emby Theme Media to Kodi[/COLOR] local Failed to authenticate. Did you login to plex.tv? @@ -409,7 +409,7 @@ Extend Plex TV Series "On Deck" view to all shows Recently Added: Append show title to episode Recently Added: Append season- and episode-number SxxExx - Would you like to download additional artwork from FanArtTV? Sync will be slower! + Would you like to download additional artwork from FanArtTV in the background? Sync when screensaver is deactivated Force Transcode Hi10P Recently Added: Also show already watched episodes @@ -447,7 +447,7 @@ Abort (Yes) or save address anyway (No)? connected plex.tv toggle successful - Look for missing fanart on FanartTV + [COLOR yellow]Look for missing fanart on FanartTV now[/COLOR] Only look for missing fanart or refresh all fanart? The scan will take quite a while and happen in the background. Refresh all Missing only diff --git a/resources/language/German/strings.xml b/resources/language/German/strings.xml index 3d89f34d..cef530e7 100644 --- a/resources/language/German/strings.xml +++ b/resources/language/German/strings.xml @@ -28,11 +28,12 @@ Benutzer müssen sich bei jedem Neustart von Kodi neu anmelden BEI ÄNDERUNGEN KODI NEU STARTEN Komplette Neusynchronisierung nötig - Zusätzliche Bilder von FanArtTV herunterladen (langsamer!) + Zusätzliche Bilder von FanArtTV herunterladen FanArtTV Film-Sets/Collections Bilder herunterladen Nicht fragen, welcher Stream/Qualität gespielt wird Trailer immer in der besten Qualität abspielen Wartezeit zwischen Bilder downloads [ms] + Artwork Verbindung Netzwerk @@ -292,7 +293,7 @@ Plex Companion Port (nur bei Bedarf ändern) Plex Companion debug log aktivieren Plex Companion GDM debug log aktivieren - Spiele Inhalt von anderen Plex Geräten ab + Plex Companion: Spiele Inhalt von anderen Plex Geräten ab Login bei plex.tv fehlgeschlagen. Bitte erneut versuchen. Netzwerk Verbindungsprobleme für plex.tv. Funktionieren Netzwerk und/oder Internet? Konnte keine Plex Server im Netzwerk finden. Abbruch... @@ -305,7 +306,7 @@ [COLOR yellow]Lokale Datenbank reparieren (allen Inhalt aktualisieren)[/COLOR] [COLOR red]Datenbank und auf Wunsch PKC zurücksetzen[/COLOR] - [COLOR yellow]Alle Plex Bilder in Kodi zwischenspeichern[/COLOR] + [COLOR yellow]Alle Plex Bilder jetzt in Kodi zwischenspeichern[/COLOR] [COLOR yellow]Plex Themes zu Kodi synchronisieren[/COLOR] lokal Plex Media Server Authentifizierung fehlgeschlagen. Haben Sie sich bei plex.tv eingeloggt? @@ -347,7 +348,7 @@ Standard Plex Ansicht "Aktuell" auf alle TV Shows erweitern "Zuletzt hinzugefügt": Serien- an Episoden-Titel anfügen "Zuletzt hinzugefügt": Staffel und Episode anfügen, SxxExx - Zusätzliche Bilder von FanArtTV herunterladen? Die Synchronisierung wird länger dauern! + Zusätzliche Bilder von FanArtTV im Hintergrund herunterladen? Sync wenn Bildschirmschoner deaktiviert wird Hi10p Codec Transkodierung erzwingen "Zuletzt hinzugefügt": gesehene Folgen anzeigen @@ -385,7 +386,7 @@ Abbrechen (Ja) oder PMS Adresse trotzdem speichern (Nein)? verbunden plex.tv wechsel OK - Nach zusätzlicher Fanart auf FanartTV suchen + [COLOR yellow]Jetzt zusätzliche Bilder auf FanartTV suchen[/COLOR] Nur nach fehlender Fanart suchen oder alle Fanart neu herunterladen? Die Suche wird lange dauern und komplett im Hintergrund stattfinden! Alle Fehlend diff --git a/resources/lib/entrypoint.py b/resources/lib/entrypoint.py index d2a5ebd7..59997340 100644 --- a/resources/lib/entrypoint.py +++ b/resources/lib/entrypoint.py @@ -275,8 +275,6 @@ def doMainListing(): # addDirectoryItem("Add user to session", "plugin://plugin.video.plexkodiconnect/?mode=adduser") addDirectoryItem(lang(39203), "plugin://plugin.video.plexkodiconnect/?mode=refreshplaylist") addDirectoryItem(lang(39204), "plugin://plugin.video.plexkodiconnect/?mode=manualsync") - if settings('FanartTV') == 'true': - addDirectoryItem(lang(39222), "plugin://plugin.video.plexkodiconnect/?mode=fanart") xbmcplugin.endOfDirectory(int(sys.argv[1])) diff --git a/resources/settings.xml b/resources/settings.xml index 101ec9c4..e7066114 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -19,17 +19,25 @@ - + + + - - + + + + + + + + @@ -38,10 +46,6 @@ - - - - @@ -49,8 +53,6 @@ - - @@ -104,6 +106,13 @@ + + + + + + + - - - - - - - - - - @@ -144,7 +143,6 @@ - From ee4f1517d51d49317ec39b12e5601fd6f299b933 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 16:19:29 +0200 Subject: [PATCH 45/86] Version bump --- addon.xml | 2 +- changelog.txt | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/addon.xml b/addon.xml index 06023b34..3ac741df 100644 --- a/addon.xml +++ b/addon.xml @@ -1,7 +1,7 @@ diff --git a/changelog.txt b/changelog.txt index 618ec8d5..3365db03 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,11 @@ +version 1.3.4 (beta only) +- Speed up sync - download all art in the background. This should especially speed up your initial sync. Remember to let Kodi sit for a while to let it download the artwork +- New setting to look for missing artwork (the non-Plex stuff ;-)) +- Rearrange the PKC settings +- Fix caching not working +- Use file settings instead of window settings, should fix some errors on changing the PKC settings +- Other small fixes + version 1.3.3 - 1.3.1 and 1.3.2 for everyone - Fix direct play & transcoding subtitles, finally! From 9c74fd4a9899f54742565f823cdf89d89823efe4 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 19:12:32 +0200 Subject: [PATCH 46/86] Fix backgroundsync KeyError --- resources/lib/librarysync.py | 26 +++++++++++++------------- 1 file changed, 13 insertions(+), 13 deletions(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index a3f25bdc..2f3e095a 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -1399,19 +1399,19 @@ class LibrarySync(Thread): if item['state'] == 9: successful = self.process_deleteditems(item) else: - successful = self.process_newitems(item) + successful, item = self.process_newitems(item) + if successful and settings('FanartTV') == 'true': + if item['mediatype'] in ('movie', 'show'): + mediaType = {'movie': 'Movie'}[item['mediatype']] + cls = {'movie': 'Movies'}[item['mediatype']] + self.fanartqueue.put({ + 'itemId': item['ratingKey'], + 'class': cls, + 'mediaType': mediaType, + 'refresh': False + }) if successful is True: deleteListe.append(i) - if (settings('FanartTV') == 'true' and - item['mediatype'] in ('movie')): - mediaType = {'movie': 'Movie'}[item['mediatype']] - cls = {'movie': 'Movies'}[item['mediatype']] - self.fanartqueue.put({ - 'itemId': item['ratingKey'], - 'class': cls, - 'mediaType': mediaType, - 'refresh': False - }) else: # Safety net if we can't process an item item['attempt'] += 1 @@ -1437,7 +1437,7 @@ class LibrarySync(Thread): xml = PF.GetPlexMetadata(ratingKey) if xml in (None, 401): log.error('Could not download data for %s, skipping' % ratingKey) - return False + return False, item log.debug("Processing new/updated PMS item: %s" % ratingKey) viewtag = xml.attrib.get('librarySectionTitle') viewid = xml.attrib.get('librarySectionID') @@ -1462,7 +1462,7 @@ class LibrarySync(Thread): music.add_updateSong(xml[0], viewtag=viewtag, viewid=viewid) - return True + return True, item def process_deleteditems(self, item): if item.get('type') == 1: From 2ca4ad7b6cd341f76ed96c0344957001c6fb432f Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 20:30:28 +0200 Subject: [PATCH 47/86] Fix capitalization --- resources/lib/player.py | 2 +- resources/lib/plexbmchelper/subscribers.py | 4 ++-- 2 files changed, 3 insertions(+), 3 deletions(-) diff --git a/resources/lib/player.py b/resources/lib/player.py index f18829d3..718caec6 100644 --- a/resources/lib/player.py +++ b/resources/lib/player.py @@ -305,7 +305,7 @@ class Player(xbmc.Player): self.stopAll() - window('Plex_currently_playing_itemid', clear=True) + window('plex_currently_playing_itemid', clear=True) window('plex_customplaylist', clear=True) window('plex_customplaylist.seektime', clear=True) window('plex_customplaylist.seektime', clear=True) diff --git a/resources/lib/plexbmchelper/subscribers.py b/resources/lib/plexbmchelper/subscribers.py index 66fc8eb1..bf6941a9 100644 --- a/resources/lib/plexbmchelper/subscribers.py +++ b/resources/lib/plexbmchelper/subscribers.py @@ -102,7 +102,7 @@ class SubscriptionManager: while not keyid: if count > 300: break - keyid = window('Plex_currently_playing_itemid') + keyid = window('plex_currently_playing_itemid') xbmc.sleep(100) count += 1 if keyid: @@ -149,7 +149,7 @@ class SubscriptionManager: self.cleanup() # Don't tell anyone if we don't know a Plex ID and are still playing # (e.g. no stop called). Used for e.g. PVR/TV without PKC usage - if (not window('Plex_currently_playing_itemid') + if (not window('plex_currently_playing_itemid') and not self.lastplayers): return True players = self.js.getPlayers() From 7e2a11ea28ddba1e447ac7722961103c8e9d9c5a Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 20:36:53 +0200 Subject: [PATCH 48/86] Don't double-update playstate of a playing item --- resources/lib/librarysync.py | 3 +++ 1 file changed, 3 insertions(+) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 2f3e095a..25a57776 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -1546,6 +1546,9 @@ class LibrarySync(Thread): continue currSess = self.sessionKeys[sessionKey] + if window('plex_currently_playing_itemid') == ratingKey: + # Don't update what we already know + continue if settings('plex_serverowned') != 'false': # Identify the user - same one as signed on with PKC? Skip # update if neither session's username nor userid match From dc127fd0bf8e146ac96af63664172c3fdbfdf3d3 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 20:38:55 +0200 Subject: [PATCH 49/86] Version bump --- addon.xml | 2 +- changelog.txt | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/addon.xml b/addon.xml index 3ac741df..66594ef1 100644 --- a/addon.xml +++ b/addon.xml @@ -1,7 +1,7 @@ diff --git a/changelog.txt b/changelog.txt index 3365db03..4f2eb222 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,7 @@ +version 1.3.5 (beta only) +- Fix backgroundsync KeyError +- Don't double-update playstate of a playing item + version 1.3.4 (beta only) - Speed up sync - download all art in the background. This should especially speed up your initial sync. Remember to let Kodi sit for a while to let it download the artwork - New setting to look for missing artwork (the non-Plex stuff ;-)) From b23bdfe65f8cd33e1650495ec47e98441e44b166 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 20:48:37 +0200 Subject: [PATCH 50/86] Fix capitalization --- resources/lib/kodimonitor.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/kodimonitor.py b/resources/lib/kodimonitor.py index 27733b13..5cac1315 100644 --- a/resources/lib/kodimonitor.py +++ b/resources/lib/kodimonitor.py @@ -254,7 +254,7 @@ class KodiMonitor(xbmc.Monitor): # Save currentFile for cleanup later and to be able to access refs window('plex_lastPlayedFiled', value=currentFile) - window('Plex_currently_playing_itemid', value=plexid) + window('plex_currently_playing_itemid', value=plexid) window("emby_%s.itemid" % tryEncode(currentFile), value=plexid) log.info('Finish playback startup') From 703b6994705315ce10089f93b56e1f8f2a4d4efc Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sat, 17 Sep 2016 20:49:22 +0200 Subject: [PATCH 51/86] Version bump --- addon.xml | 2 +- changelog.txt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/addon.xml b/addon.xml index 66594ef1..8ce8f7cf 100644 --- a/addon.xml +++ b/addon.xml @@ -1,7 +1,7 @@ diff --git a/changelog.txt b/changelog.txt index 4f2eb222..d6a9e21b 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,6 @@ +version 1.3.6 (beta only) +- Fix capitalization + version 1.3.5 (beta only) - Fix backgroundsync KeyError - Don't double-update playstate of a playing item From b5b3b8380fa80727e028999babb20200c202b854 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Wed, 21 Sep 2016 19:52:11 +0200 Subject: [PATCH 52/86] Music sync: Fix ProgrammingError --- resources/lib/itemtypes.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/itemtypes.py b/resources/lib/itemtypes.py index 170ddf5f..7b91f9d3 100644 --- a/resources/lib/itemtypes.py +++ b/resources/lib/itemtypes.py @@ -1749,7 +1749,7 @@ class Music(Items): VALUES (?, ?, ?, ?) ''' ) - kodicursor.execute(query, (artistid, songid, index, artist_name)) + kodicursor.execute(query, (artistid, songid, index, artist_name)) # Verify if album artist exists album_artists = [] From e4d736a6706823972ccbc82ed3e153c50b7c3860 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Wed, 21 Sep 2016 20:17:33 +0200 Subject: [PATCH 53/86] Don't set-up clips/trailers like other videos - Should fix PKC trying to tell the PMS where we are playing that item --- resources/lib/playbackutils.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/resources/lib/playbackutils.py b/resources/lib/playbackutils.py index 65a8f5d1..a8396195 100644 --- a/resources/lib/playbackutils.py +++ b/resources/lib/playbackutils.py @@ -259,8 +259,11 @@ class PlaybackUtils(): def setProperties(self, playurl, listitem): # Set all properties necessary for plugin path playback - itemid = self.API.getRatingKey() itemtype = self.API.getType() + if itemtype == 'clip': + log.debug('Setting up a clip/trailer, skip window variables') + return + itemid = self.API.getRatingKey() userdata = self.API.getUserData() embyitem = "emby_%s" % playurl From c2b5434a9b52f8e0afd1228621a3fc4d9b51efbf Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Wed, 21 Sep 2016 20:18:39 +0200 Subject: [PATCH 54/86] Version bump --- addon.xml | 2 +- changelog.txt | 4 ++++ 2 files changed, 5 insertions(+), 1 deletion(-) diff --git a/addon.xml b/addon.xml index 8ce8f7cf..f4193c60 100644 --- a/addon.xml +++ b/addon.xml @@ -1,7 +1,7 @@ diff --git a/changelog.txt b/changelog.txt index d6a9e21b..7e5a8ed1 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,7 @@ +version 1.3.7 (beta only) +- Music sync: Fix ProgrammingError +- Don't set-up clips/trailers like other videos (Should fix PKC trying to tell the PMS where we are playing that item) + version 1.3.6 (beta only) - Fix capitalization From c437821e3c915dac64e53728fcd6f75cc119b336 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 25 Sep 2016 17:35:40 +0200 Subject: [PATCH 55/86] Fix TypeError for manually entering PMS address - Fixes #114 --- resources/lib/entrypoint.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/entrypoint.py b/resources/lib/entrypoint.py index 59997340..a66744db 100644 --- a/resources/lib/entrypoint.py +++ b/resources/lib/entrypoint.py @@ -1362,7 +1362,7 @@ def enterPMS(): settings('plex_machineIdentifier', '') else: settings('plex_machineIdentifier', machineIdentifier) - log.info('Setting new PMS to https %s, ip %s, port %s, machineIdentifier ' + log.info('Set new PMS to https %s, ip %s, port %s, machineIdentifier %s' % (https, ip, port, machineIdentifier)) settings('https', value=https) settings('ipaddress', value=ip) From fe99fd1ec85f835b3832ed25249e9fcf4f18ff32 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 25 Sep 2016 18:34:03 +0200 Subject: [PATCH 56/86] Remove obsolete import --- resources/lib/playbackutils.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/resources/lib/playbackutils.py b/resources/lib/playbackutils.py index a8396195..0680fffb 100644 --- a/resources/lib/playbackutils.py +++ b/resources/lib/playbackutils.py @@ -11,7 +11,6 @@ import xbmc import xbmcgui import xbmcplugin -import artwork import playutils as putils import playlist from utils import window, settings, tryEncode, tryDecode @@ -39,7 +38,6 @@ class PlaybackUtils(): self.userid = window('currUserId') self.server = window('pms_server') - self.artwork = artwork.Artwork() if self.API.getType() == 'track': self.pl = playlist.Playlist(typus='music') else: From 086d5c89011ad49f447367ab4094052e5456e75f Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 25 Sep 2016 18:46:10 +0200 Subject: [PATCH 57/86] Fix "Opening Stream..." dialog not closing - Fixes #113 - See http://forum.kodi.tv/showthread.php?tid=155641 --- resources/lib/playbackutils.py | 1 + 1 file changed, 1 insertion(+) diff --git a/resources/lib/playbackutils.py b/resources/lib/playbackutils.py index 0680fffb..51d9b63b 100644 --- a/resources/lib/playbackutils.py +++ b/resources/lib/playbackutils.py @@ -206,6 +206,7 @@ class PlaybackUtils(): (homeScreen and not sizePlaylist)): # Playlist was created just now, play it. log.info("Play playlist.") + xbmcplugin.endOfDirectory(int(sys.argv[1]), True, False, False) xbmc.Player().play(kodiPl, startpos=startPos) else: From a7820a9cf65c524895ec26eb5c5d459c3f3f410e Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 25 Sep 2016 18:59:34 +0200 Subject: [PATCH 58/86] Try to prevent OperationalError: database is locked --- resources/lib/utils.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/utils.py b/resources/lib/utils.py index 0a28abb7..ceae438a 100644 --- a/resources/lib/utils.py +++ b/resources/lib/utils.py @@ -182,7 +182,7 @@ def kodiSQL(media_type="video"): else: dbPath = getKodiVideoDBPath() - connection = sqlite3.connect(dbPath) + connection = sqlite3.connect(dbPath, timeout=15.0) return connection def getKodiVideoDBPath(): From 219cdd9fbabe073159761fb7ccc676a4f6324de0 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 25 Sep 2016 19:18:27 +0200 Subject: [PATCH 59/86] Revert "Download one item at a time" --- resources/lib/librarysync.py | 18 ++++++++++-------- resources/settings.xml | 1 + 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 25a57776..fa79e5cb 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -384,6 +384,7 @@ class LibrarySync(Thread): self.vnodes = videonodes.VideoNodes() self.dialog = xbmcgui.Dialog() + self.syncThreadNumber = int(settings('syncThreadNumber')) self.installSyncDone = settings('SyncInstallRunDone') == 'true' self.showDbSync = settings('dbSyncIndicator') == 'true' self.enableMusic = settings('enableMusic') == "true" @@ -939,15 +940,16 @@ class LibrarySync(Thread): # Populate queue: GetMetadata for updateItem in self.updatelist: getMetadataQueue.put(updateItem) - # Spawn GetMetadata thread for downloading + # Spawn GetMetadata threads for downloading threads = [] - thread = ThreadedGetMetadata(getMetadataQueue, - processMetadataQueue, - getMetadataLock, - processMetadataLock) - thread.setDaemon(True) - thread.start() - threads.append(thread) + for i in range(min(self.syncThreadNumber, itemNumber)): + thread = ThreadedGetMetadata(getMetadataQueue, + processMetadataQueue, + getMetadataLock, + processMetadataLock) + thread.setDaemon(True) + thread.start() + threads.append(thread) log.info("%s download threads spawned" % len(threads)) # Spawn one more thread to process Metadata, once downloaded thread = ThreadedProcessMetadata(processMetadataQueue, diff --git a/resources/settings.xml b/resources/settings.xml index e7066114..2110da9d 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -46,6 +46,7 @@ + From 0b7e6ec0a3fe6ff665dbaf6c03b9d4ca7cd70367 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 25 Sep 2016 19:21:12 +0200 Subject: [PATCH 60/86] Compile regex only once --- resources/lib/PlexAPI.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/resources/lib/PlexAPI.py b/resources/lib/PlexAPI.py index 904066e3..eab9ad1a 100644 --- a/resources/lib/PlexAPI.py +++ b/resources/lib/PlexAPI.py @@ -57,7 +57,8 @@ import embydb_functions as embydb log = logging.getLogger("PLEX."+__name__) addonName = 'PlexKodiConnect' - +REGEX_IMDB = re.compile(r'''/(tt\d+)''') +REGEX_TVDB = re.compile(r'''tvdb://(\d+)''') ############################################################################### @@ -1450,10 +1451,10 @@ class API(): return None if providername == 'imdb': - regex = re.compile(r'''/(tt\d+)''') + regex = REGEX_IMDB elif providername == 'tvdb': # originally e.g. com.plexapp.agents.thetvdb://276564?lang=en - regex = re.compile(r'''tvdb://(\d+)''') + regex = REGEX_TVDB else: return None From 99abfea4fb28960bb213d4c746b1af968339ba01 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 25 Sep 2016 19:31:55 +0200 Subject: [PATCH 61/86] Version bump --- addon.xml | 2 +- changelog.txt | 8 ++++++++ 2 files changed, 9 insertions(+), 1 deletion(-) diff --git a/addon.xml b/addon.xml index f4193c60..c40532c0 100644 --- a/addon.xml +++ b/addon.xml @@ -1,7 +1,7 @@ diff --git a/changelog.txt b/changelog.txt index 7e5a8ed1..5e20e35a 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,11 @@ +version 1.3.8 (beta only) +- Fix TypeError for manually entering PMS address +- Fix "Opening Stream..." dialog not closing +- Try to prevent OperationalError: database is locked +- Revert "Download one item at a time" +- Remove obsolete import +- Compile regex only once + version 1.3.7 (beta only) - Music sync: Fix ProgrammingError - Don't set-up clips/trailers like other videos (Should fix PKC trying to tell the PMS where we are playing that item) From fe7d129c99865cffa6abd4c8e6c65c4832736206 Mon Sep 17 00:00:00 2001 From: tuphamnguyen Date: Sat, 8 Oct 2016 17:54:35 -0500 Subject: [PATCH 62/86] Update README.md Minor typos, nothing serious :) --- README.md | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/README.md b/README.md index afd25786..4ee5439d 100644 --- a/README.md +++ b/README.md @@ -20,9 +20,9 @@ I'm not in any way affiliated with Plex. Thank you very much for a small donatio ### IMPORTANT NOTES -1. If your are using a **low CPU device like a Raspberry Pi or a CuBox**, PKC might be instable or crash during initial sync. Lower the number of threads in the [PKC settings under Sync Options](https://github.com/croneter/PlexKodiConnect/wiki/PKC-settings#sync-options): `Limit artwork cache threads: 5` +1. If you are using a **low CPU device like a Raspberry Pi or a CuBox**, PKC might be instable or crash during initial sync. Lower the number of threads in the [PKC settings under Sync Options](https://github.com/croneter/PlexKodiConnect/wiki/PKC-settings#sync-options): `Limit artwork cache threads: 5` Don't forget to reboot Kodi after that. -2. If you post logs, your **Plex tokens** might be included. Be sure to double and tripple check for tokens before posting any logs anywhere by searching for `token` +2. If you post logs, your **Plex tokens** might be included. Be sure to double and triple check for tokens before posting any logs anywhere by searching for `token` 3. **Compatibility**: PKC is currently not compatible with Kodi's Video Extras plugin. **Deactivate Video Extras** if trailers/movies start randomly playing. @@ -107,5 +107,5 @@ The addon is not (and will not be) compatible with the MySQL database replacemen ### Credits - PlexKodiConnect shamelessly uses pretty much all the code of "Emby for Kodi" by the awesome Emby team (see https://github.com/MediaBrowser/plugin.video.emby). Thanks for sharing guys!! -- Plex Companion ("PlexBMC Helper") and other stuff was adapted from @Hippojay 's great work (see https://github.com/hippojay). +- Plex Companion ("PlexBMC Helper") and other stuff were adapted from @Hippojay 's great work (see https://github.com/hippojay). - The foundation of the Plex API is all iBaa's work (https://github.com/iBaa/PlexConnect). From d069fc7c480aa5b0364065b036f82004941e15c5 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Tue, 11 Oct 2016 18:28:07 +0200 Subject: [PATCH 63/86] Fix IndexError on deleting items --- resources/lib/librarysync.py | 9 ++++++--- 1 file changed, 6 insertions(+), 3 deletions(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index fa79e5cb..83e88146 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -1491,10 +1491,13 @@ class LibrarySync(Thread): "processing queue" for later """ for item in data: - typus = item.get('type') - state = item.get('state') + typus = int(item.get('type', 0)) + state = int(item.get('state', 0)) if state == 9 or typus in (1, 4, 10): - itemId = item.get('itemID') + itemId = str(item.get('itemID', '0')) + if itemId == '0': + log.warn('Received malformed PMS message: %s' % item) + continue # Have we already added this element? for existingItem in self.itemsToProcess: if existingItem['ratingKey'] == itemId: From 4b0583b1c9cbae183665695b640ed54d4e3130b6 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Tue, 11 Oct 2016 18:35:11 +0200 Subject: [PATCH 64/86] Prevent IndexError on unknown PMS ids --- resources/lib/librarysync.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 83e88146..900693bb 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -1440,10 +1440,14 @@ class LibrarySync(Thread): if xml in (None, 401): log.error('Could not download data for %s, skipping' % ratingKey) return False, item + try: + mediatype = xml[0].attrib['type'] + except (IndexError, KeyError): + log.error('Could not download metadata for %s' % ratingKey) + return False, item log.debug("Processing new/updated PMS item: %s" % ratingKey) viewtag = xml.attrib.get('librarySectionTitle') viewid = xml.attrib.get('librarySectionID') - mediatype = xml[0].attrib.get('type') # Attach mediatype for later item['mediatype'] = mediatype if mediatype == 'movie': From c02a95cc59c26a95b23283df096816197dd58f65 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Tue, 11 Oct 2016 18:37:47 +0200 Subject: [PATCH 65/86] Streamline code --- resources/lib/librarysync.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 900693bb..2a0188f7 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -1437,12 +1437,9 @@ class LibrarySync(Thread): def process_newitems(self, item): ratingKey = item['ratingKey'] xml = PF.GetPlexMetadata(ratingKey) - if xml in (None, 401): - log.error('Could not download data for %s, skipping' % ratingKey) - return False, item try: mediatype = xml[0].attrib['type'] - except (IndexError, KeyError): + except (IndexError, KeyError, TypeError): log.error('Could not download metadata for %s' % ratingKey) return False, item log.debug("Processing new/updated PMS item: %s" % ratingKey) From 5437f1a3554fef3d12c7a614322838118e29d5ec Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Tue, 11 Oct 2016 18:52:59 +0200 Subject: [PATCH 66/86] Ommit Plex DVR status messages - Hopefully fixes #110 --- resources/lib/librarysync.py | 6 ++++++ 1 file changed, 6 insertions(+) diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 2a0188f7..1fdef080 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -1492,9 +1492,15 @@ class LibrarySync(Thread): "processing queue" for later """ for item in data: + if 'tv.plex' in item.get('identifier', ''): + # Ommit Plex DVR messages - the Plex IDs are not corresponding + # (DVR ratingKeys are not unique and might correspond to a + # movie or episode) + continue typus = int(item.get('type', 0)) state = int(item.get('state', 0)) if state == 9 or typus in (1, 4, 10): + # Only process deleted items OR movies, episodes, tracks/songs itemId = str(item.get('itemID', '0')) if itemId == '0': log.warn('Received malformed PMS message: %s' % item) From 83f05037b89b88e1837a624af1e937521953a283 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Tue, 11 Oct 2016 19:13:42 +0200 Subject: [PATCH 67/86] Version bump --- addon.xml | 2 +- changelog.txt | 5 +++++ 2 files changed, 6 insertions(+), 1 deletion(-) diff --git a/addon.xml b/addon.xml index c40532c0..49b367d3 100644 --- a/addon.xml +++ b/addon.xml @@ -1,7 +1,7 @@ diff --git a/changelog.txt b/changelog.txt index 5e20e35a..373141c1 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,8 @@ +version 1.3.9 (beta only) +- Hopefully maximum compatibility with the new DVR component of the Plex Media Server :-) +- Ommit DVR status messages from the PMS. This should fix duplicate movies appearing +- Fix possible IndexError on deleting items + version 1.3.8 (beta only) - Fix TypeError for manually entering PMS address - Fix "Opening Stream..." dialog not closing From c747e915b1f8127ab7054b8f9d66d1aaa33a4c63 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Thu, 13 Oct 2016 11:27:31 +0200 Subject: [PATCH 68/86] Fix TypeError if no extras available - Fixes #123 --- resources/lib/PlexAPI.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/lib/PlexAPI.py b/resources/lib/PlexAPI.py index eab9ad1a..db4921c8 100644 --- a/resources/lib/PlexAPI.py +++ b/resources/lib/PlexAPI.py @@ -1721,7 +1721,7 @@ class API(): 'year': """ elements = [] - for extra in self.item.find('Extras'): + for extra in self.item.findall('Extras'): # Trailer: key = extra.attrib.get('key', None) title = extra.attrib.get('title', None) From 8099f993373660b7052958db34fd8590ffc7ef5f Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Thu, 13 Oct 2016 16:04:04 +0200 Subject: [PATCH 69/86] Remove size limitation on sync queue --- service.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/service.py b/service.py index 8183043f..a68d42e0 100644 --- a/service.py +++ b/service.py @@ -126,7 +126,7 @@ class Service(): initialsetup.InitialSetup().setup() # Queue for background sync - queue = Queue.Queue(maxsize=200) + queue = Queue.Queue() connectMsg = True if settings('connectMsg') == 'true' else False From a9f0301020afb6df7179756d7088aa631d044634 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Thu, 13 Oct 2016 16:54:06 +0200 Subject: [PATCH 70/86] Version bump --- addon.xml | 2 +- changelog.txt | 22 ++++++++++++++++++++++ 2 files changed, 23 insertions(+), 1 deletion(-) diff --git a/addon.xml b/addon.xml index 49b367d3..c251de8f 100644 --- a/addon.xml +++ b/addon.xml @@ -1,7 +1,7 @@ diff --git a/changelog.txt b/changelog.txt index 373141c1..cc4abd24 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,25 @@ +version 1.4.0 +- Compatibility with new DVR component of the Plex Media Server +- Speed up sync - download all art in the background. This should especially speed up your initial sync. Remember to let Kodi sit for a while to let it download the artwork +- New setting to look for missing artwork (the non-Plex stuff ;-)) +- Fix caching not working +- Ommit DVR status messages from the PMS. This should fix duplicate movies appearing +- Fix possible IndexError on deleting items +- Fix TypeError for manually entering PMS address +- Fix "Opening Stream..." dialog not closing +- Try to prevent OperationalError: database is locked +- Revert "Download one item at a time" +- Remove obsolete import +- Compile regex only once +- Music sync: Fix ProgrammingError +- Don't set-up clips/trailers like other videos (Should fix PKC trying to tell the PMS where we are playing that item) +- Fix capitalization +- Fix backgroundsync KeyError +- Don't double-update playstate of a playing item +- Rearrange the PKC settings +- Use file settings instead of window settings, should fix some errors on changing the PKC settings +- Other small fixes + version 1.3.9 (beta only) - Hopefully maximum compatibility with the new DVR component of the Plex Media Server :-) - Ommit DVR status messages from the PMS. This should fix duplicate movies appearing From 8179d094248b45c434ac2eeb180b668aa2e9fba6 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Thu, 13 Oct 2016 17:01:01 +0200 Subject: [PATCH 71/86] Update changelog --- changelog.txt | 2 ++ 1 file changed, 2 insertions(+) diff --git a/changelog.txt b/changelog.txt index cc4abd24..9a30eba9 100644 --- a/changelog.txt +++ b/changelog.txt @@ -18,6 +18,8 @@ version 1.4.0 - Don't double-update playstate of a playing item - Rearrange the PKC settings - Use file settings instead of window settings, should fix some errors on changing the PKC settings +- Remove size limitation on sync queue +- Fix TypeError if no extras available - Other small fixes version 1.3.9 (beta only) From 3918384bd50eff4525fc25f6e1e1b3655a583fa3 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 16 Oct 2016 13:58:59 +0200 Subject: [PATCH 72/86] Add warning to addon description --- addon.xml | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) diff --git a/addon.xml b/addon.xml index c251de8f..7162334e 100644 --- a/addon.xml +++ b/addon.xml @@ -19,10 +19,10 @@ all en GNU GENERAL PUBLIC LICENSE. Version 2, June 1991 - + https://forums.plex.tv https://github.com/croneter/PlexKodiConnect - + https://github.com/croneter/PlexKodiConnect - Connect Kodi to your Plex Media Server + Connect Kodi to your Plex Media Server. This plugin assumes that you manage all your videos with Plex (and none with Kodi). You might lose data already stored in the Kodi video and music databases (as this plugin directly changes them). Use at your own risk! \ No newline at end of file From 71954a009eb23611e1823d2152f623f79118f58d Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 16 Oct 2016 14:23:40 +0200 Subject: [PATCH 73/86] Add warning to readme --- README.md | 3 +++ 1 file changed, 3 insertions(+) diff --git a/README.md b/README.md index 4ee5439d..253850e5 100644 --- a/README.md +++ b/README.md @@ -5,6 +5,9 @@ PKC combines the best of Kodi - ultra smooth navigation, beautiful and highly cu Have a look at [some screenshots](https://github.com/croneter/PlexKodiConnect/wiki/Some-PKC-Screenshots) to see what's possible. +### Warning +This plugin assumes that you manage all your videos with Plex (and none with Kodi). You might lose data already stored in the Kodi video and music databases (as this plugin directly changes them). Use at your own risk! + ### Download and Installation [ ![Download](https://api.bintray.com/packages/croneter/PlexKodiConnect/PlexKodiConnect/images/download.svg) ](https://dl.bintray.com/croneter/PlexKodiConnect/bin/repository.plexkodiconnect/repository.plexkodiconnect-1.0.0.zip) From 4274942d49ec303af1619bf05452794c33f4d069 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Sun, 16 Oct 2016 16:44:41 +0200 Subject: [PATCH 74/86] Revert "Don't set-up clips/trailers like other videos" This reverts commit e4d736a6706823972ccbc82ed3e153c50b7c3860. --- resources/lib/playbackutils.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/resources/lib/playbackutils.py b/resources/lib/playbackutils.py index 51d9b63b..d1970f79 100644 --- a/resources/lib/playbackutils.py +++ b/resources/lib/playbackutils.py @@ -258,11 +258,8 @@ class PlaybackUtils(): def setProperties(self, playurl, listitem): # Set all properties necessary for plugin path playback - itemtype = self.API.getType() - if itemtype == 'clip': - log.debug('Setting up a clip/trailer, skip window variables') - return itemid = self.API.getRatingKey() + itemtype = self.API.getType() userdata = self.API.getUserData() embyitem = "emby_%s" % playurl From e001be5840b1cfa65cfa9df9144c35ce51dc57c1 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Mon, 17 Oct 2016 21:54:01 +0200 Subject: [PATCH 75/86] Move and increase setting sleep btw art downloads --- resources/settings.xml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/resources/settings.xml b/resources/settings.xml index 2110da9d..f12684ad 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -47,6 +47,7 @@ + @@ -109,7 +110,6 @@ - From cd65a9398901f2ede2d002acd12ac0724fe13ee8 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Mon, 17 Oct 2016 22:34:15 +0200 Subject: [PATCH 76/86] Fix Kodi crashing on low powered devices - Fixes #126 --- resources/language/English/strings.xml | 2 +- resources/language/German/strings.xml | 2 +- resources/lib/artwork.py | 11 +++++++++-- resources/lib/initialsetup.py | 12 ++++++++---- resources/settings.xml | 8 ++++---- service.py | 2 ++ 6 files changed, 25 insertions(+), 12 deletions(-) diff --git a/resources/language/English/strings.xml b/resources/language/English/strings.xml index a55487aa..d1c01c2c 100644 --- a/resources/language/English/strings.xml +++ b/resources/language/English/strings.xml @@ -306,7 +306,7 @@ Download movie set/collection art from FanArtTV Don't ask to pick a certain stream/quality Always pick best quality for trailers - Sleep between art downloads [ms] + Kodi runs on a low-power device (e.g. Raspberry Pi) Artwork diff --git a/resources/language/German/strings.xml b/resources/language/German/strings.xml index cef530e7..c6f13372 100644 --- a/resources/language/German/strings.xml +++ b/resources/language/German/strings.xml @@ -32,7 +32,7 @@ FanArtTV Film-Sets/Collections Bilder herunterladen Nicht fragen, welcher Stream/Qualität gespielt wird Trailer immer in der besten Qualität abspielen - Wartezeit zwischen Bilder downloads [ms] + Kodi läuft auf langsamer Hardware (z.B. Raspberry Pi) Artwork Verbindung diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index b226e82c..28f99b6e 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -132,7 +132,14 @@ def double_urldecode(text): class Image_Cache_Thread(Thread): xbmc_host = 'localhost' xbmc_port, xbmc_username, xbmc_password = setKodiWebServerDetails() - sleep_between = int(settings('sleep_between_art_downloads')) + sleep_between = 50 + if settings('low_powered_device') == 'true': + # Low CPU, potentially issues with limited number of threads + # Hence let Kodi wait till download is successful + timeout = (35.1, 35.1) + else: + # High CPU, no issue with limited number of threads + timeout = (0.01, 0.01) def __init__(self, queue): self.queue = queue @@ -164,7 +171,7 @@ class Image_Cache_Thread(Thread): url="http://%s:%s/image/image://%s" % (self.xbmc_host, self.xbmc_port, url), auth=(self.xbmc_username, self.xbmc_password), - timeout=(0.01, 0.01)) + timeout=self.timeout) except requests.Timeout: # We don't need the result, only trigger Kodi to start the # download. All is well diff --git a/resources/lib/initialsetup.py b/resources/lib/initialsetup.py index 0ceab110..e931adcf 100644 --- a/resources/lib/initialsetup.py +++ b/resources/lib/initialsetup.py @@ -417,6 +417,14 @@ class InitialSetup(): if settings('InstallQuestionsAnswered') == 'true': return + # Is your Kodi installed on a low-powered device like a Raspberry Pi? + # If yes, then we will reduce the strain on Kodi to prevent it from + # crashing. + if dialog.yesno(heading=addonName, line1=lang(39072)): + settings('low_powered_device', value="true") + else: + settings('low_powered_device', value="false") + # Additional settings where the user needs to choose # Direct paths (\\NAS\mymovie.mkv) or addon (http)? goToSettings = False @@ -463,10 +471,6 @@ class InitialSetup(): log.debug("User opted to use FanArtTV") settings('FanartTV', value="true") - # Is your Kodi installed on a low-powered device like a Raspberry Pi? - # If yes, then we will reduce the strain on Kodi to prevent it from - # crashing. - # Make sure that we only ask these questions upon first installation settings('InstallQuestionsAnswered', value='true') diff --git a/resources/settings.xml b/resources/settings.xml index f12684ad..21e16774 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -43,11 +43,11 @@ - - - + + + - + diff --git a/service.py b/service.py index a68d42e0..efdbd2d8 100644 --- a/service.py +++ b/service.py @@ -85,6 +85,8 @@ class Service(): log.warn("%s Version: %s" % (addonName, self.clientInfo.getVersion())) log.warn("Using plugin paths: %s" % (settings('useDirectPaths') != "true")) + log.warn("Using a low powered device: %s" + % settings('low_powered_device')) log.warn("Log Level: %s" % logLevel) # Reset window props for profile switch From 1a4660571b19576f6365c5281098e84dc3f9cbd8 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Mon, 17 Oct 2016 22:48:01 +0200 Subject: [PATCH 77/86] Only start downloading art AFTER sync completed - Will speed up sync --- resources/lib/artwork.py | 7 ++++++- 1 file changed, 6 insertions(+), 1 deletion(-) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index 28f99b6e..ea6b6f17 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -126,7 +126,6 @@ def double_urldecode(text): return unquote(unquote(text)) -@ThreadMethodsAdditionalSuspend('suspend_LibraryThread') @ThreadMethodsAdditionalStop('plex_shouldStop') @ThreadMethods class Image_Cache_Thread(Thread): @@ -145,6 +144,12 @@ class Image_Cache_Thread(Thread): self.queue = queue Thread.__init__(self) + def threadSuspended(self): + # Overwrite method to add TWO additional suspends + return (self._threadSuspended or + window('suspend_LibraryThread') or + window('plex_dbScan')) + def run(self): threadStopped = self.threadStopped threadSuspended = self.threadSuspended From 2b363b7a30d879d9cb8622f19200e7a4d6e2121b Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Mon, 17 Oct 2016 22:50:36 +0200 Subject: [PATCH 78/86] Log art caching --- resources/lib/artwork.py | 2 ++ 1 file changed, 2 insertions(+) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index ea6b6f17..f1cb43b1 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -180,6 +180,7 @@ class Image_Cache_Thread(Thread): except requests.Timeout: # We don't need the result, only trigger Kodi to start the # download. All is well + log.debug('Caching art initiated: %s' % url) break except requests.ConnectionError: # Server thinks its a DOS attack, ('error 10053') @@ -202,6 +203,7 @@ class Image_Cache_Thread(Thread): log.error("Traceback:\n%s" % traceback.format_exc()) break # We did not even get a timeout + log.debug('Cached art: %s' % url) break queue.task_done() log.debug('Downloaded art: %s' % double_urldecode(url)) From a2e2d2f5f8da66665e238dbbfc74238418b82bbe Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Mon, 17 Oct 2016 22:55:32 +0200 Subject: [PATCH 79/86] Revert "Log art caching" This reverts commit 2b363b7a30d879d9cb8622f19200e7a4d6e2121b. --- resources/lib/artwork.py | 2 -- 1 file changed, 2 deletions(-) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index f1cb43b1..ea6b6f17 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -180,7 +180,6 @@ class Image_Cache_Thread(Thread): except requests.Timeout: # We don't need the result, only trigger Kodi to start the # download. All is well - log.debug('Caching art initiated: %s' % url) break except requests.ConnectionError: # Server thinks its a DOS attack, ('error 10053') @@ -203,7 +202,6 @@ class Image_Cache_Thread(Thread): log.error("Traceback:\n%s" % traceback.format_exc()) break # We did not even get a timeout - log.debug('Cached art: %s' % url) break queue.task_done() log.debug('Downloaded art: %s' % double_urldecode(url)) From 23384c5d9edc8c989d17eff62aa52deb79c66ee0 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Mon, 17 Oct 2016 23:15:05 +0200 Subject: [PATCH 80/86] Fix art not caching efficiently --- resources/lib/artwork.py | 2 +- resources/lib/librarysync.py | 4 ---- 2 files changed, 1 insertion(+), 5 deletions(-) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index ea6b6f17..545aa0ac 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -204,7 +204,7 @@ class Image_Cache_Thread(Thread): # We did not even get a timeout break queue.task_done() - log.debug('Downloaded art: %s' % double_urldecode(url)) + log.debug('Cached art: %s' % double_urldecode(url)) # Sleep for a bit to reduce CPU strain xbmc.sleep(sleep_between) log.info("---===### Stopped Image_Cache_Thread ###===---") diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index 1fdef080..6394cbf0 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -1850,9 +1850,7 @@ class LibrarySync(Thread): # Only do this once every 10 seconds if now - lastProcessing > 10: lastProcessing = now - window('plex_dbScan', value="true") processItems() - window('plex_dbScan', clear=True) # See if there is a PMS message we need to handle try: message = queue.get(block=False) @@ -1861,10 +1859,8 @@ class LibrarySync(Thread): continue # Got a message from PMS; process it else: - window('plex_dbScan', value="true") processMessage(message) queue.task_done() - window('plex_dbScan', clear=True) # NO sleep! continue else: From 1bae637830707d6efbf25a55f8ef8c21784e067a Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Tue, 18 Oct 2016 19:49:11 +0200 Subject: [PATCH 81/86] Adjust download threads to 1 for low powered devices --- resources/language/English/strings.xml | 2 +- resources/language/German/strings.xml | 2 +- resources/lib/initialsetup.py | 1 + resources/settings.xml | 2 +- 4 files changed, 4 insertions(+), 3 deletions(-) diff --git a/resources/language/English/strings.xml b/resources/language/English/strings.xml index d1c01c2c..913a3512 100644 --- a/resources/language/English/strings.xml +++ b/resources/language/English/strings.xml @@ -349,7 +349,7 @@ - Number of trailers to play before a movie Boost audio when transcoding Burnt-in subtitle size - Limit download sync threads (recommended for rpi: 1) + Limit download sync threads (rec. for rpi: 1) Enable Plex Companion Plex Companion Port (change only if needed) Activate Plex Companion debug log diff --git a/resources/language/German/strings.xml b/resources/language/German/strings.xml index c6f13372..789debcf 100644 --- a/resources/language/German/strings.xml +++ b/resources/language/German/strings.xml @@ -288,7 +288,7 @@ - Anzahl abzuspielender Trailer vor einem Film Audio Verstärkung (audio boost) wenn transkodiert wird Grösse des Untertitels, falls burnt-in - Anzahl Download Sync Threads beschränken (Empfehlung für rpi: 1) + Download Sync Threads beschränken (Empfehlung RPI: 1) Plex Companion aktivieren Plex Companion Port (nur bei Bedarf ändern) Plex Companion debug log aktivieren diff --git a/resources/lib/initialsetup.py b/resources/lib/initialsetup.py index e931adcf..62ea9e20 100644 --- a/resources/lib/initialsetup.py +++ b/resources/lib/initialsetup.py @@ -422,6 +422,7 @@ class InitialSetup(): # crashing. if dialog.yesno(heading=addonName, line1=lang(39072)): settings('low_powered_device', value="true") + settings('syncThreadNumber', value="1") else: settings('low_powered_device', value="false") diff --git a/resources/settings.xml b/resources/settings.xml index 21e16774..9c18a978 100644 --- a/resources/settings.xml +++ b/resources/settings.xml @@ -46,7 +46,7 @@ - + From ec7d4efc8046d13cd1768700d1cb32c9b8cb118f Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Tue, 18 Oct 2016 20:01:41 +0200 Subject: [PATCH 82/86] Remove obsolete import --- resources/lib/artwork.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index 545aa0ac..645a1568 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -14,8 +14,7 @@ import xbmcgui import xbmcvfs from utils import window, settings, language as lang, kodiSQL, tryEncode, \ - tryDecode, IfExists, ThreadMethods, ThreadMethodsAdditionalSuspend, \ - ThreadMethodsAdditionalStop + tryDecode, IfExists, ThreadMethods, ThreadMethodsAdditionalStop # Disable annoying requests warnings import requests.packages.urllib3 From d71f6e88879accd9d8847557c264e4fb5131eca6 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Tue, 18 Oct 2016 20:32:16 +0200 Subject: [PATCH 83/86] Remove obsolete lock --- resources/lib/artwork.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/resources/lib/artwork.py b/resources/lib/artwork.py index 645a1568..7c789bce 100644 --- a/resources/lib/artwork.py +++ b/resources/lib/artwork.py @@ -6,7 +6,7 @@ import logging import requests import os from urllib import quote_plus, unquote -from threading import Lock, Thread +from threading import Thread import Queue import xbmc @@ -210,7 +210,6 @@ class Image_Cache_Thread(Thread): class Artwork(): - lock = Lock() enableTextureCache = settings('enableTextureCache') == "true" if enableTextureCache: queue = Queue.Queue() From 5ac807760b8e82da11ca852b688b6ac938b8616b Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Tue, 18 Oct 2016 20:47:45 +0200 Subject: [PATCH 84/86] Fix movie year for Krypton (reset your Kodi DB!) - Fixes #79 --- resources/lib/itemtypes.py | 37 ++++++++++++++++++++++++++----------- 1 file changed, 26 insertions(+), 11 deletions(-) diff --git a/resources/lib/itemtypes.py b/resources/lib/itemtypes.py index 7b91f9d3..65490717 100644 --- a/resources/lib/itemtypes.py +++ b/resources/lib/itemtypes.py @@ -430,17 +430,32 @@ class Movies(Items): % (itemid, title)) # Update the movie entry - query = ' '.join(( - - "UPDATE movie", - "SET c00 = ?, c01 = ?, c02 = ?, c03 = ?, c04 = ?, c05 = ?, c06 = ?,", - "c07 = ?, c09 = ?, c10 = ?, c11 = ?, c12 = ?, c14 = ?, c15 = ?,", - "c16 = ?, c18 = ?, c19 = ?, c21 = ?, c22 = ?, c23 = ?", - "WHERE idMovie = ?" - )) - kodicursor.execute(query, (title, plot, shortplot, tagline, votecount, rating, writer, - year, imdb, sorttitle, runtime, mpaa, genre, director, title, studio, trailer, - country, playurl, pathid, movieid)) + if self.kodiversion > 16: + query = ' '.join(( + "UPDATE movie", + "SET c00 = ?, c01 = ?, c02 = ?, c03 = ?, c04 = ?, c05 = ?," + "c06 = ?, c07 = ?, c09 = ?, c10 = ?, c11 = ?, c12 = ?," + "c14 = ?, c15 = ?, c16 = ?, c18 = ?, c19 = ?, c21 = ?," + "c22 = ?, c23 = ?, premiered = ?", + "WHERE idMovie = ?" + )) + kodicursor.execute(query, (title, plot, shortplot, tagline, + votecount, rating, writer, year, imdb, sorttitle, runtime, + mpaa, genre, director, title, studio, trailer, country, + playurl, pathid, year, movieid)) + else: + query = ' '.join(( + "UPDATE movie", + "SET c00 = ?, c01 = ?, c02 = ?, c03 = ?, c04 = ?, c05 = ?," + "c06 = ?, c07 = ?, c09 = ?, c10 = ?, c11 = ?, c12 = ?," + "c14 = ?, c15 = ?, c16 = ?, c18 = ?, c19 = ?, c21 = ?," + "c22 = ?, c23 = ?", + "WHERE idMovie = ?" + )) + kodicursor.execute(query, (title, plot, shortplot, tagline, + votecount, rating, writer, year, imdb, sorttitle, runtime, + mpaa, genre, director, title, studio, trailer, country, + playurl, pathid, movieid)) # Update the checksum in emby table emby_db.updateReference(itemid, checksum) From b05fe88c07af3523e4ae4b110c153e45adf69541 Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Tue, 18 Oct 2016 20:50:46 +0200 Subject: [PATCH 85/86] Version bump --- addon.xml | 2 +- changelog.txt | 7 +++++++ 2 files changed, 8 insertions(+), 1 deletion(-) diff --git a/addon.xml b/addon.xml index 7162334e..0ee967e3 100644 --- a/addon.xml +++ b/addon.xml @@ -1,7 +1,7 @@ diff --git a/changelog.txt b/changelog.txt index 9a30eba9..3122dcef 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,10 @@ +version 1.4.1 (beta only) +- Fix Kodi crashing on low powered devices +- Fix movie year for Krypton (reset your Kodi DB!) +- Only start downloading art AFTER sync completed +- Add warning to addon description +- Revert "Don't set-up clips/trailers like other videos" + version 1.4.0 - Compatibility with new DVR component of the Plex Media Server - Speed up sync - download all art in the background. This should especially speed up your initial sync. Remember to let Kodi sit for a while to let it download the artwork From 38115fb5a99a680255bd5e8057fc5c87c354ae5d Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Thu, 20 Oct 2016 18:55:54 +0200 Subject: [PATCH 86/86] Version bump --- addon.xml | 2 +- changelog.txt | 3 +++ 2 files changed, 4 insertions(+), 1 deletion(-) diff --git a/addon.xml b/addon.xml index 0ee967e3..8d848b4a 100644 --- a/addon.xml +++ b/addon.xml @@ -1,7 +1,7 @@ diff --git a/changelog.txt b/changelog.txt index 3122dcef..c7e9b2ac 100644 --- a/changelog.txt +++ b/changelog.txt @@ -1,3 +1,6 @@ +version 1.4.2 +Make previous version available for everyone + version 1.4.1 (beta only) - Fix Kodi crashing on low powered devices - Fix movie year for Krypton (reset your Kodi DB!)