From ae9d4924c2a3c87156f43d52dac9377c93f9632b Mon Sep 17 00:00:00 2001 From: tomkat83 Date: Fri, 11 Mar 2016 14:42:14 +0100 Subject: [PATCH] Redesign fast sync --- resources/lib/PlexAPI.py | 34 +--- resources/lib/PlexFunctions.py | 20 +++ resources/lib/embydb_functions.py | 33 ++++ resources/lib/kodidb_functions.py | 64 +++++++- resources/lib/kodimonitor.py | 14 +- resources/lib/librarysync.py | 247 ++++++++++++++++++++---------- resources/lib/utils.py | 78 +++++++++- 7 files changed, 365 insertions(+), 125 deletions(-) diff --git a/resources/lib/PlexAPI.py b/resources/lib/PlexAPI.py index 33f57771..04ec4341 100644 --- a/resources/lib/PlexAPI.py +++ b/resources/lib/PlexAPI.py @@ -1476,36 +1476,6 @@ class API(): """ return self.part - def DateToKodi(self, stamp): - """ - converts a Unix time stamp (seconds passed sinceJanuary 1 1970) to a - propper, human-readable time stamp used by Kodi - - Output: Y-m-d h:m:s = 2009-04-05 23:16:04 - """ - # DATEFORMAT = xbmc.getRegion('dateshort') - # TIMEFORMAT = xbmc.getRegion('meridiem') - # date_time = time.localtime(stamp) - # if DATEFORMAT[1] == 'd': - # localdate = time.strftime('%d-%m-%Y', date_time) - # elif DATEFORMAT[1] == 'm': - # localdate = time.strftime('%m-%d-%Y', date_time) - # else: - # localdate = time.strftime('%Y-%m-%d', date_time) - # if TIMEFORMAT != '/': - # localtime = time.strftime('%I:%M%p', date_time) - # else: - # localtime = time.strftime('%H:%M', date_time) - # return localtime + ' ' + localdate - try: - # DATEFORMAT = xbmc.getRegion('dateshort') - # TIMEFORMAT = xbmc.getRegion('meridiem') - date_time = time.localtime(float(stamp)) - localdate = time.strftime('%Y-%m-%d %H:%M:%S', date_time) - except: - localdate = None - return localdate - def getType(self): """ Returns the type of media, e.g. 'movie' or 'clip' for trailers @@ -1543,7 +1513,7 @@ class API(): """ Returns the date when this library item was created """ - return self.DateToKodi(self.item.attrib.get('addedAt', None)) + return utils.DateToKodi(self.item.attrib.get('addedAt', None)) def getUserData(self): """ @@ -1576,7 +1546,7 @@ class API(): played = True try: - lastPlayedDate = self.DateToKodi(int(item['lastViewedAt'])) + lastPlayedDate = utils.DateToKodi(int(item['lastViewedAt'])) except: lastPlayedDate = None diff --git a/resources/lib/PlexFunctions.py b/resources/lib/PlexFunctions.py index 6c740346..e8c4ace3 100644 --- a/resources/lib/PlexFunctions.py +++ b/resources/lib/PlexFunctions.py @@ -3,6 +3,7 @@ from urllib import urlencode from ast import literal_eval from urlparse import urlparse, parse_qs import re +import time from xbmcaddon import Addon @@ -371,3 +372,22 @@ def PMSHttpsEnabled(url): # couldn't get an xml - switch to http traffic logMsg('PMSHttpsEnabled', 'PMS on %s talks HTTPS' % url, 1) return False + + +def scrobble(ratingKey, state): + """ + Tells the PMS to set an item's watched state to state="watched" or + state="unwatched" + """ + args = { + 'key': ratingKey, + 'identifier': 'com.plexapp.plugins.library' + } + if state == "watched": + url = "{server}/:/scrobble?" + urlencode(args) + elif state == "unwatched": + url = "{server}/:/unscrobble?" + urlencode(args) + else: + return + downloadutils.DownloadUtils().downloadUrl(url, type="GET") + logMsg("Toggled watched state for Plex item %s" % ratingKey, 1) diff --git a/resources/lib/embydb_functions.py b/resources/lib/embydb_functions.py index b7c64768..77526cef 100644 --- a/resources/lib/embydb_functions.py +++ b/resources/lib/embydb_functions.py @@ -152,6 +152,22 @@ class Embydb_Functions(): )) self.embycursor.execute(query, (viewid,)) + def getItem_byFileId(self, fileId): + """ + Returns the Plex itemId by using the Kodi fileId + """ + query = ' '.join(( + "SELECT emby_id", + "FROM emby", + "WHERE kodi_fileid = ?" + )) + try: + self.embycursor.execute(query, (fileId,)) + item = self.embycursor.fetchone()[0] + return item + except: + return None + def getItem_byId(self, embyid): embycursor = self.embycursor @@ -183,6 +199,23 @@ class Embydb_Functions(): return items + def getPlexId(self, kodiid): + """ + Returns the Plex ID usind the Kodiid. Result: + (Plex Id, Parent's Plex Id) + """ + query = ' '.join(( + "SELECT emby_id, parent_id", + "FROM emby", + "WHERE kodi_id = ?" + )) + try: + self.embycursor.execute(query, (kodiid)) + item = self.embycursor.fetchone() + return item + except: + return None + def getItem_byKodiId(self, kodiid, mediatype): embycursor = self.embycursor diff --git a/resources/lib/kodidb_functions.py b/resources/lib/kodidb_functions.py index 00965a8f..27e990e4 100644 --- a/resources/lib/kodidb_functions.py +++ b/resources/lib/kodidb_functions.py @@ -17,7 +17,7 @@ class GetKodiDB(): do stuff with kodi_db Parameters: - itemType: itemtype for Kodi DB, e.g. 'video' + itemType: itemtype for Kodi DB, e.g. 'video', 'music' On exiting "with" (no matter what), commits get automatically committed and the db gets closed @@ -701,6 +701,68 @@ class Kodidb_Functions(): ) cursor.execute(query, (fileid, 2, subtitletrack)) + def getResumes(self): + """ + VIDEOS + + Returns all Kodi idFile that have a resume point set (not unwatched + ones or items that have already been completely watched) + """ + cursor = self.cursor + + query = ' '.join(( + "SELECT idFile", + "FROM bookmark" + )) + try: + rows = cursor.execute(query) + except: + return [] + ids = [] + for row in rows: + ids.append(row[0]) + return ids + + def getUnplayedMusicItems(self): + """ + MUSIC + + Returns all Kodi Item idFile that have not yet been completely played + """ + query = ' '.join(( + "SELECT idPath", + "FROM song", + "WHERE iTimesPlayed IS NULL OR iTimesPlayed = ''" + )) + try: + rows = self.cursor.execute(query) + except: + return [] + ids = [] + for row in rows: + ids.append(row[0]) + return ids + + def getUnplayedItems(self): + """ + VIDEOS + + Returns all Kodi Item idFile that have not yet been completely played + """ + query = ' '.join(( + "SELECT idFile", + "FROM files", + "WHERE playCount IS NULL OR playCount = ''" + )) + try: + rows = self.cursor.execute(query) + except: + return [] + ids = [] + for row in rows: + ids.append(row[0]) + return ids + def addPlaystate(self, fileid, resume_seconds, total_seconds, playcount, dateplayed): cursor = self.cursor diff --git a/resources/lib/kodimonitor.py b/resources/lib/kodimonitor.py index 0a6f8e5b..8c558daa 100644 --- a/resources/lib/kodimonitor.py +++ b/resources/lib/kodimonitor.py @@ -11,8 +11,7 @@ import downloadutils import embydb_functions as embydb import playbackutils as pbutils import utils - -from urllib import urlencode +from PlexFunctions import scrobble ############################################################################### @@ -144,16 +143,11 @@ class KodiMonitor(xbmc.Monitor): utils.window('emby_skipWatched%s' % itemid, clear=True) else: # notify the server - args = {'key': itemid, - 'identifier': 'com.plexapp.plugins.library'} if playcount != 0: - url = "{server}/:/scrobble?" + urlencode(args) - doUtils.downloadUrl(url, type="GET") - self.logMsg("Mark as watched for itemid: %s" % itemid, 1) + scrobble(itemid, 'watched') else: - url = "{server}/:/unscrobble?" + urlencode(args) - doUtils.downloadUrl(url, type="GET") - self.logMsg("Mark as unwatched for itemid: %s" % itemid, 1) + scrobble(itemid, 'unwatched') + finally: embycursor.close() diff --git a/resources/lib/librarysync.py b/resources/lib/librarysync.py index b38ef481..9394fe8e 100644 --- a/resources/lib/librarysync.py +++ b/resources/lib/librarysync.py @@ -238,6 +238,12 @@ class LibrarySync(Thread): self.enableBackgroundSync = True if utils.settings( 'enableBackgroundSync') == "true" else False + # Time offset between Kodi and PMS in seconds (=Koditime - PMStime) + self.timeoffset = 0 + # Time in seconds to look into the past when looking for PMS changes + # (safety margin - the larger, the more items we need to process) + self.syncPast = 30 + Thread.__init__(self) def showKodiNote(self, message, forced=False, icon="plex"): @@ -265,6 +271,102 @@ class LibrarySync(Thread): time=7000, sound=True) + def syncPMStime(self): + """ + PMS does not provide a means to get a server timestamp. This is a work- + around. + """ + self.logMsg('Synching time with PMS server', 0) + # Find a PMS item where we can toggle the view state to enforce a + # change in lastViewedAt + with kodidb.GetKodiDB('video') as kodi_db: + unplayedIds = kodi_db.getUnplayedItems() + resumeIds = kodi_db.getResumes() + self.logMsg('resumeIds: %s' % resumeIds, 1) + + plexId = False + for unplayedId in unplayedIds: + if unplayedId not in resumeIds: + # Found an item we can work with! + kodiId = unplayedId + self.logMsg('Found kodiId: %s' % kodiId, 1) + # Get Plex ID using the Kodi ID + with embydb.GetEmbyDB() as emby_db: + plexId = emby_db.getItem_byFileId(kodiId) + if plexId: + self.logMsg('Found plexId: %s' % plexId, 1) + break + + # Try getting a music item if we did not find a video item + if not plexId: + self.logMsg("Could not find a video item to sync time with", 0) + with kodidb.GetKodiDB('music') as kodi_db: + unplayedIds = kodi_db.getUnplayedMusicItems() + # We don't care about resuming songs in the middle + for unplayedId in unplayedIds: + # Found an item we can work with! + kodiId = unplayedId + self.logMsg('Found kodiId: %s' % kodiId, 1) + # Get Plex ID using the Kodi ID + with embydb.GetEmbyDB() as emby_db: + plexId = emby_db.getItem_byFileId(kodiId) + if plexId: + self.logMsg('Found plexId: %s' % plexId, 1) + break + else: + self.logMsg("Could not find an item to sync time with", -1) + self.logMsg("Aborting PMS-Kodi time sync", -1) + return + + # Get the Plex item's metadata + xml = PlexFunctions.GetPlexMetadata(plexId) + if not xml: + self.logMsg("Could not download metadata, aborting time sync", -1) + return + libraryId = xml[0].attrib['librarySectionID'] + # Get a PMS timestamp to start our question with + timestamp = xml[0].attrib.get('lastViewedAt') + if not timestamp: + timestamp = xml[0].attrib.get('updatedAt') + self.logMsg('Using items updatedAt=%s' % timestamp, 1) + if not timestamp: + timestamp = xml[0].attrib.get('addedAt') + self.logMsg('Using items addedAt=%s' % timestamp, 1) + # Set the timer + koditime = utils.getUnixTimestamp() + # Toggle watched state + PlexFunctions.scrobble(plexId, 'watched') + # Let the PMS process this first! + xbmc.sleep(2000) + # Get all PMS items to find the item we changed + items = PlexFunctions.GetAllPlexLeaves(libraryId, + lastViewedAt=timestamp) + # Toggle watched state back + PlexFunctions.scrobble(plexId, 'unwatched') + # Get server timestamp for this change + plextime = None + for item in items: + if item.attrib['ratingKey'] == plexId: + plextime = item.attrib.get('lastViewedAt') + break + if not plextime: + self.logMsg("Could not set the items watched state, abort", -1) + return + + # Calculate time offset Kodi-PMS + self.timeoffset = int(koditime) - int(plextime) + self.logMsg("Time offset Koditime - PMStime in seconds: %s" + % str(self.timeoffset), 0) + + def getPMSfromKodiTime(self, koditime): + """ + Uses self.timeoffset to return the PMS time for a given Kodi timestamp + (in unix time) + + Feed with integers + """ + return koditime - self.timeoffset + def resetProcessedItems(self): """ Resets the list of PMS items that we have already processed @@ -279,7 +381,7 @@ class LibrarySync(Thread): 'track': {} } - def getFastUpdateList(self, xml, plexType, viewName, viewId, update=True): + def getFastUpdateList(self, xml, plexType, viewName, viewId): """ THIS METHOD NEEDS TO BE FAST! => e.g. no API calls @@ -305,71 +407,43 @@ class LibrarySync(Thread): self.allPlexElementsId APPENDED(!!) dict = {itemid: checksum} """ - # Updated items are prefered over userdata updates! - if update: - # Needs to call other methods than if we're only updating userdata - for item in xml: - itemId = item.attrib.get('ratingKey') - # Skipping items 'title=All episodes' without a 'ratingKey' - if not itemId: + # Needs to call other methods than if we're only updating userdata + for item in xml: + itemId = item.attrib.get('ratingKey') + # Skipping items 'title=All episodes' without a 'ratingKey' + if not itemId: + continue + + lastViewedAt = item.attrib.get('lastViewedAt') + updatedAt = item.attrib.get('updatedAt') + + # returns the tuple (lastViewedAt, updatedAt) for the + # specific item + res = self.processed[plexType].get(itemId) + if res: + # Only look at the updatedAt flag! + # tuple: (lastViewedAt, updatedAt) + if res[1] == updatedAt: + # Nothing to update, we have already processed this + # item continue - - lastViewedAt = item.attrib.get('lastViewedAt') - updatedAt = item.attrib.get('updatedAt') - - # returns the tuple (lastViewedAt, updatedAt) for the - # specific item - res = self.processed[plexType].get(itemId) - if res: - if res == (lastViewedAt, updatedAt): - # Nothing to update, we have already processed this - # item - continue - title = item.attrib.get('title', 'Missing Title Name') - # We need to process this: - self.updatelist.append({ - 'itemId': itemId, - 'itemType': PlexFunctions.GetItemClassFromType( - plexType), - 'method': PlexFunctions.GetMethodFromPlexType(plexType), - 'viewName': viewName, - 'viewId': viewId, - 'title': title - }) - # And safe to self.processed: - self.processed[plexType][itemId] = (lastViewedAt, updatedAt) - else: - # Needs to call other methods than if we're only updating userdata - for item in xml: - itemId = item.attrib.get('ratingKey') - # Skipping items 'title=All episodes' without a 'ratingKey' - if not itemId: - continue - - lastViewedAt = item.attrib.get('lastViewedAt') - updatedAt = item.attrib.get('updatedAt') - - # returns the tuple (lastViewedAt, updatedAt) for the - # specific item - res = self.processed[plexType].get(itemId) - if res: - if res == (lastViewedAt, updatedAt): - # Nothing to update, we have already processed this - # item - continue - title = item.attrib.get('title', 'Missing Title Name') - # We need to process this: - self.updatelist.append({ - 'itemId': itemId, - 'itemType': PlexFunctions.GetItemClassFromType( - plexType), - 'method': 'updateUserdata', - 'viewName': viewName, - 'viewId': viewId, - 'title': title - }) - # And safe to self.processed: - self.processed[plexType][itemId] = (lastViewedAt, updatedAt) + title = item.attrib.get('title', 'Missing Title Name') + # We need to process this: + self.updatelist.append({ + 'itemId': itemId, + 'itemType': PlexFunctions.GetItemClassFromType( + plexType), + 'method': PlexFunctions.GetMethodFromPlexType(plexType), + 'viewName': viewName, + 'viewId': viewId, + 'title': title + }) + # And safe to self.processed: + self.processed[plexType][itemId] = (lastViewedAt, updatedAt) + # Quickly log + if self.updatelist: + self.logMsg('fastSync updatelist: %s' % self.updatelist, 1) + self.logMsg('fastSync processed list: %s' % self.processed, 1) def fastSync(self): """ @@ -392,8 +466,7 @@ class LibrarySync(Thread): } } """ - self.compare = True - # Get last sync time + # Get last sync time and look a bit in the past (safety margin) lastSync = self.lastSync - self.syncPast # Set new timestamp NOW because sync might take a while self.saveLastSync() @@ -408,8 +481,8 @@ class LibrarySync(Thread): for view in self.views: self.updatelist = [] # Get items per view - items = PlexFunctions.GetAllPlexLeaves(view['id'], - updatedAt=lastSync) + items = PlexFunctions.GetAllPlexLeaves( + view['id'], updatedAt=self.getPMSfromKodiTime(lastSync)) # Just skip if something went wrong if not items: continue @@ -439,14 +512,11 @@ class LibrarySync(Thread): # We don't need to refresh the Kodi library for deltas!! # Start with an empty ElementTree and attach items to update movieupdate = False - movieXML = etree.Element('root') episodeupdate = False - episodeXML = etree.Element('root') songupdate = False - musicXML = etree.Element('root') for view in self.views: - items = PlexFunctions.GetAllPlexLeaves(view['id'], - lastViewedAt=lastSync) + items = PlexFunctions.GetAllPlexLeaves( + view['id'], lastViewedAt=self.getPMSfromKodiTime(lastSync)) for item in items: itemId = item.attrib.get('ratingKey') # Skipping items 'title=All episodes' without a 'ratingKey' @@ -460,21 +530,32 @@ class LibrarySync(Thread): # specific item res = self.processed[plexType].get(itemId) if res: - if res == (lastViewedAt, updatedAt): + # Only look at lastViewedAt + if res[0] == lastViewedAt: # Nothing to update, we have already processed this # item continue if plexType == 'movie': - movieXML.append(item) movieupdate = True + try: + movieXML.append(item) + except: + movieXML = etree.Element('root') + movieXML.append(item) elif plexType == 'episode': - episodeXML.append(item) episodeupdate = True + try: + episodeXML.append(item) + except: + episodeXML = etree.Element('root') + episodeXML.append(item) elif plexType == 'track': - musicXML.append(item) songupdate = True - else: - self.logMsg('Unknown plex type %s' % plexType, -1) + try: + musicXML.append(item) + except: + musicXML = etree.Element('root') + musicXML.append(item) # And safe to self.processed: self.processed[plexType][itemId] = (lastViewedAt, updatedAt) @@ -1382,6 +1463,8 @@ class LibrarySync(Thread): log("Db version: %s" % settings('dbCreatedWithVersion'), 0) log("Initial start-up full sync starting", 0) librarySync = fullSync(manualrun=True) + # Initialize time offset Kodi - PMS + self.syncPMStime() window('emby_dbScan', clear=True) if librarySync: log("Initial start-up full sync successful", 0) @@ -1440,9 +1523,11 @@ class LibrarySync(Thread): elif enableBackgroundSync: # Run full lib scan approx every 30min if count >= 1800: + count = 0 # Also reset self.processed, just in case self.resetProcessedItems() - count = 0 + # Recalculate time offset Kodi - PMS + self.syncPMStime() window('emby_dbScan', value="true") log('Running background full lib scan', 0) fullSync(manualrun=True) diff --git a/resources/lib/utils.py b/resources/lib/utils.py index 84642439..53ac5c38 100644 --- a/resources/lib/utils.py +++ b/resources/lib/utils.py @@ -19,12 +19,88 @@ import xbmcaddon import xbmcgui import xbmcvfs - ############################################################################### addonName = xbmcaddon.Addon().getAddonInfo('name') +def DateToKodi(stamp): + """ + converts a Unix time stamp (seconds passed sinceJanuary 1 1970) to a + propper, human-readable time stamp used by Kodi + + Output: Y-m-d h:m:s = 2009-04-05 23:16:04 + """ + # DATEFORMAT = xbmc.getRegion('dateshort') + # TIMEFORMAT = xbmc.getRegion('meridiem') + # date_time = time.localtime(stamp) + # if DATEFORMAT[1] == 'd': + # localdate = time.strftime('%d-%m-%Y', date_time) + # elif DATEFORMAT[1] == 'm': + # localdate = time.strftime('%m-%d-%Y', date_time) + # else: + # localdate = time.strftime('%Y-%m-%d', date_time) + # if TIMEFORMAT != '/': + # localtime = time.strftime('%I:%M%p', date_time) + # else: + # localtime = time.strftime('%H:%M', date_time) + # return localtime + ' ' + localdate + try: + # DATEFORMAT = xbmc.getRegion('dateshort') + # TIMEFORMAT = xbmc.getRegion('meridiem') + date_time = time.localtime(float(stamp)) + localdate = time.strftime('%Y-%m-%d %H:%M:%S', date_time) + except: + localdate = None + return localdate + + +def changePlayState(itemType, kodiId, playCount, lastplayed): + """ + YET UNUSED + + kodiId: int or str + playCount: int or str + lastplayed: str or int unix timestamp + """ + logMsg("changePlayState", "start", 1) + lastplayed = DateToKodi(lastplayed) + + kodiId = int(kodiId) + playCount = int(playCount) + method = { + 'movie': ' VideoLibrary.SetMovieDetails', + 'episode': 'VideoLibrary.SetEpisodeDetails', + 'musicvideo': ' VideoLibrary.SetMusicVideoDetails', # TODO + 'show': 'VideoLibrary.SetTVShowDetails', # TODO + '': 'AudioLibrary.SetAlbumDetails', # TODO + '': 'AudioLibrary.SetArtistDetails', # TODO + 'track': 'AudioLibrary.SetSongDetails' + } + params = { + 'movie': { + 'movieid': kodiId, + 'playcount': playCount, + 'lastplayed': lastplayed + }, + 'episode': { + 'episodeid': kodiId, + 'playcount': playCount, + 'lastplayed': lastplayed + } + } + query = { + "jsonrpc": "2.0", + "id": 1, + } + query['method'] = method[itemType] + query['params'] = params[itemType] + result = xbmc.executeJSONRPC(json.dumps(query)) + result = json.loads(result) + result = result.get('result') + logMsg("changePlayState", "JSON result was: %s" % result, 1) + + def IfExists(path): """ Kodi's xbmcvfs.exists is broken - it caches the results for directories.