Redesign fast sync

resources/lib/PlexAPI.py

@@ -1476,36 +1476,6 @@ class API():
         """
         return self.part
 
-    def DateToKodi(self, stamp):
-        """
-        converts a Unix time stamp (seconds passed sinceJanuary 1 1970) to a
-        propper, human-readable time stamp used by Kodi
-
-        Output: Y-m-d h:m:s = 2009-04-05 23:16:04
-        """
-        # DATEFORMAT = xbmc.getRegion('dateshort')
-        # TIMEFORMAT = xbmc.getRegion('meridiem')
-        # date_time = time.localtime(stamp)
-        # if DATEFORMAT[1] == 'd':
-        #     localdate = time.strftime('%d-%m-%Y', date_time)
-        # elif DATEFORMAT[1] == 'm':
-        #     localdate = time.strftime('%m-%d-%Y', date_time)
-        # else:
-        #     localdate = time.strftime('%Y-%m-%d', date_time)
-        # if TIMEFORMAT != '/':
-        #     localtime = time.strftime('%I:%M%p', date_time)
-        # else:
-        #     localtime = time.strftime('%H:%M', date_time)
-        # return localtime + '  ' + localdate
-        try:
-            # DATEFORMAT = xbmc.getRegion('dateshort')
-            # TIMEFORMAT = xbmc.getRegion('meridiem')
-            date_time = time.localtime(float(stamp))
-            localdate = time.strftime('%Y-%m-%d %H:%M:%S', date_time)
-        except:
-            localdate = None
-        return localdate
-
     def getType(self):
         """
         Returns the type of media, e.g. 'movie' or 'clip' for trailers
@@ -1543,7 +1513,7 @@ class API():
         """
         Returns the date when this library item was created
         """
-        return self.DateToKodi(self.item.attrib.get('addedAt', None))
+        return utils.DateToKodi(self.item.attrib.get('addedAt', None))
 
     def getUserData(self):
         """
@@ -1576,7 +1546,7 @@ class API():
             played = True
 
         try:
-            lastPlayedDate = self.DateToKodi(int(item['lastViewedAt']))
+            lastPlayedDate = utils.DateToKodi(int(item['lastViewedAt']))
         except:
             lastPlayedDate = None
 

resources/lib/PlexFunctions.py

@@ -3,6 +3,7 @@ from urllib import urlencode
 from ast import literal_eval
 from urlparse import urlparse, parse_qs
 import re
+import time
 
 from xbmcaddon import Addon
 
@@ -371,3 +372,22 @@ def PMSHttpsEnabled(url):
         # couldn't get an xml - switch to http traffic
         logMsg('PMSHttpsEnabled', 'PMS on %s talks HTTPS' % url, 1)
         return False
+
+
+def scrobble(ratingKey, state):
+    """
+    Tells the PMS to set an item's watched state to state="watched" or
+    state="unwatched"
+    """
+    args = {
+        'key': ratingKey,
+        'identifier': 'com.plexapp.plugins.library'
+    }
+    if state == "watched":
+        url = "{server}/:/scrobble?" + urlencode(args)
+    elif state == "unwatched":
+        url = "{server}/:/unscrobble?" + urlencode(args)
+    else:
+        return
+    downloadutils.DownloadUtils().downloadUrl(url, type="GET")
+    logMsg("Toggled watched state for Plex item %s" % ratingKey, 1)

resources/lib/embydb_functions.py

@@ -152,6 +152,22 @@ class Embydb_Functions():
         ))
         self.embycursor.execute(query, (viewid,))
 
+    def getItem_byFileId(self, fileId):
+        """
+        Returns the Plex itemId by using the Kodi fileId
+        """
+        query = ' '.join((
+            "SELECT emby_id",
+            "FROM emby",
+            "WHERE kodi_fileid = ?"
+        ))
+        try:
+            self.embycursor.execute(query, (fileId,))
+            item = self.embycursor.fetchone()[0]
+            return item
+        except:
+            return None
+
     def getItem_byId(self, embyid):
 
         embycursor = self.embycursor
@@ -183,6 +199,23 @@ class Embydb_Functions():
 
         return items
 
+    def getPlexId(self, kodiid):
+        """
+        Returns the Plex ID usind the Kodiid. Result:
+            (Plex Id, Parent's Plex Id)
+        """
+        query = ' '.join((
+            "SELECT emby_id, parent_id",
+            "FROM emby",
+            "WHERE kodi_id = ?"
+        ))
+        try:
+            self.embycursor.execute(query, (kodiid))
+            item = self.embycursor.fetchone()
+            return item
+        except:
+            return None
+
     def getItem_byKodiId(self, kodiid, mediatype):
 
         embycursor = self.embycursor

resources/lib/kodidb_functions.py

@@ -17,7 +17,7 @@ class GetKodiDB():
                do stuff with kodi_db
 
     Parameters:
-        itemType:       itemtype for Kodi DB, e.g. 'video'
+        itemType:       itemtype for Kodi DB, e.g. 'video', 'music'
 
     On exiting "with" (no matter what), commits get automatically committed
     and the db gets closed
@@ -701,6 +701,68 @@ class Kodidb_Functions():
                 )
                 cursor.execute(query, (fileid, 2, subtitletrack))
 
+    def getResumes(self):
+        """
+        VIDEOS
+
+        Returns all Kodi idFile that have a resume point set (not unwatched
+        ones or items that have already been completely watched)
+        """
+        cursor = self.cursor
+
+        query = ' '.join((
+            "SELECT idFile",
+            "FROM bookmark"
+        ))
+        try:
+            rows = cursor.execute(query)
+        except:
+            return []
+        ids = []
+        for row in rows:
+            ids.append(row[0])
+        return ids
+
+    def getUnplayedMusicItems(self):
+        """
+        MUSIC
+
+        Returns all Kodi Item idFile that have not yet been completely played
+        """
+        query = ' '.join((
+            "SELECT idPath",
+            "FROM song",
+            "WHERE iTimesPlayed IS NULL OR iTimesPlayed = ''"
+        ))
+        try:
+            rows = self.cursor.execute(query)
+        except:
+            return []
+        ids = []
+        for row in rows:
+            ids.append(row[0])
+        return ids
+
+    def getUnplayedItems(self):
+        """
+        VIDEOS
+
+        Returns all Kodi Item idFile that have not yet been completely played
+        """
+        query = ' '.join((
+            "SELECT idFile",
+            "FROM files",
+            "WHERE playCount IS NULL OR playCount = ''"
+        ))
+        try:
+            rows = self.cursor.execute(query)
+        except:
+            return []
+        ids = []
+        for row in rows:
+            ids.append(row[0])
+        return ids
+
     def addPlaystate(self, fileid, resume_seconds, total_seconds, playcount, dateplayed):
         
         cursor = self.cursor

resources/lib/kodimonitor.py

@@ -11,8 +11,7 @@ import downloadutils
 import embydb_functions as embydb
 import playbackutils as pbutils
 import utils
-
+from PlexFunctions import scrobble
-from urllib import urlencode
 
 ###############################################################################
 
@@ -144,16 +143,11 @@ class KodiMonitor(xbmc.Monitor):
                         utils.window('emby_skipWatched%s' % itemid, clear=True)
                     else:
                         # notify the server
-                        args = {'key': itemid,
-                                'identifier': 'com.plexapp.plugins.library'}
                         if playcount != 0:
-                            url = "{server}/:/scrobble?" + urlencode(args)
+                            scrobble(itemid, 'watched')
-                            doUtils.downloadUrl(url, type="GET")
-                            self.logMsg("Mark as watched for itemid: %s" % itemid, 1)
                         else:
-                            url = "{server}/:/unscrobble?" + urlencode(args)
+                            scrobble(itemid, 'unwatched')
-                            doUtils.downloadUrl(url, type="GET")
+
-                            self.logMsg("Mark as unwatched for itemid: %s" % itemid, 1)
                 finally:
                     embycursor.close()
 

resources/lib/librarysync.py

@@ -238,6 +238,12 @@ class LibrarySync(Thread):
         self.enableBackgroundSync = True if utils.settings(
             'enableBackgroundSync') == "true" else False
 
+        # Time offset between Kodi and PMS in seconds (=Koditime - PMStime)
+        self.timeoffset = 0
+        # Time in seconds to look into the past when looking for PMS changes
+        # (safety margin - the larger, the more items we need to process)
+        self.syncPast = 30
+
         Thread.__init__(self)
 
     def showKodiNote(self, message, forced=False, icon="plex"):
@@ -265,6 +271,102 @@ class LibrarySync(Thread):
                 time=7000,
                 sound=True)
 
+    def syncPMStime(self):
+        """
+        PMS does not provide a means to get a server timestamp. This is a work-
+        around.
+        """
+        self.logMsg('Synching time with PMS server', 0)
+        # Find a PMS item where we can toggle the view state to enforce a
+        # change in lastViewedAt
+        with kodidb.GetKodiDB('video') as kodi_db:
+            unplayedIds = kodi_db.getUnplayedItems()
+            resumeIds = kodi_db.getResumes()
+        self.logMsg('resumeIds: %s' % resumeIds, 1)
+
+        plexId = False
+        for unplayedId in unplayedIds:
+            if unplayedId not in resumeIds:
+                # Found an item we can work with!
+                kodiId = unplayedId
+                self.logMsg('Found kodiId: %s' % kodiId, 1)
+                # Get Plex ID using the Kodi ID
+                with embydb.GetEmbyDB() as emby_db:
+                    plexId = emby_db.getItem_byFileId(kodiId)
+                if plexId:
+                    self.logMsg('Found plexId: %s' % plexId, 1)
+                    break
+
+        # Try getting a music item if we did not find a video item
+        if not plexId:
+            self.logMsg("Could not find a video item to sync time with", 0)
+            with kodidb.GetKodiDB('music') as kodi_db:
+                unplayedIds = kodi_db.getUnplayedMusicItems()
+                # We don't care about resuming songs in the middle
+            for unplayedId in unplayedIds:
+                # Found an item we can work with!
+                kodiId = unplayedId
+                self.logMsg('Found kodiId: %s' % kodiId, 1)
+                # Get Plex ID using the Kodi ID
+                with embydb.GetEmbyDB() as emby_db:
+                    plexId = emby_db.getItem_byFileId(kodiId)
+                if plexId:
+                    self.logMsg('Found plexId: %s' % plexId, 1)
+                    break
+            else:
+                self.logMsg("Could not find an item to sync time with", -1)
+                self.logMsg("Aborting PMS-Kodi time sync", -1)
+                return
+
+        # Get the Plex item's metadata
+        xml = PlexFunctions.GetPlexMetadata(plexId)
+        if not xml:
+            self.logMsg("Could not download metadata, aborting time sync", -1)
+            return
+        libraryId = xml[0].attrib['librarySectionID']
+        # Get a PMS timestamp to start our question with
+        timestamp = xml[0].attrib.get('lastViewedAt')
+        if not timestamp:
+            timestamp = xml[0].attrib.get('updatedAt')
+            self.logMsg('Using items updatedAt=%s' % timestamp, 1)
+            if not timestamp:
+                timestamp = xml[0].attrib.get('addedAt')
+                self.logMsg('Using items addedAt=%s' % timestamp, 1)
+        # Set the timer
+        koditime = utils.getUnixTimestamp()
+        # Toggle watched state
+        PlexFunctions.scrobble(plexId, 'watched')
+        # Let the PMS process this first!
+        xbmc.sleep(2000)
+        # Get all PMS items to find the item we changed
+        items = PlexFunctions.GetAllPlexLeaves(libraryId,
+                                               lastViewedAt=timestamp)
+        # Toggle watched state back
+        PlexFunctions.scrobble(plexId, 'unwatched')
+        # Get server timestamp for this change
+        plextime = None
+        for item in items:
+            if item.attrib['ratingKey'] == plexId:
+                plextime = item.attrib.get('lastViewedAt')
+                break
+        if not plextime:
+            self.logMsg("Could not set the items watched state, abort", -1)
+            return
+
+        # Calculate time offset Kodi-PMS
+        self.timeoffset = int(koditime) - int(plextime)
+        self.logMsg("Time offset Koditime - PMStime in seconds: %s"
+                    % str(self.timeoffset), 0)
+
+    def getPMSfromKodiTime(self, koditime):
+        """
+        Uses self.timeoffset to return the PMS time for a given Kodi timestamp
+        (in unix time)
+
+        Feed with integers
+        """
+        return koditime - self.timeoffset
+
     def resetProcessedItems(self):
         """
         Resets the list of PMS items that we have already processed
@@ -279,7 +381,7 @@ class LibrarySync(Thread):
             'track': {}
         }
 
-    def getFastUpdateList(self, xml, plexType, viewName, viewId, update=True):
+    def getFastUpdateList(self, xml, plexType, viewName, viewId):
         """
         THIS METHOD NEEDS TO BE FAST! => e.g. no API calls
 
@@ -305,8 +407,6 @@ class LibrarySync(Thread):
             self.allPlexElementsId      APPENDED(!!) dict
                 = {itemid: checksum}
         """
-        # Updated items are prefered over userdata updates!
-        if update:
         # Needs to call other methods than if we're only updating userdata
         for item in xml:
             itemId = item.attrib.get('ratingKey')
@@ -321,7 +421,9 @@ class LibrarySync(Thread):
             # specific item
             res = self.processed[plexType].get(itemId)
             if res:
-                    if res == (lastViewedAt, updatedAt):
+                # Only look at the updatedAt flag!
+                # tuple: (lastViewedAt, updatedAt)
+                if res[1] == updatedAt:
                     # Nothing to update, we have already processed this
                     # item
                     continue
@@ -338,38 +440,10 @@ class LibrarySync(Thread):
             })
             # And safe to self.processed:
             self.processed[plexType][itemId] = (lastViewedAt, updatedAt)
-        else:
+        # Quickly log
-            # Needs to call other methods than if we're only updating userdata
+        if self.updatelist:
-            for item in xml:
+            self.logMsg('fastSync updatelist: %s' % self.updatelist, 1)
-                itemId = item.attrib.get('ratingKey')
+            self.logMsg('fastSync processed list: %s' % self.processed, 1)
-                # Skipping items 'title=All episodes' without a 'ratingKey'
-                if not itemId:
-                    continue
-
-                lastViewedAt = item.attrib.get('lastViewedAt')
-                updatedAt = item.attrib.get('updatedAt')
-
-                # returns the tuple (lastViewedAt, updatedAt) for the
-                # specific item
-                res = self.processed[plexType].get(itemId)
-                if res:
-                    if res == (lastViewedAt, updatedAt):
-                        # Nothing to update, we have already processed this
-                        # item
-                        continue
-                title = item.attrib.get('title', 'Missing Title Name')
-                # We need to process this:
-                self.updatelist.append({
-                    'itemId': itemId,
-                    'itemType': PlexFunctions.GetItemClassFromType(
-                        plexType),
-                    'method': 'updateUserdata',
-                    'viewName': viewName,
-                    'viewId': viewId,
-                    'title': title
-                })
-                # And safe to self.processed:
-                self.processed[plexType][itemId] = (lastViewedAt, updatedAt)
 
     def fastSync(self):
         """
@@ -392,8 +466,7 @@ class LibrarySync(Thread):
                 }
         }
         """
-        self.compare = True
+        # Get last sync time and look a bit in the past (safety margin)
-        # Get last sync time
         lastSync = self.lastSync - self.syncPast
         # Set new timestamp NOW because sync might take a while
         self.saveLastSync()
@@ -408,8 +481,8 @@ class LibrarySync(Thread):
         for view in self.views:
             self.updatelist = []
             # Get items per view
-            items = PlexFunctions.GetAllPlexLeaves(view['id'],
+            items = PlexFunctions.GetAllPlexLeaves(
-                                                   updatedAt=lastSync)
+                view['id'], updatedAt=self.getPMSfromKodiTime(lastSync))
             # Just skip if something went wrong
             if not items:
                 continue
@@ -439,14 +512,11 @@ class LibrarySync(Thread):
         # We don't need to refresh the Kodi library for deltas!!
         # Start with an empty ElementTree and attach items to update
         movieupdate = False
-        movieXML = etree.Element('root')
         episodeupdate = False
-        episodeXML = etree.Element('root')
         songupdate = False
-        musicXML = etree.Element('root')
         for view in self.views:
-            items = PlexFunctions.GetAllPlexLeaves(view['id'],
+            items = PlexFunctions.GetAllPlexLeaves(
-                                                   lastViewedAt=lastSync)
+                view['id'], lastViewedAt=self.getPMSfromKodiTime(lastSync))
             for item in items:
                 itemId = item.attrib.get('ratingKey')
                 # Skipping items 'title=All episodes' without a 'ratingKey'
@@ -460,21 +530,32 @@ class LibrarySync(Thread):
                 # specific item
                 res = self.processed[plexType].get(itemId)
                 if res:
-                    if res == (lastViewedAt, updatedAt):
+                    # Only look at lastViewedAt
+                    if res[0] == lastViewedAt:
                         # Nothing to update, we have already processed this
                         # item
                         continue
                 if plexType == 'movie':
-                    movieXML.append(item)
                     movieupdate = True
+                    try:
+                        movieXML.append(item)
+                    except:
+                        movieXML = etree.Element('root')
+                        movieXML.append(item)
                 elif plexType == 'episode':
-                    episodeXML.append(item)
                     episodeupdate = True
+                    try:
+                        episodeXML.append(item)
+                    except:
+                        episodeXML = etree.Element('root')
+                        episodeXML.append(item)
                 elif plexType == 'track':
-                    musicXML.append(item)
                     songupdate = True
-                else:
+                    try:
-                    self.logMsg('Unknown plex type %s' % plexType, -1)
+                        musicXML.append(item)
+                    except:
+                        musicXML = etree.Element('root')
+                        musicXML.append(item)
                 # And safe to self.processed:
                 self.processed[plexType][itemId] = (lastViewedAt, updatedAt)
 
@@ -1382,6 +1463,8 @@ class LibrarySync(Thread):
                 log("Db version: %s" % settings('dbCreatedWithVersion'), 0)
                 log("Initial start-up full sync starting", 0)
                 librarySync = fullSync(manualrun=True)
+                # Initialize time offset Kodi - PMS
+                self.syncPMStime()
                 window('emby_dbScan', clear=True)
                 if librarySync:
                     log("Initial start-up full sync successful", 0)
@@ -1440,9 +1523,11 @@ class LibrarySync(Thread):
                 elif enableBackgroundSync:
                     # Run full lib scan approx every 30min
                     if count >= 1800:
+                        count = 0
                         # Also reset self.processed, just in case
                         self.resetProcessedItems()
-                        count = 0
+                        # Recalculate time offset Kodi - PMS
+                        self.syncPMStime()
                         window('emby_dbScan', value="true")
                         log('Running background full lib scan', 0)
                         fullSync(manualrun=True)

resources/lib/utils.py

@@ -19,12 +19,88 @@ import xbmcaddon
 import xbmcgui
 import xbmcvfs
 
-
 ###############################################################################
 
 addonName = xbmcaddon.Addon().getAddonInfo('name')
 
 
+def DateToKodi(stamp):
+        """
+        converts a Unix time stamp (seconds passed sinceJanuary 1 1970) to a
+        propper, human-readable time stamp used by Kodi
+
+        Output: Y-m-d h:m:s = 2009-04-05 23:16:04
+        """
+        # DATEFORMAT = xbmc.getRegion('dateshort')
+        # TIMEFORMAT = xbmc.getRegion('meridiem')
+        # date_time = time.localtime(stamp)
+        # if DATEFORMAT[1] == 'd':
+        #     localdate = time.strftime('%d-%m-%Y', date_time)
+        # elif DATEFORMAT[1] == 'm':
+        #     localdate = time.strftime('%m-%d-%Y', date_time)
+        # else:
+        #     localdate = time.strftime('%Y-%m-%d', date_time)
+        # if TIMEFORMAT != '/':
+        #     localtime = time.strftime('%I:%M%p', date_time)
+        # else:
+        #     localtime = time.strftime('%H:%M', date_time)
+        # return localtime + '  ' + localdate
+        try:
+            # DATEFORMAT = xbmc.getRegion('dateshort')
+            # TIMEFORMAT = xbmc.getRegion('meridiem')
+            date_time = time.localtime(float(stamp))
+            localdate = time.strftime('%Y-%m-%d %H:%M:%S', date_time)
+        except:
+            localdate = None
+        return localdate
+
+
+def changePlayState(itemType, kodiId, playCount, lastplayed):
+    """
+    YET UNUSED
+
+    kodiId: int or str
+    playCount: int or str
+    lastplayed: str or int unix timestamp
+    """
+    logMsg("changePlayState", "start", 1)
+    lastplayed = DateToKodi(lastplayed)
+
+    kodiId = int(kodiId)
+    playCount = int(playCount)
+    method = {
+        'movie': ' VideoLibrary.SetMovieDetails',
+        'episode': 'VideoLibrary.SetEpisodeDetails',
+        'musicvideo': ' VideoLibrary.SetMusicVideoDetails',  # TODO
+        'show': 'VideoLibrary.SetTVShowDetails',  # TODO
+        '': 'AudioLibrary.SetAlbumDetails',  # TODO
+        '': 'AudioLibrary.SetArtistDetails',  # TODO
+        'track': 'AudioLibrary.SetSongDetails'
+    }
+    params = {
+        'movie': {
+            'movieid': kodiId,
+            'playcount': playCount,
+            'lastplayed': lastplayed
+        },
+        'episode': {
+            'episodeid': kodiId,
+            'playcount': playCount,
+            'lastplayed': lastplayed
+        }
+    }
+    query = {
+        "jsonrpc": "2.0",
+        "id": 1,
+    }
+    query['method'] = method[itemType]
+    query['params'] = params[itemType]
+    result = xbmc.executeJSONRPC(json.dumps(query))
+    result = json.loads(result)
+    result = result.get('result')
+    logMsg("changePlayState", "JSON result was: %s" % result, 1)
+
+
 def IfExists(path):
     """
     Kodi's xbmcvfs.exists is broken - it caches the results for directories.