cleaned sync code - added progress

fixed background sync with websocket

resources/lib/LibrarySync.py

@@ -28,13 +28,19 @@ dataPath = os.path.join(addondir,"library")
 movieLibrary = os.path.join(dataPath,'movies')
 tvLibrary = os.path.join(dataPath,'tvshows')
 
+WINDOW = xbmcgui.Window( 10000 )
+
 class LibrarySync():   
         
-    def syncDatabase(self):
+    def FullLibrarySync(self):
+        
+        #show the progress dialog
+        pDialog = xbmcgui.DialogProgressBG()
+        pDialog.create('Emby for Kodi', 'Performing full sync')
         
         #set some variable to check if this is the first run
         addon = xbmcaddon.Addon(id='plugin.video.emby')
-        WINDOW = xbmcgui.Window( 10000 )
+        
 
         startupDone = WINDOW.getProperty("startup") == "done"
         syncInstallRunDone = addon.getSetting("SyncInstallRunDone") == "true"
@@ -55,15 +61,20 @@ class LibrarySync():
                 connection.commit()
             
             # sync movies
-            self.MoviesSync(connection,cursor,True)
+            self.MoviesFullSync(connection,cursor,pDialog)
             #sync Tvshows and episodes
-            self.TvShowsSync(connection,cursor,True)
+            self.TvShowsFullSync(connection,cursor,pDialog)
 
             # set the install done setting
             if(syncInstallRunDone == False and completed):
                 addon = xbmcaddon.Addon(id='plugin.video.emby') #force a new instance of the addon
                 addon.setSetting("SyncInstallRunDone", "true")        
             
+            # Force refresh the library
+            xbmc.executebuiltin("UpdateLibrary(video)")
+            xbmc.executebuiltin("Container.Refresh")
+            xbmc.executebuiltin("Container.Update")
+            
             # set prop to show we have run for the first time
             WINDOW.setProperty("startup", "done")
             
@@ -72,12 +83,12 @@ class LibrarySync():
             utils.logMsg("Sync DB", "syncDatabase Exiting", 0)
             cursor.close()
 
+        if(pDialog != None):
+            pDialog.close()
+        
         return True      
       
-    def MoviesSync(self,connection,cursor,installFirstRun,itemList = []):
+    def MoviesFullSync(self,connection,cursor, pDialog):
-        
-        pDialog = xbmcgui.DialogProgressBG()
-        pDialog.create('Sync DB', 'Sync Movies')
                
         views = ReadEmbyDB().getCollections("movies")
         
@@ -86,18 +97,26 @@ class LibrarySync():
         
         for view in views:
             
-            allMB3Movies = ReadEmbyDB().getMovies(view.get('id'))
+            allEmbyMovies = ReadEmbyDB().getMovies(view.get('id'))
             allKodiMovies = ReadKodiDB().getKodiMovies(connection, cursor)
             
             for kodimovie in allKodiMovies:
                 allKodiMovieIds.append(kodimovie[1])
             
+            total = len(allEmbyMovies) + 1
+            count = 1
+            
             #### PROCESS ADDS AND UPDATES ###
-            for item in allMB3Movies:
+            for item in allEmbyMovies:
                     
                 if not item.get('IsFolder'):                    
                     allEmbyMovieIds.append(item["Id"])
                     
+                    if(pDialog != None):
+                        progressTitle = "Processing " + view.get('title') + " (" + str(count) + " of " + str(total) + ")"
+                        pDialog.update(0, "Emby for Kodi - Running Sync", progressTitle)
+                        count = 1        
+                    
                     kodiMovie = None
                     for kodimovie in allKodiMovies:
                         if kodimovie[1] == item["Id"]:
@@ -106,7 +125,7 @@ class LibrarySync():
                     if kodiMovie == None:
                         WriteKodiDB().addOrUpdateMovieToKodiLibrary(item["Id"],connection, cursor, view.get('title'))
                     else:
-                        if kodiMovie[2] != API().getChecksum(item) or item["Id"] in itemList:
+                        if kodiMovie[2] != API().getChecksum(item):
                             WriteKodiDB().addOrUpdateMovieToKodiLibrary(item["Id"],connection, cursor, view.get('title'))
             
             #### PROCESS DELETES #####
@@ -114,15 +133,10 @@ class LibrarySync():
             for kodiId in allKodiMovieIds:
                 if not kodiId in allEmbyMovieIds:
                     WINDOW.setProperty(kodiId,"deleted")
-                    WriteKodiDB().deleteMovieFromKodiLibrary(kodiId, connection, cursor)
+                    WriteKodiDB().deleteItemFromKodiLibrary(kodiId, connection, cursor)
 
-        if(pDialog != None):
-            pDialog.close()
          
-    def TvShowsSync(self,connection,cursor,installFirstRun,itemList = []):
+    def TvShowsFullSync(self,connection,cursor,pDialog):
-        
-        pDialog = xbmcgui.DialogProgressBG()
-        pDialog.create('Sync DB', 'Sync TV Shows')
                
         views = ReadEmbyDB().getCollections("tvshows")
         
@@ -134,6 +148,9 @@ class LibrarySync():
             allEmbyTvShows = ReadEmbyDB().getTvShows(view.get('id'))
             allKodiTvShows = ReadKodiDB().getKodiTvShows(connection, cursor)
             
+            total = len(allEmbyTvShows) + 1
+            count = 1
+            
             for kodishow in allKodiTvShows:
                 allKodiTvShowIds.append(kodishow[1])
             
@@ -144,6 +161,11 @@ class LibrarySync():
                 if item.get('IsFolder') and item.get('RecursiveItemCount') != 0:                   
                     allEmbyTvShowIds.append(item["Id"])
                     
+                    if(pDialog != None):
+                        progressTitle = "Processing " + view.get('title') + " (" + str(count) + " of " + str(total) + ")"
+                        pDialog.update(0, "Emby for Kodi - Running Sync", progressTitle)
+                        count = 1        
+                    
                     #build a list with all Id's and get the existing entry (if exists) in Kodi DB
                     kodiShow = None
                     for kodishow in allKodiTvShows:
@@ -156,24 +178,21 @@ class LibrarySync():
                     else:
                         kodiId = kodishow[0]
                         # If there are changes to the item, perform a full sync of the item
-                        if kodiShow[2] != API().getChecksum(item) or item["Id"] in itemList:
+                        if kodiShow[2] != API().getChecksum(item):
                             WriteKodiDB().addOrUpdateTvShowToKodiLibrary(item["Id"],connection, cursor, view.get('title'))
                             
                     #### PROCESS EPISODES ######
-                    self.EpisodesSync(connection,cursor,installFirstRun, item["Id"], kodiId, itemList)
+                    self.EpisodesFullSync(connection,cursor,item["Id"], kodiId)
             
             #### TVSHOW: PROCESS DELETES #####
             allEmbyTvShowIds = set(allEmbyTvShowIds)
             for kodiId in allKodiTvShowIds:
                 if not kodiId in allEmbyTvShowIds:
                     WINDOW.setProperty(kodiId,"deleted")
-                    WriteKodiDB().deleteTvShowFromKodiLibrary(kodiId, connection, cursor)
+                    WriteKodiDB().deleteItemFromKodiLibrary(kodiId, connection, cursor)
-
-        if(pDialog != None):
-            pDialog.close()
 
          
-    def EpisodesSync(self,connection,cursor,installFirstRun, embyShowId, kodiShowId, itemList = []):
+    def EpisodesFullSync(self,connection,cursor,embyShowId, kodiShowId):
         
         WINDOW = xbmcgui.Window( 10000 )
         
@@ -202,7 +221,7 @@ class LibrarySync():
                 WriteKodiDB().addOrUpdateEpisodeToKodiLibrary(item["Id"], kodiShowId, connection, cursor)
             else:
                 # If there are changes to the item, perform a full sync of the item
-                if kodiEpisode[2] != API().getChecksum(item) or item["Id"] in itemList:
+                if kodiEpisode[2] != API().getChecksum(item):
                     WriteKodiDB().addOrUpdateEpisodeToKodiLibrary(item["Id"], kodiShowId, connection, cursor)
         
         #### EPISODES: PROCESS DELETES #####
@@ -210,120 +229,58 @@ class LibrarySync():
         for kodiId in allKodiEpisodeIds:
             if (not kodiId in allEmbyEpisodeIds):
                 WINDOW.setProperty(kodiId,"deleted")
-                WriteKodiDB().deleteEpisodeFromKodiLibrary(kodiId, connection, cursor)
+                WriteKodiDB().deleteItemFromKodiLibrary(kodiId, connection, cursor)
     
 
-    def MusicVideosSync(self, fullsync, installFirstRun,connection, cursor):
+    def IncrementalSync(self, itemList):
+        #this will only perform sync for items received by the websocket
         
-        addon = xbmcaddon.Addon(id='plugin.video.emby')
-        WINDOW = xbmcgui.Window( 10000 )
-        pDialog = None
-        
-        try:
-            dbSyncIndication = addon.getSetting("dbSyncIndication")
-                
-            if(installFirstRun or dbSyncIndication == "Dialog Progress"):
-                pDialog = xbmcgui.DialogProgress()
-            elif(dbSyncIndication == "BG Progress"):
         pDialog = xbmcgui.DialogProgressBG()
+        pDialog.create('Emby for Kodi', 'Performing incremental sync...')
         
-            if(pDialog != None):
+        connection = utils.KodiSQL()
-                pDialog.create('Sync DB', 'Sync DB')
+        cursor = connection.cursor()
         
-            allEmbyMusicVideoIds = list()
+        #### PROCESS MOVIES ####
-
+        views = ReadEmbyDB().getCollections("movies")
-            progressTitle = ""
+        for view in views:
-            
+            allEmbyMovies = ReadEmbyDB().getMovies(view.get('id'), itemList)
-            #process new musicvideos
+            for item in allEmbyMovies:
-            allMB3MusicVideos = ReadEmbyDB().getMusicVideos(True, fullsync)
-            allKodiIds = set(ReadKodiDB().getKodiMusicVideoIds(True))
-        
-            if(self.ShouldStop(pDialog)):
-                return False            
-        
-            if(allMB3MusicVideos == None):
-                return False
-        
-            if(pDialog != None):
-                progressTitle = "Sync DB : Processing Musicvideos"
-                pDialog.update(0, progressTitle)
-                total = len(allMB3MusicVideos) + 1
-                count = 1
-            
-            for item in allMB3MusicVideos:
                     
                 if not item.get('IsFolder'):                    
-                    allEmbyMusicVideoIds.append(item["Id"])
+                    WriteKodiDB().addOrUpdateMovieToKodiLibrary(item["Id"],connection, cursor, view.get('title'))
                     
-                    if item["Id"] not in allKodiIds:
+        #### PROCESS TV SHOWS ####
-                        WriteKodiDB().addMusicVideoToKodiLibrary(item, connection, cursor)
+        views = ReadEmbyDB().getCollections("tvshows")              
+        for view in views:
+            allEmbyTvShows = ReadEmbyDB().getTvShows(view.get('id'),itemList)
+            for item in allEmbyTvShows:
+                if item.get('IsFolder') and item.get('RecursiveItemCount') != 0:                   
+                    kodiId = WriteKodiDB().addOrUpdateTvShowToKodiLibrary(item["Id"],connection, cursor, view.get('title'))
                     
-                    if(self.ShouldStop(pDialog)):
+        #### PROCESS EPISODES ######
-                        return False
+        for item in itemList:
                 
-                    # update progress bar
+            MBitem = ReadEmbyDB().getItem(item)
-                    if(pDialog != None):
-                        percentage = int(((float(count) / float(total)) * 100))
-                        pDialog.update(percentage, progressTitle, "Adding Musicvideo: " + str(count))
-                        count += 1
             
-            if(self.ShouldStop(pDialog)):
+            if MBitem["Type"] == "Episode":
-                return False
                 
-            if(pDialog != None):
+                #get the tv show
-                progressTitle = "Sync DB : Processing musicvideos"
+                cursor.execute("SELECT kodi_id FROM emby WHERE media_type='tvshow' AND emby_id=?", (MBitem["SeriesId"],))
-                pDialog.update(0, progressTitle, "")
+                result = cursor.fetchall()
-                total = len(allMB3MusicVideos) + 1
+                if result:
-                count = 1                    
+                    kodi_show_id = result[0]
-            
-            #process updates
-            allKodiMusicVideos = ReadKodiDB().getKodiMusicVideos(True)
-            for item in allMB3MusicVideos:
-                
-                if not item.get('IsFolder'):
-                    
-                    if allKodiMusicVideos != None:
-                        kodimusicvideo = allKodiMusicVideos.get(item["Id"], None)
                 else:
-                        kodimusicvideo = None
+                    kodi_show_id = None
                 
-                    if(kodimusicvideo != None):
+                if kodi_show_id:
-                        WriteKodiDB().updateMusicVideoToKodiLibrary_Batched(item, kodimusicvideo)
+                    WriteKodiDB().addOrUpdateEpisodeToKodiLibrary(item["Id"], kodi_show_id, connection, cursor)
-                    
-                    if(self.ShouldStop(pDialog)):
-                        return False
-                
-                    # update progress bar
-                    if(pDialog != None):
-                        percentage = int(((float(count) / float(total)) * 100))
-                        pDialog.update(percentage, progressTitle, "Updating MusicVideo: " + str(count))
-                        count += 1
         
         
-            if(pDialog != None):
+        cursor.close()
-                progressTitle = "Removing Deleted Items"
-                pDialog.update(0, progressTitle, "")
-            
-            if(self.ShouldStop(pDialog)):
-                return False            
-            
-            # process any deletes only at fullsync
-            if fullsync:
-                allKodiIds = ReadKodiDB().getKodiMusicVideoIds(True)
-                allEmbyMusicVideoIds = set(allEmbyMusicVideoIds)
-                for kodiId in allKodiIds:
-                    if not kodiId in allEmbyMusicVideoIds:
-                        WriteKodiDB().deleteMusicVideoFromKodiLibrary(kodiId)
-            
-            if(self.ShouldStop(pDialog)):
-                return False            
-            
-        finally:
         if(pDialog != None):
             pDialog.close()
     
-        return True  
-
     def ShouldStop(self, prog):
         
         if(prog != None and type(prog) == xbmcgui.DialogProgress):
@@ -333,7 +290,6 @@ class LibrarySync():
         if(xbmc.Player().isPlaying() or xbmc.abortRequested):
             return True
 
-        WINDOW = xbmcgui.Window( 10000 )
         if(WINDOW.getProperty("SyncDatabaseShouldStop") == "true"):
             return True
 

resources/lib/ReadEmbyDB.py

@@ -14,7 +14,7 @@ addon = xbmcaddon.Addon(id='plugin.video.emby')
 
 class ReadEmbyDB():   
     
-    def getMovies(self, id):
+    def getMovies(self, id, itemList = []):
         
         result = None
         doUtils = DownloadUtils()
@@ -30,21 +30,23 @@ class ReadEmbyDB():
         if (jsonData[u'Items'] != ""):
             result = jsonData[u'Items']
             
+        # Work around to only return items from the given list
+        if (result != None and len(result) > 0 and len(itemList) > 0):
+            newResult = []
+            for item in result:
+                if (item[u'Id'] in itemList):
+                    newResult.append(item)
+            result = newResult
+            
         return result
 
-    def getMusicVideos(self, fullinfo = False, fullSync = True):
+    def getMusicVideos(self, id, itemList = []):
         
         result = None
         doUtils = DownloadUtils()
 
-        if not fullSync:
+        #only get basic info for our sync-compares
-            sortstring = "&Limit=20&SortBy=DateCreated"
-        else:
         sortstring = "&SortBy=SortName"
-        
-        if fullinfo:
-            url = "{server}/mediabrowser/Users/{UserId}/items?%s&Fields=Path,Genres,SortName,Studios,Writer,ProductionYear,Taglines,CommunityRating,OfficialRating,CumulativeRunTimeTicks,Metascore,AirTime,DateCreated,MediaStreams,People,Overview&Recursive=true&SortOrder=Descending&IncludeItemTypes=MusicVideo&format=json&ImageTypeLimit=1" % sortstring
-        else:
         url = "{server}/mediabrowser/Users/{UserId}/items?%s&Fields=CumulativeRunTimeTicks&Recursive=true&SortOrder=Descending&IncludeItemTypes=MusicVideo&CollapseBoxSetItems=false&format=json&ImageTypeLimit=1" % sortstring
         
         jsonData = doUtils.downloadUrl(url)
@@ -54,6 +56,14 @@ class ReadEmbyDB():
         if (jsonData[u'Items'] != ""):
             result = jsonData[u'Items']
             
+        # Work around to only return items from the given list
+        if (result != None and len(result) > 0 and len(itemList) > 0):
+            newResult = []
+            for item in result:
+                if (item[u'Id'] in itemList):
+                    newResult.append(item)
+            result = newResult
+
         return result
         
     def getItem(self, id):
@@ -82,7 +92,7 @@ class ReadEmbyDB():
 
         return result
     
-    def getTvShows(self, id):
+    def getTvShows(self, id, itemList = []):
         
         result = None
         doUtils = DownloadUtils()
@@ -98,6 +108,14 @@ class ReadEmbyDB():
         if (jsonData[u'Items'] != ""):
             result = jsonData[u'Items']
             
+        # Work around to only return items from the given list
+        if (result != None and len(result) > 0 and len(itemList) > 0):
+            newResult = []
+            for item in result:
+                if (item[u'Id'] in itemList):
+                    newResult.append(item)
+            result = newResult
+
         return result
     
     def getTVShowSeasons(self, tvShowId):
@@ -116,7 +134,7 @@ class ReadEmbyDB():
 
         return result
     
-    def getEpisodes(self, showId):
+    def getEpisodes(self, showId, itemList = []):
         
         result = None
         doUtils = DownloadUtils()  
@@ -130,6 +148,14 @@ class ReadEmbyDB():
         if (jsonData[u'Items'] != ""):
             result = jsonData[u'Items']
             
+        # Work around to only return items from the given list
+        if (result != None and len(result) > 0 and len(itemList) > 0):
+            newResult = []
+            for item in result:
+                if (item[u'Id'] in itemList):
+                    newResult.append(item)
+            result = newResult
+                
         return result
     
     def getLatestEpisodes(self, fullinfo = False, itemList = []):

resources/lib/WebSocketClient.py

@@ -20,6 +20,7 @@ from DownloadUtils import DownloadUtils
 from PlaybackUtils import PlaybackUtils
 from LibrarySync import LibrarySync
 from WriteKodiDB import WriteKodiDB
+from ReadEmbyDB import ReadEmbyDB
 
 pendingUserDataList = []
 pendingItemsToRemove = []
@@ -183,31 +184,24 @@ class WebSocketThread(threading.Thread):
             connection = utils.KodiSQL()
             cursor = connection.cursor()
             self.logMsg("Message : Doing LibraryChanged : Items Removed : Calling deleteEpisodeFromKodiLibraryByMbId: " + item, 0)
-            WriteKodiDB().deleteEpisodeFromKodiLibrary(item, connection, cursor)
+            WriteKodiDB().deleteItemFromKodiLibrary(item, connection, cursor)
-            self.logMsg("Message : Doing LibraryChanged : Items Removed : Calling deleteMovieFromKodiLibrary: " + item, 0)
-            WriteKodiDB().deleteMovieFromKodiLibrary(item, connection, cursor)
-            self.logMsg("Message : Doing LibraryChanged : Items Removed : Calling deleteMusicVideoFromKodiLibrary: " + item, 0)
-            WriteKodiDB().deleteMusicVideoFromKodiLibrary(item, connection, cursor)
             cursor.close()
 
     def update_items(self, itemsToUpdate):
         # doing adds and updates
         if(len(itemsToUpdate) > 0):
             self.logMsg("Message : Doing LibraryChanged : Processing Added and Updated : " + str(itemsToUpdate), 0)
-            connection = utils.KodiSQL()
+            LibrarySync().IncrementalSync(itemsToUpdate)
-            cursor = connection.cursor()
-            LibrarySync().MoviesSync(connection, cursor, False, itemsToUpdate)
-            LibrarySync().TvShowsSync(connection, cursor, False, itemsToUpdate)
-            cursor.close()
 
     def user_data_update(self, userDataList):
-    
+        itemsToUpdate = list()
         for userData in userDataList:
-            self.logMsg("Message : Doing UserDataChanged : UserData : " + str(userData), 0)
             itemId = userData.get("ItemId")
             if(itemId != None):
-                self.logMsg("Message : Doing UserDataChanged : calling updatePlayCount with ID : " + str(itemId), 0)
+                itemsToUpdate.append(itemId)
-                LibrarySync().updatePlayCount(itemId)
+        if(len(itemsToUpdate) > 0):
+            self.logMsg("Message : Doing UserDataChanged : Processing Updated : " + str(itemsToUpdate), 0)
+            LibrarySync().IncrementalSync(itemsToUpdate)
                 
     def on_error(self, ws, error):
         self.logMsg("Error : " + str(error))

resources/lib/WriteKodiDB.py

@@ -535,31 +535,25 @@ class WriteKodiDB():
         #commit changes
         connection.commit()
 
-    def deleteMovieFromKodiLibrary(self, id, connection, cursor ):
+    def deleteItemFromKodiLibrary(self, id, connection, cursor ):
+        
+        cursor.execute("SELECT kodi_id, media_type FROM emby WHERE emby_id=?", (id,))
+        result = cursor.fetchone()
+        if result:
+            kodi_id = result[0]
+            media_type = result[1]
+            if media_type == "movie":
                 utils.logMsg("deleting movie from Kodi library --> ",id)
-        cursor.execute("SELECT kodi_id FROM emby WHERE emby_id=?", (id,))
-        kodi_id = cursor.fetchone()[0]
                 cursor.execute("DELETE FROM movie WHERE idMovie = ?", (kodi_id,))
-        connection.commit()
+            if media_type == "episode":
- 
-    def deleteMusicVideoFromKodiLibrary(self, id ):
-        utils.logMsg("deleting musicvideo from Kodi library",id)
-        kodiItem = ReadKodiDB().getKodiMusicVideo(id)
-        if kodiItem != None:
-            xbmc.executeJSONRPC('{"jsonrpc": "2.0", "method": "VideoLibrary.RemoveMusicVideo", "params": { "musicvideoid": %i}, "id": 1 }' %(kodiItem["musicvideoid"]))
-         
-    def deleteEpisodeFromKodiLibrary(self, id, connection, cursor ):
                 utils.logMsg("deleting episode from Kodi library --> ",id)
-        cursor.execute("SELECT kodi_id FROM emby WHERE emby_id=?", (id,))
-        kodi_id = cursor.fetchone()[0]
                 cursor.execute("DELETE FROM episode WHERE idEpisode = ?", (kodi_id,))
-        connection.commit()
+            if media_type == "tvshow":
-                      
-    def deleteTVShowFromKodiLibrary(self, id, connection, cursor):
                 utils.logMsg("deleting tvshow from Kodi library --> ",id)
-        cursor.execute("SELECT kodi_id FROM emby WHERE emby_id=?", (id,))
-        kodi_id = cursor.fetchone()[0]
                 cursor.execute("DELETE FROM tvshow WHERE idShow = ?", (kodi_id,))
+            if media_type == "musicvideo":
+                utils.logMsg("deleting musicvideo from Kodi library --> ",id)
+                cursor.execute("DELETE FROM musicvideo WHERE idMVideo = ?", (kodi_id,))
             connection.commit()
      
     def updateSeasons(self,embyTvShowId, kodiTvShowId, connection, cursor):

service.py

@@ -116,11 +116,9 @@ class Service():
                     #full sync
                     if(startupComplete == False):
                         self.logMsg("Doing_Db_Sync: syncDatabase (Started)")
-                        libSync = librarySync.syncDatabase()
+                        libSync = librarySync.FullLibrarySync()
                         self.logMsg("Doing_Db_Sync: syncDatabase (Finished) " + str(libSync))
 
-                        # Force refresh newly set thumbnails
-                        xbmc.executebuiltin("UpdateLibrary(video)")
                         if(libSync):
                             startupComplete = True
                     else: